├── MetaNN
    ├── Release
    │   └── .d
    ├── facilities
    │   ├── null_param.h
    │   └── traits.h
    ├── policies
    │   ├── policy_macro_end.h
    │   ├── inject_policy.h
    │   ├── policy_container.h
    │   ├── policy_macro_begin.h
    │   ├── change_policy.h
    │   └── policy_selector.h
    ├── operators
    │   ├── facilities
    │   │   ├── oper_seq.h
    │   │   ├── traits.h
    │   │   ├── tags.h
    │   │   ├── category_cal.h
    │   │   └── organizer.h
    │   └── collapse.h
    ├── layers
    │   ├── facilities
    │   │   ├── common_io.h
    │   │   ├── traits.h
    │   │   ├── policies.h
    │   │   └── interface_fun.h
    │   ├── compose
    │   │   ├── linear_layer.h
    │   │   └── single_layer.h
    │   ├── elementary
    │   │   ├── add_layer.h
    │   │   ├── tanh_layer.h
    │   │   ├── sigmoid_layer.h
    │   │   ├── element_mul_layer.h
    │   │   ├── abs_layer.h
    │   │   ├── softmax_layer.h
    │   │   └── interpolate_layer.h
    │   └── cost
    │   │   └── negative_log_likelihood_layer.h
    ├── data
    │   ├── facilities
    │   │   ├── tags.h
    │   │   ├── lower_access.h
    │   │   ├── continuous_memory.h
    │   │   └── allocators.h
    │   ├── matrices
    │   │   ├── matrices.h
    │   │   ├── zero_matrix.h
    │   │   ├── one_hot_vector.h
    │   │   └── cpu_matrix.h
    │   ├── batch
    │   │   ├── batch.h
    │   │   └── scalar.h
    │   └── scalar.h
    ├── evaluate
    │   ├── facilities
    │   │   ├── eval_unit.h
    │   │   ├── eval_pool.h
    │   │   ├── eval_buffer.h
    │   │   └── eval_group.h
    │   └── cpu
    │   │   └── trival_eval_pool.h
    ├── model_rel
    │   ├── param_initializer
    │   │   ├── facilities
    │   │   │   ├── fill_with_spec_dist.h
    │   │   │   ├── policies.h
    │   │   │   └── traits.h
    │   │   ├── gaussian_filler.h
    │   │   ├── constant_filler.h
    │   │   ├── uniform_filler.h
    │   │   ├── param_initializer.h
    │   │   └── var_scale_filler.h
    │   └── grad_col
    │   │   └── grad_collector.h
    ├── data_copy
    │   └── data_copy.h
    ├── meta_nn.h
    └── MetaNN.mk
├── .gitignore
├── Test
    ├── data
    │   ├── test_array.h
    │   ├── test_scalar.h
    │   ├── test_duplicate.h
    │   ├── test_batch_matrix.h
    │   ├── test_batch_scalar.h
    │   ├── test_zero_matrix.h
    │   ├── test_general_matrix.h
    │   ├── test_one_hot_vector.h
    │   ├── test_trival_matrix.h
    │   ├── test_scalar.cpp
    │   ├── test_zero_matrix.cpp
    │   ├── test_batch_scalar.cpp
    │   ├── test_trival_matrix.cpp
    │   ├── test_one_hot_vector.cpp
    │   ├── test_duplicate.cpp
    │   ├── test_general_matrix.cpp
    │   └── test_array.cpp
    ├── operators
    │   ├── test_abs.h
    │   ├── test_add.h
    │   ├── test_sign.h
    │   ├── test_tanh.h
    │   ├── test_divide.h
    │   ├── test_dot.h
    │   ├── test_sigmoid.h
    │   ├── test_softmax.h
    │   ├── test_collapse.h
    │   ├── test_substract.h
    │   ├── test_transpose.h
    │   ├── test_element_mul.h
    │   ├── test_interpolate.h
    │   ├── test_tanh_derivative.h
    │   ├── test_sigmoid_derivative.h
    │   ├── test_softmax_derivative.h
    │   ├── test_negative_log_likelihood.h
    │   ├── test_negative_log_likelihood_derivative.h
    │   ├── test_collapse.cpp
    │   ├── test_transpose.cpp
    │   ├── test_tanh.cpp
    │   ├── test_softmax.cpp
    │   ├── test_abs.cpp
    │   ├── test_interpolate.cpp
    │   ├── test_sigmoid.cpp
    │   ├── test_sign.cpp
    │   ├── test_tanh_derivative.cpp
    │   ├── test_dot.cpp
    │   ├── test_negative_log_likelihood.cpp
    │   ├── test_sigmoid_derivative.cpp
    │   └── test_negative_log_likelihood_derivative.cpp
    ├── evaluate
    │   ├── test_eval_plan.h
    │   └── test_eval_plan.cpp
    ├── layers
    │   ├── recurrent
    │   │   ├── test_gru.h
    │   │   └── test_gru_2.h
    │   ├── elementary
    │   │   ├── test_abs_layer.h
    │   │   ├── test_add_layer.h
    │   │   ├── test_bias_layer.h
    │   │   ├── test_sigmoid_layer.h
    │   │   ├── test_tanh_layer.h
    │   │   ├── test_weight_layer.h
    │   │   ├── test_element_mul_layer.h
    │   │   ├── test_softmax_layer.h
    │   │   ├── test_interpolate_layer.h
    │   │   └── test_softmax_layer.cpp
    │   ├── compose
    │   │   ├── test_compose_kernel.h
    │   │   ├── test_linear_layer.h
    │   │   └── test_single_layer.h
    │   └── cost
    │   │   └── test_negative_log_likelihood_layer.h
    ├── policies
    │   ├── test_change_policy.h
    │   ├── test_policy_selector.h
    │   ├── test_policy_operations.h
    │   ├── test_change_policy.cpp
    │   ├── test_policy_operations.cpp
    │   └── test_policy_selector.cpp
    ├── facilities
    │   ├── test_var_type_dict.h
    │   ├── calculate_tags.h
    │   ├── test_var_type_dict.cpp
    │   └── data_gen.h
    ├── model_rel
    │   └── param_initializer
    │   │   ├── test_constant_filler.h
    │   │   ├── test_gaussian_filler.h
    │   │   ├── test_var_scale_filter.h
    │   │   ├── test_constant_filler.cpp
    │   │   ├── test_gaussian_filler.cpp
    │   │   └── test_var_scale_filter.cpp
    └── main.cpp
├── code_index.pdf
├── readme.md
├── license
└── code_index.txt


/MetaNN/Release/.d:
--------------------------------------------------------------------------------
1 | 
2 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | /Test/bin
2 | /Test/obj
3 | 


--------------------------------------------------------------------------------
/Test/data/test_array.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_array();


--------------------------------------------------------------------------------
/Test/data/test_scalar.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_scalar();


--------------------------------------------------------------------------------
/Test/operators/test_abs.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_abs();


--------------------------------------------------------------------------------
/Test/operators/test_add.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_add();


--------------------------------------------------------------------------------
/Test/operators/test_sign.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_sign();


--------------------------------------------------------------------------------
/Test/operators/test_tanh.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_tanh();


--------------------------------------------------------------------------------
/Test/data/test_duplicate.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_duplicate();


--------------------------------------------------------------------------------
/Test/operators/test_divide.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_divide();


--------------------------------------------------------------------------------
/Test/operators/test_dot.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_dot();
4 | 


--------------------------------------------------------------------------------
/Test/operators/test_sigmoid.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_sigmoid();


--------------------------------------------------------------------------------
/Test/operators/test_softmax.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_softmax();


--------------------------------------------------------------------------------
/Test/data/test_batch_matrix.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_batch_matrix();


--------------------------------------------------------------------------------
/Test/data/test_batch_scalar.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_batch_scalar();


--------------------------------------------------------------------------------
/Test/data/test_zero_matrix.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_zero_matrix();


--------------------------------------------------------------------------------
/Test/evaluate/test_eval_plan.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_eval_plan();


--------------------------------------------------------------------------------
/Test/layers/recurrent/test_gru.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_gru();
4 | 


--------------------------------------------------------------------------------
/Test/operators/test_collapse.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_collapse();


--------------------------------------------------------------------------------
/Test/operators/test_substract.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_substract();


--------------------------------------------------------------------------------
/Test/operators/test_transpose.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_transpose();


--------------------------------------------------------------------------------
/Test/data/test_general_matrix.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_general_matrix();


--------------------------------------------------------------------------------
/Test/data/test_one_hot_vector.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_one_hot_vector();


--------------------------------------------------------------------------------
/Test/data/test_trival_matrix.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_trival_matrix();


--------------------------------------------------------------------------------
/Test/layers/recurrent/test_gru_2.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_gru_2();
4 | 


--------------------------------------------------------------------------------
/Test/operators/test_element_mul.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_element_mul();


--------------------------------------------------------------------------------
/Test/operators/test_interpolate.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_interpolate();


--------------------------------------------------------------------------------
/Test/layers/elementary/test_abs_layer.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_abs_layer();


--------------------------------------------------------------------------------
/Test/layers/elementary/test_add_layer.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_add_layer();


--------------------------------------------------------------------------------
/Test/layers/elementary/test_bias_layer.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_bias_layer();


--------------------------------------------------------------------------------
/Test/policies/test_change_policy.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_change_policy();


--------------------------------------------------------------------------------
/Test/policies/test_policy_selector.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_policy_selector();


--------------------------------------------------------------------------------
/code_index.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bluealert/MetaNN-book/HEAD/code_index.pdf


--------------------------------------------------------------------------------
/Test/facilities/test_var_type_dict.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_var_type_dict();
4 | 


--------------------------------------------------------------------------------
/Test/layers/compose/test_compose_kernel.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_compose_kernel();


--------------------------------------------------------------------------------
/Test/layers/elementary/test_sigmoid_layer.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_sigmoid_layer();


--------------------------------------------------------------------------------
/Test/layers/elementary/test_tanh_layer.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_tanh_layer();
4 | 


--------------------------------------------------------------------------------
/Test/operators/test_tanh_derivative.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_tanh_derivative();


--------------------------------------------------------------------------------
/Test/policies/test_policy_operations.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_policy_operations();


--------------------------------------------------------------------------------
/Test/layers/compose/test_linear_layer.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_linear_layer();
4 | 


--------------------------------------------------------------------------------
/Test/layers/compose/test_single_layer.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_single_layer();
4 | 


--------------------------------------------------------------------------------
/Test/layers/elementary/test_weight_layer.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_weight_layer();
4 | 


--------------------------------------------------------------------------------
/Test/operators/test_sigmoid_derivative.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_sigmoid_derivative();


--------------------------------------------------------------------------------
/Test/operators/test_softmax_derivative.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_softmax_derivative();


--------------------------------------------------------------------------------
/Test/layers/elementary/test_element_mul_layer.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_element_mul_layer();


--------------------------------------------------------------------------------
/Test/layers/elementary/test_softmax_layer.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_softmax_layer();
4 | 


--------------------------------------------------------------------------------
/Test/layers/elementary/test_interpolate_layer.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_interpolate_layer();
4 | 


--------------------------------------------------------------------------------
/Test/model_rel/param_initializer/test_constant_filler.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_constant_filler();


--------------------------------------------------------------------------------
/Test/model_rel/param_initializer/test_gaussian_filler.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_gaussian_filler();


--------------------------------------------------------------------------------
/Test/operators/test_negative_log_likelihood.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_negative_log_likelihood();


--------------------------------------------------------------------------------
/Test/model_rel/param_initializer/test_var_scale_filter.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_var_scale_filter();


--------------------------------------------------------------------------------
/MetaNN/facilities/null_param.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | namespace MetaNN
4 | {
5 | struct NullParameter
6 | { };
7 | }
8 | 


--------------------------------------------------------------------------------
/Test/layers/cost/test_negative_log_likelihood_layer.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_negative_log_likelihood_layer();
4 | 


--------------------------------------------------------------------------------
/Test/operators/test_negative_log_likelihood_derivative.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | void test_negative_log_likelihood_derivative();


--------------------------------------------------------------------------------
/MetaNN/policies/policy_macro_end.h:
--------------------------------------------------------------------------------
1 | #undef TypePolicyObj
2 | #undef ValuePolicyObj
3 | #undef TypePolicyTemplate
4 | #undef ValuePolicyTemplate


--------------------------------------------------------------------------------
/Test/facilities/calculate_tags.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | #include <MetaNN/meta_nn.h>
4 | 
5 | using CheckElement = float;
6 | using CheckDevice = MetaNN::DeviceTags::CPU;


--------------------------------------------------------------------------------
/MetaNN/operators/facilities/oper_seq.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | namespace MetaNN
 4 | {
 5 | template <typename...TCases>
 6 | struct OperSeqContainer;
 7 | 
 8 | template <typename TOpTag>
 9 | struct OperSeq_;
10 | }


--------------------------------------------------------------------------------
/MetaNN/layers/facilities/common_io.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <MetaNN/facilities/var_type_dict.h>
 4 | 
 5 | namespace MetaNN
 6 | {
 7 | struct LayerIO : public VarTypeDict<LayerIO> {};
 8 | 
 9 | struct CostLayerIn : public VarTypeDict<CostLayerIn, struct CostLayerLabel> {};
10 | 
11 | struct RnnLayerHiddenBefore;
12 | }
13 | 


--------------------------------------------------------------------------------
/MetaNN/data/facilities/tags.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | namespace MetaNN
 4 | {
 5 | /// data types
 6 | struct CategoryTags
 7 | {
 8 |     struct Scalar;
 9 |     struct Matrix;
10 |     struct BatchScalar;
11 |     struct BatchMatrix;
12 | };
13 | 
14 | /// device types
15 | struct DeviceTags
16 | {
17 |     struct CPU;
18 | };
19 | }
20 | 


--------------------------------------------------------------------------------
/MetaNN/data/matrices/matrices.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <MetaNN/data/facilities/traits.h>
 4 | namespace MetaNN
 5 | {
 6 | // matrices
 7 | template<typename TElement, typename TDevice>
 8 | class Matrix;
 9 | 
10 | template <typename TElement, typename TDevice>
11 | constexpr bool IsMatrix<Matrix<TElement, TDevice>> = true;
12 | }
13 | 


--------------------------------------------------------------------------------
/MetaNN/data/facilities/lower_access.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | namespace MetaNN
 4 | {
 5 | /// lower access
 6 | template<typename TData>
 7 | struct LowerAccessImpl;
 8 | 
 9 | template <typename TData>
10 | auto LowerAccess(TData&& p)
11 | {
12 |     using RawType = RemConstRef<TData>;
13 |     return LowerAccessImpl<RawType>(std::forward<TData>(p));
14 | }
15 | }
16 | 


--------------------------------------------------------------------------------
/MetaNN/evaluate/facilities/eval_unit.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <unordered_map>
 4 | #include <MetaNN/data/facilities/tags.h>
 5 | #include <vector>
 6 | 
 7 | namespace MetaNN
 8 | {
 9 | template <typename TDevice>
10 | class BaseEvalUnit
11 | {
12 | public:
13 |     using DeviceType = TDevice;
14 |     virtual ~BaseEvalUnit() = default;
15 | 
16 |     virtual void Eval() = 0;
17 | };
18 | }


--------------------------------------------------------------------------------
/MetaNN/operators/facilities/traits.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | namespace MetaNN
 4 | {
 5 | template <typename TOpTag, typename TOp1, typename...TOperands>
 6 | struct OperElementType_
 7 | {
 8 |     using type = typename TOp1::ElementType;
 9 | };
10 | 
11 | template <typename TOpTag, typename TOp1, typename...TOperands>
12 | struct OperDeviceType_
13 | {
14 |     using type = typename TOp1::DeviceType;
15 | };
16 | }


--------------------------------------------------------------------------------
/readme.md:
--------------------------------------------------------------------------------
1 | # 关于这个分支
2 | MetaNN 中包含了很多新的编程尝试，其开发过程也是一个不断总结与提升的过程。我花了一些时间，写了一本书（《C++模板元编程实战》），总结了开发过程中与元编程相关的一些技术。这个分支中的代码对应了该书中的内容。
3 | 
4 | MetaNN 的衍化并不快，但到目前为止一直在进行。衍化是在主分支上进行的，这个分支与主分支的代码会存在较多的差异，而随着开发的进行，这种差异也会越来越大。如果希望结合代码来阅读《C++模板元编程实战》这本书，那么应当关注于这个分支的代码。如果希望获取到 MetaNN 中最新的代码，同时对比与书中讨论内容的差异，那么应当获取主分支中的代码。
5 | 
6 | 除了内容上的差异外，主分支与本分支的编译环境也有所差异。本分支使用了 Makefile 来编译代码，在三个环境（Ubuntu + GCC, Ubuntu + clang, Windows + mingw-64）上进行了测试。但主分支则使用 Codelite 来组织代码，同时只在 Ubuntu + GCC 上进行了测试。


--------------------------------------------------------------------------------
/MetaNN/policies/inject_policy.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <MetaNN/policies/policy_container.h>
 4 | namespace MetaNN
 5 | {
 6 | //template <typename...TParams>
 7 | //template<template <typename TPolicyCont, typename...> class T, typename...TPolicies>
 8 | //using InjectPolicy = T<PolicyContainer<TPolicies...>, TParams...>;
 9 | 
10 | template<template <typename TPolicyCont> class T, typename...TPolicies>
11 | using InjectPolicy = T<PolicyContainer<TPolicies...>>;
12 | }


--------------------------------------------------------------------------------
/MetaNN/evaluate/facilities/eval_pool.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <MetaNN/evaluate/facilities/eval_unit.h>
 4 | #include <memory>
 5 | 
 6 | namespace MetaNN
 7 | {
 8 | enum class EvalPoolEnum
 9 | {
10 |     Trival
11 | };
12 | 
13 | template <typename TDevice>
14 | class BaseEvalPool
15 | {
16 | public:
17 |     virtual ~BaseEvalPool() = default;
18 |     virtual void Process(std::shared_ptr<BaseEvalUnit<TDevice>>&) = 0;
19 |     virtual void Barrier() = 0;
20 | };
21 | 
22 | template <typename TDevice>
23 | class TrivalEvalPool;
24 | }


--------------------------------------------------------------------------------
/MetaNN/data/batch/batch.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <MetaNN/data/facilities/continuous_memory.h>
 4 | #include <MetaNN/data/facilities/lower_access.h>
 5 | #include <MetaNN/data/facilities/tags.h>
 6 | 
 7 | namespace MetaNN
 8 | {
 9 | template<typename TElement, typename TDevice, typename TCategory>
10 | class Batch;
11 | 
12 | template <typename TElement, typename TDevice>
13 | constexpr bool IsBatchMatrix<Batch<TElement, TDevice, CategoryTags::Matrix>> = true;
14 | 
15 | template <typename TElement, typename TDevice>
16 | constexpr bool IsBatchScalar<Batch<TElement, TDevice, CategoryTags::Scalar>> = true;
17 | }


--------------------------------------------------------------------------------
/MetaNN/policies/policy_container.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | namespace MetaNN
 4 | {
 5 | template <typename...TPolicies>
 6 | struct PolicyContainer;
 7 | 
 8 | template <typename T>
 9 | constexpr bool IsPolicyContainer = false;
10 | 
11 | template <typename...T>
12 | constexpr bool IsPolicyContainer<PolicyContainer<T...>> = true;
13 | 
14 | template <typename TLayerName, typename...TPolicies>
15 | struct SubPolicyContainer;
16 | 
17 | template <typename T>
18 | constexpr bool IsSubPolicyContainer = false;
19 | 
20 | template <typename TLayer, typename...T>
21 | constexpr bool IsSubPolicyContainer<SubPolicyContainer<TLayer, T...>> = true;
22 | }


--------------------------------------------------------------------------------
/MetaNN/evaluate/cpu/trival_eval_pool.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <MetaNN/data/facilities/tags.h>
 4 | #include <MetaNN/evaluate/facilities/eval_pool.h>
 5 | namespace MetaNN
 6 | {
 7 | template <>
 8 | class TrivalEvalPool<DeviceTags::CPU> : public BaseEvalPool<DeviceTags::CPU>
 9 | {
10 | public:
11 |     static TrivalEvalPool& Instance()
12 |     {
13 |         static TrivalEvalPool inst;
14 |         return inst;
15 |     }
16 | private:
17 |     TrivalEvalPool() = default;
18 | public:
19 |     void Process(std::shared_ptr<BaseEvalUnit<DeviceTags::CPU>>& eu) override
20 |     {
21 |         eu->Eval();
22 |     }
23 | 
24 |     void Barrier() override {}
25 | };
26 | }


--------------------------------------------------------------------------------
/license:
--------------------------------------------------------------------------------
1 | Copyright (c) Wei Li.
2 | All rights reserved.
3 |  
4 | Redistribution and use in source and binary forms are permitted provided that the above copyright notice and this paragraph are duplicated in all such forms and that any documentation, advertising materials, and other materials related to such distribution and use acknowledge that the software was developed by Wei Li. The name of Wei Li may not be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
5 | 
6 | 


--------------------------------------------------------------------------------
/MetaNN/evaluate/facilities/eval_buffer.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <memory>
 4 | #include <MetaNN/evaluate/facilities/eval_handle.h>
 5 | 
 6 | namespace MetaNN
 7 | {
 8 | template <typename TData>
 9 | class EvalBuffer
10 | {
11 | public:
12 |     using DataType = TData;
13 |     
14 |     auto Handle() const
15 |     {
16 |         return m_handle;
17 |     }
18 |     
19 |     auto ConstHandle() const
20 |     {
21 |         return ConstEvalHandle<EvalHandle<TData>>(m_handle);
22 |     }
23 |     
24 |     bool IsEvaluated() const noexcept
25 |     {
26 |         return m_handle.IsEvaluated();
27 |     }
28 |     
29 | private:
30 |     EvalHandle<TData> m_handle;
31 | };
32 | }


--------------------------------------------------------------------------------
/MetaNN/operators/facilities/tags.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | namespace MetaNN
 4 | {
 5 | struct UnaryOpTags
 6 | {
 7 |     struct Abs;
 8 |     struct Sigmoid;
 9 |     struct Sign;
10 |     struct Tanh;
11 |     struct Transpose;
12 |     struct Collapse;
13 |     struct VecSoftmax;
14 | };
15 | 
16 | struct BinaryOpTags
17 | {
18 |     struct Add;
19 |     struct Substract;
20 |     struct ElementMul;
21 |     struct Divide;
22 |     struct Dot;
23 |     struct NegativeLogLikelihood;
24 |     struct SigmoidDerivative;
25 |     struct TanhDerivative;
26 |     struct VecSoftmaxDerivative;
27 | };
28 | 
29 | struct TernaryOpTags
30 | {
31 |     struct Interpolate;
32 |     struct NegativeLogLikelihoodDerivative;
33 | };
34 | }


--------------------------------------------------------------------------------
/MetaNN/model_rel/param_initializer/facilities/fill_with_spec_dist.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | namespace MetaNN
 4 | {
 5 | namespace NSInitializer
 6 | {
 7 | template <typename TElem, typename TDist, typename TEngine>
 8 | void FillWithDist(Matrix<TElem, DeviceTags::CPU>& data, TDist& dist, TEngine& engine)
 9 | {
10 |     if (!data.AvailableForWrite())
11 |     {
12 |         throw std::runtime_error("Matrix is sharing weight, cannot fill-in.");
13 |     }
14 |     
15 |     auto mem = LowerAccess(data);
16 |     const size_t rowNum = data.RowNum();
17 |     const size_t colNum = data.ColNum();
18 |     const size_t tgtPackNum = mem.RowLen();
19 |     auto r = mem.MutableRawMemory();
20 |     
21 |     for (size_t i = 0; i < rowNum; ++i)
22 |     {
23 |         for (size_t j = 0; j < colNum; ++j)
24 |         {
25 |             r[j] = (TElem)(dist(engine));
26 |         }
27 |         r += tgtPackNum;
28 |     }
29 | }
30 | }
31 | }


--------------------------------------------------------------------------------
/Test/data/test_scalar.cpp:
--------------------------------------------------------------------------------
 1 | #include "test_scalar.h"
 2 | #include <iostream>
 3 | #include <cassert>
 4 | #include <set>
 5 | #include <MetaNN/meta_nn.h>
 6 | using namespace std;
 7 | using namespace MetaNN;
 8 | 
 9 | namespace
10 | {
11 | void test_scalar1()
12 | {
13 |     cout << "Test scalar case 1...\t";
14 |     static_assert(IsScalar<Scalar<int, DeviceTags::CPU>>, "Test Error");
15 |     static_assert(IsScalar<Scalar<int, DeviceTags::CPU> &>, "Test Error");
16 |     static_assert(IsScalar<Scalar<int, DeviceTags::CPU> &&>, "Test Error");
17 |     static_assert(IsScalar<const Scalar<int, DeviceTags::CPU> &>, "Test Error");
18 |     static_assert(IsScalar<const Scalar<int, DeviceTags::CPU> &&>, "Test Error");
19 |     
20 |     Scalar<float, DeviceTags::CPU> pi(3.1415926f);
21 |     assert(pi == pi);
22 |     
23 |     auto x = pi.EvalRegister();
24 |     assert(x.Data() == pi);
25 |     cout << "done" << endl;
26 | }    
27 | }
28 | 
29 | void test_scalar()
30 | {
31 |     test_scalar1();
32 | }


--------------------------------------------------------------------------------
/Test/policies/test_change_policy.cpp:
--------------------------------------------------------------------------------
 1 | #include <iostream>
 2 | #include <cassert>
 3 | #include <set>
 4 | #include <MetaNN/meta_nn.h>
 5 | using namespace std;
 6 | using namespace MetaNN;
 7 | 
 8 | namespace
 9 | {
10 | struct Tag1;
11 | struct Tag2;
12 | struct Tag3;
13 | 
14 | void test_change_policy1()
15 | {
16 |     cout << "Test change policy case 1...\t";
17 |     using input = PolicyContainer<PBatchMode,
18 |                                   SubPolicyContainer<Tag1, PNoBatchMode>>;
19 | 
20 |     using check1 = ChangePolicy<PEnableBptt, input>;
21 |     static_assert(is_same<check1, PolicyContainer<PBatchMode, SubPolicyContainer<Tag1, PNoBatchMode>, PEnableBptt>>::value, "Check Error");
22 | 
23 |     using check2 = ChangePolicy<PNoBatchMode, input>;
24 |     static_assert(is_same<check2, PolicyContainer<SubPolicyContainer<Tag1, PNoBatchMode>, PNoBatchMode>>::value, "Check Error");
25 |     cout << "done" << endl;
26 | }
27 | }
28 | void test_change_policy()
29 | {
30 |     test_change_policy1();
31 | }
32 | 


--------------------------------------------------------------------------------
/Test/facilities/test_var_type_dict.cpp:
--------------------------------------------------------------------------------
 1 | #include "test_var_type_dict.h"
 2 | #include <iostream>
 3 | #include <cassert>
 4 | #include <cmath>
 5 | #include <set>
 6 | #include <MetaNN/meta_nn.h>
 7 | using namespace std;
 8 | using namespace MetaNN;
 9 | 
10 | namespace
11 | {
12 | struct A; struct B; struct Weight; struct XX;
13 | 
14 | struct FParams : public VarTypeDict<struct A,
15 |                                     struct B,
16 |                                     struct Weight> {};
17 | 
18 | template <typename TIn>
19 | float fun(const TIn& in) {
20 |     auto a = in.template Get<A>();
21 |     auto b = in.template Get<B>();
22 |     auto weight = in.template Get<Weight>();
23 | 
24 |     return a * weight + b * (1 - weight);
25 | }
26 | }
27 | 
28 | void test_var_type_dict()
29 | {
30 |     cout << "Test named params...\t";
31 |     auto res = fun(FParams::Create()
32 |                              .Set<A>(1.3f)
33 |                              .Set<B>(2.4f)
34 |                              .Set<Weight>(0.1f));
35 |     assert(fabs(res - 0.1 * 1.3 - 0.9 * 2.4) < 0.0001);
36 |     cout << "done\n";
37 | }
38 | 


--------------------------------------------------------------------------------
/Test/model_rel/param_initializer/test_constant_filler.cpp:
--------------------------------------------------------------------------------
 1 | #include "test_constant_filler.h"
 2 | #include <MetaNN/meta_nn.h>
 3 | #include <cassert>
 4 | #include <iostream>
 5 | using namespace MetaNN;
 6 | using namespace std;
 7 | 
 8 | namespace
 9 | {
10 | void test_constant_filler1()
11 | {
12 |     cout << "test constant filler case 1 ...";
13 |     
14 |     ConstantFiller filler(0);
15 |     Matrix<float, DeviceTags::CPU> mat (11, 13);
16 |     filler.Fill(mat, 11, 13);
17 |     for (size_t i = 0; i < 11; ++i)
18 |     {
19 |         for (size_t j = 0; j < 13; ++j)
20 |         {
21 |             assert(fabs(mat(i, j)) < 0.0001);
22 |         }
23 |     }
24 |     
25 |     ConstantFiller filler2(1.5f);
26 |     Matrix<float, DeviceTags::CPU> mat2 (21, 33);
27 |     filler2.Fill(mat2, 21, 33);
28 |     for (size_t i = 0; i < 21; ++i)
29 |     {
30 |         for (size_t j = 0; j < 33; ++j)
31 |         {
32 |             assert(fabs(mat2(i, j) - 1.5) < 0.0001);
33 |         }
34 |     }
35 |     
36 |     cout << "done" << endl;
37 | }
38 | }
39 | 
40 | void test_constant_filler()
41 | {
42 |     test_constant_filler1();
43 | }
44 | 


--------------------------------------------------------------------------------
/Test/facilities/data_gen.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | #include <MetaNN/meta_nn.h>
 3 | 
 4 | template <typename TElem>
 5 | inline auto GenMatrix(std::size_t r, std::size_t c, TElem start = 0, TElem scale = 1)
 6 | {
 7 |     using namespace MetaNN;
 8 |     Matrix<TElem, MetaNN::DeviceTags::CPU> res(r, c);
 9 |     for (std::size_t i = 0; i < r; ++i)
10 |     {
11 |         for (std::size_t j = 0; j < c; ++j)
12 |         {
13 |             res.SetValue(i, j, (TElem)(start * scale));
14 |             start += 1.0f;
15 |         }
16 |     }
17 |     return res;
18 | }
19 | 
20 | template <typename TElem>
21 | inline auto GenBatchMatrix(size_t r, size_t c, size_t d, float start = 0, float scale = 1)
22 | {
23 |     using namespace MetaNN;
24 |     Batch<TElem, MetaNN::DeviceTags::CPU, CategoryTags::Matrix> res(d, r, c);
25 |     for (size_t i = 0; i < r; ++i)
26 |     {
27 |         for (size_t j = 0; j < c; ++j)
28 |         {
29 |             for (size_t k = 0; k < d; ++k)
30 |             {
31 |                 res.SetValue(k, i, j, (TElem)(start * scale));
32 |                 start += 1.0f;
33 |             }
34 |         }
35 |     }
36 |     return res;
37 | }
38 | 
39 | 


--------------------------------------------------------------------------------
/MetaNN/policies/policy_macro_begin.h:
--------------------------------------------------------------------------------
 1 | #define TypePolicyObj(PolicyName, Ma, Mi, Val) \
 2 | struct PolicyName : virtual public Ma\
 3 | { \
 4 |     using MinorClass = Ma::Mi##TypeCate; \
 5 |     using Mi = Ma::Mi##TypeCate::Val; \
 6 | }
 7 | 
 8 | #define ValuePolicyObj(PolicyName, Ma, Mi, Val) \
 9 | struct PolicyName : virtual public Ma \
10 | { \
11 |     using MinorClass = Ma::Mi##ValueCate; \
12 | private: \
13 |     using type1 = decltype(Ma::Mi); \
14 |     using type2 = RemConstRef<type1>; \
15 | public: \
16 |     static constexpr type2 Mi = static_cast<type2>(Val); \
17 | }
18 | 
19 | #define TypePolicyTemplate(PolicyName, Ma, Mi) \
20 | template <typename T> \
21 | struct PolicyName : virtual public Ma \
22 | { \
23 |     using MinorClass = Ma::Mi##TypeCate; \
24 |     using Mi = T; \
25 | }
26 | 
27 | #define ValuePolicyTemplate(PolicyName, Ma, Mi) \
28 | template <RemConstRef<decltype(Ma::Mi)> T> \
29 | struct PolicyName : virtual public Ma \
30 | { \
31 |     using MinorClass = Ma::Mi##ValueCate; \
32 | private: \
33 |     using type1 = decltype(Ma::Mi); \
34 |     using type2 = RemConstRef<type1>; \
35 | public: \
36 |     static constexpr type2 Mi = T; \
37 | }
38 | 


--------------------------------------------------------------------------------
/MetaNN/model_rel/param_initializer/gaussian_filler.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <random>
 4 | #include <stdexcept>
 5 | #include <type_traits>
 6 | #include <MetaNN/model_rel/param_initializer/facilities/fill_with_spec_dist.h>
 7 | 
 8 | namespace MetaNN
 9 | {
10 | template <typename TPolicyCont = PolicyContainer<>>
11 | class GaussianFiller
12 | {
13 |     using TRandomEngine = typename PolicySelect<InitPolicy, TPolicyCont>::RandEngine;
14 | public:
15 |     GaussianFiller(double p_mean, double p_std,
16 |                    unsigned seed = std::random_device{}())
17 |         : m_engine(seed)
18 |         , m_mean(p_mean)
19 |         , m_std(p_std)
20 |     {
21 |         if (p_std <= 0)
22 |         {
23 |             throw std::runtime_error("Invalid std.");
24 |         }
25 |     }
26 |     
27 |     template <typename TData>
28 |     void Fill(TData& data, size_t /*fanin*/, size_t /*fanout*/)
29 |     {
30 |         using ElementType = typename TData::ElementType;
31 |         std::normal_distribution<ElementType> dist(m_mean, m_std);
32 |         NSInitializer::FillWithDist(data, dist, m_engine);
33 |     }
34 |     
35 | private:
36 |     TRandomEngine m_engine;
37 |     double m_mean;
38 |     double m_std;
39 | };
40 | }


--------------------------------------------------------------------------------
/MetaNN/data_copy/data_copy.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <MetaNN/data/matrices/cpu_matrix.h>
 4 | #include <stdexcept>
 5 | 
 6 | namespace MetaNN
 7 | {
 8 | template <typename TElem>
 9 | void DataCopy(const Matrix<TElem, DeviceTags::CPU>& src,
10 |               Matrix<TElem, DeviceTags::CPU>& dst)
11 | {
12 |     const size_t rowNum = src.RowNum();
13 |     const size_t colNum = src.ColNum();
14 |     if ((rowNum != dst.RowNum()) || (colNum != dst.ColNum()))
15 |     {
16 |         throw std::runtime_error("Error in data-copy: Matrix dimension mismatch.");
17 |     }
18 |     
19 |     const auto mem_src = LowerAccess(src);
20 |     auto mem_dst = LowerAccess(dst);
21 | 
22 |     const size_t srcPackNum = mem_src.RowLen();
23 |     const size_t dstPackNum = mem_dst.RowLen();
24 | 
25 |     const TElem* r1 = mem_src.RawMemory();
26 |     TElem* r = mem_dst.MutableRawMemory();
27 |         
28 |     if ((srcPackNum == colNum) && (dstPackNum == colNum))
29 |     {
30 |         memcpy(r, r1, sizeof(TElem) * rowNum * colNum);
31 |     }
32 |     else
33 |     {
34 |         for (size_t i = 0; i < rowNum; ++i)
35 |         {
36 |             memcpy(r, r1, sizeof(TElem) * colNum);
37 |             r += dstPackNum;
38 |             r1 += srcPackNum;
39 |         }
40 |     }
41 | }
42 | }


--------------------------------------------------------------------------------
/MetaNN/model_rel/param_initializer/constant_filler.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <random>
 4 | #include <stdexcept>
 5 | #include <type_traits>
 6 | 
 7 | namespace MetaNN
 8 | {
 9 | namespace NSConstantFiller
10 | {
11 | template <typename TElem>
12 | void Fill(Matrix<TElem, DeviceTags::CPU>& mat, const double& val)
13 | {
14 |     if (!mat.AvailableForWrite())
15 |     {
16 |         throw std::runtime_error("Matrix is sharing weight, cannot fill-in.");
17 |     }
18 |     
19 |     auto mem = LowerAccess(mat);
20 |     const size_t rowNum = mat.RowNum();
21 |     const size_t colNum = mat.ColNum();
22 |     const size_t tgtPackNum = mem.RowLen();
23 |     auto r = mem.MutableRawMemory();
24 |     
25 |     for (size_t i = 0; i < rowNum; ++i)
26 |     {
27 |         for (size_t j = 0; j < colNum; ++j)
28 |         {
29 |             r[j] = static_cast<TElem>(val);
30 |         }
31 |         r += tgtPackNum;
32 |     }
33 | }
34 | }
35 | 
36 | class ConstantFiller
37 | {
38 | public:
39 |     ConstantFiller(double val = 0)
40 |         : m_value(val)
41 |     {}
42 |     
43 |     template <typename TData>
44 |     void Fill(TData& data, size_t /*fanin*/, size_t /*fanout*/)
45 |     {
46 |         NSConstantFiller::Fill(data, m_value);
47 |     }
48 |     
49 | private:
50 |     double m_value;
51 | };
52 | }


--------------------------------------------------------------------------------
/Test/data/test_zero_matrix.cpp:
--------------------------------------------------------------------------------
 1 | #include "test_zero_matrix.h"
 2 | #include "../facilities/calculate_tags.h"
 3 | #include <iostream>
 4 | #include <cassert>
 5 | #include <MetaNN/meta_nn.h>
 6 | using namespace std;
 7 | using namespace MetaNN;
 8 | 
 9 | namespace
10 | {
11 | void TestZeroMatrix1()
12 | {
13 |     cout << "Test zero matrix case 1...\t";
14 |     static_assert(IsMatrix<ZeroMatrix<int, CheckDevice>>, "Test Error");
15 |     static_assert(IsMatrix<ZeroMatrix<int, CheckDevice> &>, "Test Error");
16 |     static_assert(IsMatrix<ZeroMatrix<int, CheckDevice> &&>, "Test Error");
17 |     static_assert(IsMatrix<const ZeroMatrix<int, CheckDevice> &>, "Test Error");
18 |     static_assert(IsMatrix<const ZeroMatrix<int, CheckDevice> &&>, "Test Error");
19 | 
20 |     auto rm = ZeroMatrix<int, CheckDevice>(10, 20);
21 |     assert(rm.RowNum() == 10);
22 |     assert(rm.ColNum() == 20);
23 | 
24 |     const auto& evalHandle = rm.EvalRegister();
25 |     EvalPlan<DeviceTags::CPU>::Eval();
26 | 
27 |     auto rm1 = evalHandle.Data();
28 |     for (size_t i=0; i<10; ++i)
29 |     {
30 |         for (size_t j=0; j<20; ++j)
31 |         {
32 |             assert(rm1(i, j) == 0);
33 |         }
34 |     }
35 | 
36 |     cout << "done" << endl;
37 | }
38 | }
39 | 
40 | void test_zero_matrix()
41 | {
42 |     TestZeroMatrix1();
43 | }
44 | 


--------------------------------------------------------------------------------
/Test/model_rel/param_initializer/test_gaussian_filler.cpp:
--------------------------------------------------------------------------------
 1 | #include "test_gaussian_filler.h"
 2 | #include <MetaNN/meta_nn.h>
 3 | #include <cassert>
 4 | #include <iostream>
 5 | using namespace MetaNN;
 6 | using namespace std;
 7 | 
 8 | namespace
 9 | {
10 | void test_gaussian_filler1()
11 | {
12 |     cout << "test gaussian filler case 1 ...";
13 |     
14 |     GaussianFiller filler(1.5, 3.3);
15 |     Matrix<float, DeviceTags::CPU> mat (1000, 3000);
16 |     filler.Fill(mat, 1000, 3000);
17 |     
18 |     float mean = 0;
19 |     for (size_t i = 0; i < mat.RowNum(); ++i)
20 |     {
21 |         for (size_t j = 0; j < mat.ColNum(); ++j)
22 |         {
23 |             mean += mat(i, j);
24 |         }
25 |     }
26 |     mean /= mat.RowNum() * mat.ColNum();
27 |     
28 |     float var = 0;
29 |     for (size_t i = 0; i < mat.RowNum(); ++i)
30 |     {
31 |         for (size_t j = 0; j < mat.ColNum(); ++j)
32 |         {
33 |             var += (mat(i, j) - mean) * (mat(i, j) - mean);
34 |         }
35 |     }
36 |     var /= mat.RowNum() * mat.ColNum();
37 |     
38 |     // mean = 1.5, std = 3.3
39 |     cout << "mean-delta = " << fabs(mean-1.5) << " std-delta = " << fabs(sqrt(var)-3.3) << ' ';
40 |     cout << "done" << endl;
41 | }
42 | }
43 | 
44 | void test_gaussian_filler()
45 | {
46 |     test_gaussian_filler1();
47 | }
48 | 


--------------------------------------------------------------------------------
/Test/policies/test_policy_operations.cpp:
--------------------------------------------------------------------------------
 1 | #include <iostream>
 2 | #include <cassert>
 3 | #include <set>
 4 | #include <MetaNN/meta_nn.h>
 5 | using namespace std;
 6 | using namespace MetaNN;
 7 | 
 8 | namespace
 9 | {
10 | struct Tag1;
11 | struct Tag2;
12 | struct Tag3;
13 | 
14 | void test_policy_operations1()
15 | {
16 |     cout << "Test policy operations case 1...\t";
17 |     using input = PolicyContainer<PBatchMode,
18 |                                   SubPolicyContainer<Tag1, PNoBatchMode,
19 |                                                      SubPolicyContainer<Tag2>>>;
20 |     using check1 = SubPolicyPicker<input, Tag3>;
21 |     static_assert(is_same<check1, PolicyContainer<PBatchMode>>::value, "Check Error");
22 | 
23 |     using check2 = SubPolicyPicker<input, Tag1>;
24 |     static_assert(is_same<check2, PolicyContainer<PNoBatchMode, SubPolicyContainer<Tag2>>>::value, "Check Error");
25 | 
26 |     using check3 = SubPolicyPicker<check2, Tag3>;
27 |     static_assert(is_same<check3, PolicyContainer<PNoBatchMode>>::value, "Check Error");
28 | 
29 |     using check4 = SubPolicyPicker<check2, Tag2>;
30 |     static_assert(is_same<check4, PolicyContainer<PNoBatchMode>>::value, "Check Error");
31 |     cout << "done" << endl;
32 | }
33 | }
34 | void test_policy_operations()
35 | {
36 |     test_policy_operations1();
37 | }
38 | 


--------------------------------------------------------------------------------
/MetaNN/data/facilities/continuous_memory.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <MetaNN/data/facilities/allocators.h>
 4 | namespace MetaNN
 5 | {
 6 | template <typename TElem, typename TDevice>
 7 | class ContinuousMemory
 8 | {
 9 |     static_assert(std::is_same<RemConstRef<TElem>, TElem>::value);
10 |     using ElementType = TElem;
11 | public:
12 |     explicit ContinuousMemory(size_t p_size)
13 |         : m_mem(Allocator<TDevice>::template Allocate<ElementType>(p_size))
14 |         , m_memStart(m_mem.get())
15 |     {}
16 | 
17 |     ContinuousMemory(std::shared_ptr<ElementType> p_mem, ElementType* p_memStart)
18 |         : m_mem(std::move(p_mem))
19 |         , m_memStart(p_memStart)
20 |     {}
21 | 
22 |     auto RawMemory() const { return m_memStart; }
23 | 
24 |     const std::shared_ptr<ElementType> SharedPtr() const
25 |     {
26 |         return m_mem;
27 |     }
28 | 
29 |     size_t UseCount() const
30 |     {
31 |         return m_mem.use_count();
32 |     }
33 | 
34 |     bool operator== (const ContinuousMemory& val) const
35 |     {
36 |         return (m_mem == val.m_mem) && (m_memStart == val.m_memStart);
37 |     }
38 | 
39 |     bool operator!= (const ContinuousMemory& val) const
40 |     {
41 |         return !(operator==(val));
42 |     }
43 | 
44 | private:
45 |     std::shared_ptr<ElementType> m_mem;
46 |     ElementType*                 m_memStart;
47 | };
48 | }


--------------------------------------------------------------------------------
/MetaNN/data/scalar.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <MetaNN/data/facilities/traits.h>
 4 | #include <MetaNN/data/facilities/tags.h>
 5 | #include <MetaNN/evaluate/facilities/eval_handle.h>
 6 | 
 7 | namespace MetaNN
 8 | {
 9 | template <typename TElem, typename TDevice = DeviceTags::CPU>
10 | class Scalar
11 | {
12 |     static_assert(std::is_same<RemConstRef<TElem>, TElem>::value);
13 | public:
14 |     using ElementType = TElem;
15 |     using DeviceType = TDevice;
16 |     
17 | public:
18 |     Scalar(ElementType elem = ElementType())
19 |         : m_elem(elem) {}
20 |      
21 |     auto& Value()
22 |     {
23 |         return m_elem;
24 |     }
25 |    
26 |     auto Value() const
27 |     {
28 |         return m_elem;
29 |     }
30 |     
31 |     bool operator== (const Scalar& val) const
32 |     {
33 |         return m_elem == val.m_elem;
34 |     }
35 | 
36 |     template <typename TOtherType>
37 |     bool operator== (const TOtherType&) const
38 |     {
39 |         return false;
40 |     }
41 | 
42 |     template <typename TData>
43 |     bool operator!= (const TData& val) const
44 |     {
45 |         return !(operator==(val));
46 |     }
47 |     
48 |     auto EvalRegister() const
49 |     {
50 |         return MakeConstEvalHandle(*this);
51 |     }
52 |     
53 | private:
54 |     ElementType m_elem;
55 | };
56 | 
57 | template <typename TElem, typename TDevice>
58 | constexpr bool IsScalar<Scalar<TElem, TDevice>> = true;
59 | }


--------------------------------------------------------------------------------
/MetaNN/evaluate/facilities/eval_group.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <MetaNN/evaluate/facilities/eval_unit.h>
 4 | #include <list>
 5 | #include <memory>
 6 | 
 7 | namespace MetaNN
 8 | {
 9 | template <typename TDevice>
10 | class BaseEvalGroup
11 | {
12 | public:
13 |     virtual ~BaseEvalGroup() = default;
14 |     virtual std::shared_ptr<BaseEvalUnit<TDevice>> GetEvalUnit() = 0;
15 |     virtual void Merge(BaseEvalUnit<TDevice>&) = 0;
16 |     virtual void Merge(BaseEvalUnit<TDevice>&&) = 0;
17 | };
18 | 
19 | template <typename TEvalUnit>
20 | class TrivalEvalGroup : public BaseEvalGroup<typename TEvalUnit::DeviceType>
21 | {
22 |     using DeviceType = typename TEvalUnit::DeviceType;
23 | public:
24 |     std::shared_ptr<BaseEvalUnit<DeviceType>> GetEvalUnit() override
25 |     {
26 |         std::shared_ptr<BaseEvalUnit<DeviceType>> res;
27 |         if (!m_unitList.empty())
28 |         {
29 |             res = std::make_shared<TEvalUnit>(std::move(m_unitList.front()));
30 |             m_unitList.pop_front();
31 |         }
32 |         return res;
33 |     }
34 | 
35 |     void Merge(BaseEvalUnit<DeviceType>& unit) override
36 |     {
37 |         m_unitList.push_back(static_cast<TEvalUnit&>(unit));
38 |     }
39 |     
40 |     void Merge(BaseEvalUnit<DeviceType>&& unit) override
41 |     {
42 |         m_unitList.push_back(static_cast<TEvalUnit&&>(unit));
43 |     }
44 | 
45 | private:
46 |     std::list<TEvalUnit> m_unitList;
47 | };
48 | }


--------------------------------------------------------------------------------
/MetaNN/model_rel/param_initializer/uniform_filler.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <random>
 4 | #include <stdexcept>
 5 | #include <type_traits>
 6 | #include <MetaNN/model_rel/param_initializer/facilities/fill_with_spec_dist.h>
 7 | 
 8 | namespace MetaNN
 9 | {
10 | template <typename TPolicyCont = PolicyContainer<>>
11 | class UniformFiller
12 | {
13 |     using TRandomEngine = typename PolicySelect<InitPolicy, TPolicyCont>::RandEngine;
14 | public:
15 |     UniformFiller(double min, double max,
16 |                   unsigned seed = std::random_device{}())
17 |         : m_engine(seed)
18 |         , m_min(min)
19 |         , m_max(max)
20 |     {
21 |         if (min >= max)
22 |         {
23 |             throw std::runtime_error("min is larger or equal to max");
24 |         }
25 |     }
26 |     
27 |     template <typename TData>
28 |     void Fill(TData& data, size_t /*fanin*/, size_t /*fanout*/)
29 |     {
30 |         using ElementType = typename TData::ElementType;
31 |         using DistType = std::conditional_t<std::is_integral<ElementType>::value,
32 |                                             std::uniform_int_distribution<ElementType>,
33 |                                             std::uniform_real_distribution<ElementType>>;
34 |         DistType dist(m_min, m_max);
35 |         NSInitializer::FillWithDist(data, dist, m_engine);
36 |     }
37 |     
38 | private:
39 |     TRandomEngine m_engine;
40 |     double m_min;
41 |     double m_max;
42 | };
43 | }


--------------------------------------------------------------------------------
/MetaNN/layers/compose/linear_layer.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <MetaNN/layers/compose/compose_kernel.h>
 4 | #include <MetaNN/layers/elementary/bias_layer.h>
 5 | #include <MetaNN/layers/elementary/weight_layer.h>
 6 | 
 7 | namespace MetaNN
 8 | {
 9 | template <typename TPolicies> class LinearLayer;
10 | 
11 | template <>
12 | struct Sublayerof<LinearLayer>
13 | {
14 |     struct Weight;
15 |     struct Bias;
16 | };
17 | 
18 | namespace NSLinearLayer
19 | {
20 |     using WeightSublayer = Sublayerof<LinearLayer>::Weight;
21 |     using BiasSublayer = Sublayerof<LinearLayer>::Bias;
22 | 
23 |     using Topology = ComposeTopology<SubLayer<WeightSublayer, WeightLayer>,
24 |                                      SubLayer<BiasSublayer, BiasLayer>,
25 |                                      InConnect<LayerIO, WeightSublayer, LayerIO>,
26 |                                      InternalConnect<WeightSublayer, LayerIO, BiasSublayer, LayerIO>,
27 |                                      OutConnect<BiasSublayer, LayerIO, LayerIO>>;
28 | 
29 |     template <typename TPolicies>
30 |     using Base = ComposeKernel<LayerIO, LayerIO, TPolicies, Topology>;
31 | }
32 | 
33 | 
34 | template <typename TPolicies>
35 | class LinearLayer : public NSLinearLayer::Base<TPolicies>
36 | {
37 |     using TBase = NSLinearLayer::Base<TPolicies>;
38 | 
39 | public:
40 |     LinearLayer(const std::string& p_name, size_t p_inputLen, size_t p_outputLen)
41 |         : TBase(TBase::CreateSubLayers()
42 |                         .template Set<NSLinearLayer::WeightSublayer>(p_name + "-weight", p_inputLen, p_outputLen)
43 |                         .template Set<NSLinearLayer::BiasSublayer>(p_name + "-bias", p_outputLen))
44 |     { }
45 | };
46 | }
47 | 


--------------------------------------------------------------------------------
/MetaNN/model_rel/param_initializer/facilities/policies.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <random>
 4 | #include <MetaNN/policies/policy_macro_begin.h>
 5 | namespace MetaNN
 6 | {
 7 | struct InitPolicy
 8 | {
 9 |     using MajorClass = InitPolicy;
10 |     
11 |     struct OverallTypeCate;
12 |     struct WeightTypeCate;
13 |     struct BiasTypeCate;
14 |     
15 |     using Overall = void;
16 |     using Weight = void;
17 |     using Bias = void;
18 |     
19 |     
20 |     struct RandEngineTypeCate;
21 |     using RandEngine = std::mt19937;
22 | };
23 | 
24 | TypePolicyTemplate(PInitializerIs,       InitPolicy, Overall);
25 | TypePolicyTemplate(PWeightInitializerIs, InitPolicy, Weight);
26 | TypePolicyTemplate(PBiasInitializerIs,   InitPolicy, Bias);
27 | TypePolicyTemplate(PRandomGeneratorIs,   InitPolicy, RandEngine);
28 | 
29 | struct VarScaleFillerPolicy
30 | {
31 |     using MajorClass = VarScaleFillerPolicy;
32 |     
33 |     struct DistributeTypeCate
34 |     {
35 |         struct Uniform;
36 |         struct Norm;
37 |     };
38 |     using Distribute = DistributeTypeCate::Uniform;
39 |     
40 |     struct ScaleModeTypeCate
41 |     {
42 |         struct FanIn;
43 |         struct FanOut;
44 |         struct FanAvg;
45 |     };
46 |     using ScaleMode = ScaleModeTypeCate::FanAvg;
47 | };
48 | TypePolicyObj(PNormVarScale,    VarScaleFillerPolicy, Distribute, Norm);
49 | TypePolicyObj(PUniformVarScale, VarScaleFillerPolicy, Distribute, Uniform);
50 | TypePolicyObj(PVarScaleFanIn,   VarScaleFillerPolicy, ScaleMode,  FanIn);
51 | TypePolicyObj(PVarScaleFanOut,  VarScaleFillerPolicy, ScaleMode,  FanOut);
52 | TypePolicyObj(PVarScaleFanAvg,  VarScaleFillerPolicy, ScaleMode,  FanAvg);
53 | }
54 | 
55 | #include <MetaNN/policies/policy_macro_end.h>


--------------------------------------------------------------------------------
/Test/data/test_batch_scalar.cpp:
--------------------------------------------------------------------------------
 1 | #include "test_general_matrix.h"
 2 | #include "../facilities/calculate_tags.h"
 3 | #include <iostream>
 4 | #include <cassert>
 5 | #include <set>
 6 | #include <MetaNN/meta_nn.h>
 7 | using namespace std;
 8 | using namespace MetaNN;
 9 | 
10 | namespace
11 | {
12 | void test_case1()
13 | {
14 |     cout << "Test batch scalar case 1...\t";
15 |     static_assert(IsBatchScalar<Batch<CheckElement, CheckDevice, CategoryTags::Scalar>>, "Test Error");
16 |     static_assert(IsBatchScalar<Batch<CheckElement, CheckDevice, CategoryTags::Scalar>&>, "Test Error");
17 |     static_assert(IsBatchScalar<Batch<CheckElement, CheckDevice, CategoryTags::Scalar>&&>, "Test Error");
18 |     static_assert(IsBatchScalar<const Batch<CheckElement, CheckDevice, CategoryTags::Scalar>&>, "Test Error");
19 |     static_assert(IsBatchScalar<const Batch<CheckElement, CheckDevice, CategoryTags::Scalar>&&>, "Test Error");
20 | 
21 |     Batch<CheckElement, CheckDevice, CategoryTags::Scalar> check;
22 |     assert(check.BatchNum() == 0);
23 | 
24 |     check = Batch<CheckElement, CheckDevice, CategoryTags::Scalar>(13);
25 |     assert(check.BatchNum() == 13);
26 | 
27 |     int c = 0;
28 |     for (size_t i=0; i<13; ++i)
29 |     {
30 |         check.SetValue(i, (float)(c++));
31 |     }
32 | 
33 |     const Batch<CheckElement, CheckDevice, CategoryTags::Scalar> c2 = check;
34 |     c = 0;
35 |     for (size_t i=0; i<13; ++i)
36 |     {
37 |         assert(c2[i] == (float)(c++));
38 |     }
39 | 
40 |     auto evalHandle = check.EvalRegister();
41 |     auto cm = evalHandle.Data();
42 | 
43 |     for (size_t i = 0; i < cm.BatchNum(); ++i)
44 |     {
45 |         assert(cm[i] == check[i]);
46 |     }
47 |     cout << "done" << endl;
48 | }
49 | }
50 | 
51 | void test_batch_scalar()
52 | {
53 |     test_case1();
54 | }


--------------------------------------------------------------------------------
/Test/evaluate/test_eval_plan.cpp:
--------------------------------------------------------------------------------
 1 | #include "test_eval_plan.h"
 2 | #include "../facilities/calculate_tags.h"
 3 | #include <iostream>
 4 | #include <cassert>
 5 | #include <set>
 6 | #include <MetaNN/meta_nn.h>
 7 | using namespace std;
 8 | using namespace MetaNN;
 9 | 
10 | namespace
11 | {
12 | void TestEvalPlan1()
13 | {
14 |     cout << "Test eval plan case 1...\t";
15 |     auto rm = OneHotVector<int, CheckDevice>(100, 37);
16 |     
17 |     auto eh = rm.EvalRegister();
18 |     try
19 |     {
20 |         eh.Data();
21 |         assert(false);
22 |     }
23 |     catch (std::runtime_error&)
24 |     {
25 |     }
26 |     cout << "done" << endl;
27 | }
28 | 
29 | void TestEvalPlan2()
30 | {
31 |     cout << "Test eval plan case 2...\t";
32 |     auto rm = OneHotVector<int, CheckDevice>(100, 37);
33 |     
34 |     auto eh1 = rm.EvalRegister();
35 |     EvalPlan<CheckDevice>::Eval();
36 |     auto eh2 = rm.EvalRegister();
37 |     assert(eh1.Data() == eh2.Data());
38 |     cout << "done" << endl;
39 | }
40 | 
41 | void TestEvalPlan3()
42 | {
43 |     cout << "Test eval plan case 3...\t";
44 |     auto rm = OneHotVector<int, CheckDevice>(100, 37);
45 |     auto rm2 = rm;
46 |     
47 |     auto eh1 = rm.EvalRegister();
48 |     EvalPlan<CheckDevice>::Eval();
49 |     auto eh2 = rm2.EvalRegister();
50 |     assert(eh1.Data() == eh2.Data());
51 |     cout << "done" << endl;
52 | }
53 | 
54 | void TestEvalPlan4()
55 | {
56 |     cout << "Test eval plan case 4...\t";
57 |     auto rm = OneHotVector<int, CheckDevice>(100, 37);
58 |     
59 |     auto eh1 = rm.EvalRegister();
60 |     auto eh2 = rm.EvalRegister();
61 |     EvalPlan<CheckDevice>::Eval();
62 |     assert(eh1.Data() == eh2.Data());
63 |     cout << "done" << endl;
64 | }
65 | }
66 | 
67 | void test_eval_plan()
68 | {
69 |     TestEvalPlan1();
70 |     TestEvalPlan2();
71 |     TestEvalPlan3();
72 |     TestEvalPlan4();
73 | }


--------------------------------------------------------------------------------
/MetaNN/layers/elementary/add_layer.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | #include <MetaNN/facilities/var_type_dict.h>
 3 | #include <MetaNN/layers/facilities/common_io.h>
 4 | #include <MetaNN/layers/facilities/interface_fun.h>
 5 | #include <MetaNN/layers/facilities/policies.h>
 6 | #include <MetaNN/policies/policy_operations.h>
 7 | 
 8 | namespace MetaNN
 9 | {
10 | using AddLayerInput = VarTypeDict<struct AddLayerIn1, struct AddLayerIn2>;
11 | 
12 | template <typename TPolicies>
13 | class AddLayer
14 | {
15 |     static_assert(IsPolicyContainer<TPolicies>, "TPolicies is not a policy container.");
16 |     using CurLayerPolicy = PlainPolicy<TPolicies>;
17 | 
18 | public:
19 |     static constexpr bool IsFeedbackOutput = PolicySelect<FeedbackPolicy, CurLayerPolicy>::IsFeedbackOutput;
20 |     static constexpr bool IsUpdate = false;
21 |     using InputType = AddLayerInput;
22 |     using OutputType = LayerIO;
23 | 
24 | public:
25 |     template <typename TIn>
26 |     auto FeedForward(const TIn& p_in)
27 |     {
28 |         const auto& val1 = p_in.template Get<AddLayerIn1>();
29 |         const auto& val2 = p_in.template Get<AddLayerIn2>();
30 | 
31 |         using rawType1 = std::decay_t<decltype(val1)>;
32 |         using rawType2 = std::decay_t<decltype(val2)>;
33 | 
34 |         static_assert(!std::is_same<rawType1, NullParameter>::value, "parameter1 is invalid");
35 |         static_assert(!std::is_same<rawType2, NullParameter>::value, "parameter2 is invalid");
36 | 
37 |         return OutputType::Create().template Set<LayerIO>(val1 + val2);
38 |     }
39 | 
40 |     template <typename TGrad>
41 |     auto FeedBackward(TGrad&& p_grad)
42 |     {
43 |         if constexpr (IsFeedbackOutput)
44 |         {
45 |             auto res = p_grad.template Get<LayerIO>();
46 |             return AddLayerInput::Create().template Set<AddLayerIn1>(res)
47 |                                           .template Set<AddLayerIn2>(res);
48 |         }
49 |         else
50 |         {
51 |             return AddLayerInput::Create();
52 |         }
53 |     }
54 | };
55 | }
56 | 


--------------------------------------------------------------------------------
/MetaNN/data/facilities/allocators.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <MetaNN/data/facilities/tags.h>
 4 | #include <memory>
 5 | #include <mutex>
 6 | #include <unordered_map>
 7 | #include <deque>
 8 | 
 9 | namespace MetaNN
10 | {
11 | template <typename TDevice>
12 | struct Allocator;
13 | 
14 | template <>
15 | struct Allocator<DeviceTags::CPU>
16 | {
17 | private:
18 |     struct AllocHelper
19 |     {
20 |         std::unordered_map<size_t, std::deque<void*> > memBuffer;
21 |         ~AllocHelper()
22 |         {
23 |             for (auto& p : memBuffer)
24 |             {
25 |                 auto& refVec = p.second;
26 |                 for (auto& p1 : refVec)
27 |                 {
28 |                     char* buf = (char*)(p1);
29 |                     delete []buf;
30 |                 }
31 |                 refVec.clear();
32 |             }
33 |         }
34 |     };
35 | 
36 |     struct DesImpl
37 |     {
38 |         DesImpl(std::deque<void*>& p_refPool)
39 |             : m_refPool(p_refPool) {}
40 | 
41 |         void operator () (void* p_val) const
42 |         {
43 |             std::lock_guard<std::mutex> guard(GetMutex());
44 |             m_refPool.push_back(p_val);
45 |         }
46 |     private:
47 |         std::deque<void*>& m_refPool;
48 |     };
49 | 
50 | public:
51 |     template<typename T>
52 |     static std::shared_ptr<T> Allocate(size_t p_elemSize)
53 |     {
54 |         if (p_elemSize == 0)
55 |         {
56 |             return nullptr;
57 |         }
58 |         p_elemSize = (p_elemSize * sizeof(T) + 1023) & (size_t(-1) ^ 1023);
59 | 
60 |         std::lock_guard<std::mutex> guard(GetMutex());
61 | 
62 |         static AllocHelper allocateHelper;
63 |         auto& slot = allocateHelper.memBuffer[p_elemSize];
64 |         if (slot.empty())
65 |         {
66 |             auto raw_buf = (T*)new char[p_elemSize];
67 |             return std::shared_ptr<T>(raw_buf, DesImpl(slot));
68 |         }
69 |         else
70 |         {
71 |             void* mem = slot.back();
72 |             slot.pop_back();
73 |             return std::shared_ptr<T>((T*)mem, DesImpl(slot));
74 |         }
75 |     }
76 |     
77 | private:
78 |     static std::mutex& GetMutex()
79 |     {
80 |         static std::mutex inst;
81 |         return inst;
82 |     }
83 | };
84 | }
85 | 


--------------------------------------------------------------------------------
/MetaNN/operators/facilities/category_cal.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <MetaNN/data/facilities/traits.h>
 4 | #include <tuple>
 5 | 
 6 | namespace MetaNN
 7 | {
 8 | template <typename TOpTag, typename THeadCate, typename...TRemainCate>
 9 | struct OperCategory_;
10 | 
11 | namespace NSOperCateCal
12 | {
13 | /// Same category check
14 | template <typename TCate, typename...TRemain>
15 | struct SameCate_
16 | {
17 |     constexpr static bool value = true;
18 | };
19 | 
20 | template <typename TCate, typename TCur, typename...TRemain>
21 | struct SameCate_<TCate, TCur, TRemain...>
22 | {
23 |     constexpr static bool tmp = std::is_same<TCate, TCur>::value;
24 |     constexpr static bool value = AndValue<tmp,
25 |                                            SameCate_<TCate, TRemain...>>;
26 | };
27 | 
28 | template <typename TCateCont, typename...TData>
29 | struct Data2Cate_
30 | {
31 |     using type = TCateCont;
32 | };
33 | 
34 | template <typename...TProcessed, typename TCur, typename...TRemain>
35 | struct Data2Cate_<std::tuple<TProcessed...>, TCur, TRemain...>
36 | {
37 |     using tmp1 = DataCategory<TCur>;
38 |     using tmp2 = std::tuple<TProcessed..., tmp1>;
39 |     using type = typename Data2Cate_<tmp2, TRemain...>::type;
40 | };
41 | 
42 | template <typename THead, typename...TRemain>
43 | using Data2Cate = typename Data2Cate_<std::tuple<>, THead, TRemain...>::type;
44 | 
45 | /// category sequence to category result
46 | template <typename TOpTag, typename TCateContainer>
47 | struct CateInduce_;
48 | 
49 | template <typename TOpTag, typename...TCates>
50 | struct CateInduce_<TOpTag, std::tuple<TCates...>>
51 | {
52 |     using type = typename OperCategory_<TOpTag, TCates...>::type;
53 | };
54 | }
55 | 
56 | template <typename TOpTag, typename THeadCate, typename...TRemainCate>
57 | struct OperCategory_
58 | {
59 |     static_assert(NSOperCateCal::SameCate_<THeadCate, TRemainCate...>::value,
60 |                   "Data category mismatch.");
61 |     using type = THeadCate;
62 | };
63 | 
64 | template <typename TOpTag, typename THead, typename...TRemain>
65 | using OperCateCal = typename NSOperCateCal::CateInduce_<TOpTag, 
66 |                                                         NSOperCateCal::Data2Cate<THead, TRemain...>>::type;
67 | }


--------------------------------------------------------------------------------
/Test/operators/test_collapse.cpp:
--------------------------------------------------------------------------------
 1 | #include "test_collapse.h"
 2 | #include "../facilities/data_gen.h"
 3 | 
 4 | #include <MetaNN/meta_nn.h>
 5 | #include <cassert>
 6 | #include <cmath>
 7 | #include <iostream>
 8 | using namespace MetaNN;
 9 | using namespace std;
10 | 
11 | namespace
12 | {
13 | void test_collapse1()
14 | {
15 |     cout << "Test collapse case 1 ...\t";
16 |     auto rm1 = GenBatchMatrix<float>(4, 5, 7, 1.0f, 0.0001f);
17 |     auto t = Collapse(rm1);
18 |     assert(t.RowNum() == 4);
19 |     assert(t.ColNum() == 5);
20 |     
21 |     auto handle = t.EvalRegister();
22 |     EvalPlan<DeviceTags::CPU>::Eval();
23 |     auto t_r = handle.Data();
24 |     
25 |     for (size_t i = 0; i < 4; ++i)
26 |     {
27 |         for (size_t j = 0; j<5; ++j)
28 |         {
29 |             float aim = 0;
30 |             for (size_t k = 0; k < 7; ++k)
31 |             {
32 |                 aim += rm1[k](i, j);
33 |             }
34 |             assert(fabs(t_r(i, j) - aim) < 0.0001);
35 |         }
36 |     }
37 |     cout << "done" << endl;
38 | }
39 | 
40 | void test_collapse2()
41 | {
42 |     cout << "Test collapse case 2 ...\t";
43 |     {
44 |         auto rm1 = GenBatchMatrix<float>(4, 5, 7, 1.0f, 0.0001f);
45 |         auto collapse1 = Collapse(rm1);
46 |         auto collapse2 = Collapse(rm1);
47 | 
48 |         assert(collapse1 == collapse2);
49 | 
50 |         auto handle1 = collapse1.EvalRegister();
51 |         auto handle2 = collapse1.EvalRegister();
52 |         EvalPlan<DeviceTags::CPU>::Eval();
53 |         auto cm1 = handle1.Data();
54 |         auto cm2 = handle2.Data();
55 |         assert(cm1 == cm2);
56 |     }
57 |     {
58 |         auto rm1 = GenBatchMatrix<float>(4, 5, 7, 1.0f, 0.0001f);
59 |         auto collapse1 = Collapse(rm1);
60 |         auto collapse2 = collapse1;
61 | 
62 |         assert(collapse1 == collapse2);
63 | 
64 |         const auto& evalHandle1 = collapse1.EvalRegister();
65 |         const auto& evalHandle2 = collapse2.EvalRegister();
66 |         EvalPlan<DeviceTags::CPU>::Eval();
67 | 
68 |         auto cm1 = evalHandle1.Data();
69 |         auto cm2 = evalHandle2.Data();
70 |         assert(cm1 == cm2);
71 |     }
72 |     cout << "done" << endl;
73 | }
74 | }
75 | 
76 | void test_collapse()
77 | {
78 |     test_collapse1();
79 |     test_collapse2();
80 | }
81 | 


--------------------------------------------------------------------------------
/MetaNN/policies/change_policy.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <MetaNN/policies/policy_container.h>
 4 | 
 5 | namespace MetaNN
 6 | {
 7 | template <typename TNewPolicy, typename TOriContainer>
 8 | struct ChangePolicy_;
 9 | 
10 | template <typename TNewPolicy, typename... TPolicies>
11 | struct ChangePolicy_<TNewPolicy, PolicyContainer<TPolicies...>>
12 | {
13 | private:
14 |     using newMajor = typename TNewPolicy::MajorClass;
15 |     using newMinor = typename TNewPolicy::MinorClass;
16 | 
17 |     template <typename TPC, typename... TP> struct DropAppend_;
18 | 
19 |     template <typename... TFilteredPolicies>
20 |     struct DropAppend_<PolicyContainer<TFilteredPolicies...>>
21 |     {
22 |         using type = PolicyContainer<TFilteredPolicies..., TNewPolicy>;
23 |     };
24 | 
25 |     template <typename... TFilteredPolicies, typename TCurPolicy, typename... TP>
26 |     struct DropAppend_<PolicyContainer<TFilteredPolicies...>,
27 |                        TCurPolicy, TP...>
28 |     {
29 |         template <bool isArray, typename TDummy = void>
30 |         struct ArrayBasedSwitch_
31 |         {
32 |             template <typename TMajor, typename TMinor, typename TD = void>
33 |             struct _impl
34 |             {
35 |                 using type = PolicyContainer<TFilteredPolicies..., TCurPolicy>;
36 |             };
37 | 
38 |             template <typename TD>
39 |             struct _impl<newMajor, newMinor, TD>
40 |             {
41 |                 using type = PolicyContainer<TFilteredPolicies...>;
42 |             };
43 |             using type = typename _impl<typename TCurPolicy::MajorClass,
44 |                                         typename TCurPolicy::MinorClass>::type;
45 |         };
46 | 
47 |         template <typename TDummy>
48 |         struct ArrayBasedSwitch_<true, TDummy>
49 |         {
50 |             using type = PolicyContainer<TFilteredPolicies..., TCurPolicy>;
51 |         };
52 | 
53 |         using t1 = typename ArrayBasedSwitch_<IsSubPolicyContainer<TCurPolicy>>::type;
54 |         using type = typename DropAppend_<t1, TP...>::type;
55 |     };
56 | 
57 | public:
58 |     using type = typename DropAppend_<PolicyContainer<>, TPolicies...>::type;
59 | };
60 | 
61 | template <typename TNewPolicy, typename TOriContainer>
62 | using ChangePolicy = typename ChangePolicy_<TNewPolicy, TOriContainer>::type;
63 | }


--------------------------------------------------------------------------------
/MetaNN/facilities/traits.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <cstddef>
 4 | 
 5 | namespace MetaNN
 6 | {
 7 | template <typename T>
 8 | struct Identity_
 9 | {
10 |     using type = T;
11 | };
12 | 
13 | // Or
14 | template <bool cur, typename TNext>
15 | constexpr static bool OrValue = true;
16 | 
17 | template <typename TNext>
18 | constexpr static bool OrValue<false, TNext> = TNext::value;
19 | 
20 | // And
21 | template <bool cur, typename TNext>
22 | constexpr static bool AndValue = false;
23 | 
24 | template <typename TNext>
25 | constexpr static bool AndValue<true, TNext> = TNext::value;
26 | 
27 | // Array empty
28 | template <typename TArray>
29 | struct IsArrayEmpty_;
30 | 
31 | template <template <typename...> class Cont, typename T1, typename...T2>
32 | struct IsArrayEmpty_<Cont<T1, T2...>>
33 | {
34 |     constexpr static bool value = false;
35 | };
36 | 
37 | template <template <typename...> class Cont>
38 | struct IsArrayEmpty_<Cont<>>
39 | {
40 |     constexpr static bool value = true;
41 | };
42 | 
43 | template <typename TArray>
44 | constexpr static bool IsArrayEmpty = IsArrayEmpty_<TArray>::value;
45 | 
46 | // Array size
47 | template <typename TArray>
48 | struct ArraySize_;
49 | 
50 | template <template <typename...> class Cont, typename...T>
51 | struct ArraySize_<Cont<T...>>
52 | {
53 |     constexpr static size_t value = sizeof...(T);
54 | };
55 | 
56 | template <typename TArray>
57 | constexpr static size_t ArraySize = ArraySize_<TArray>::value;
58 | 
59 | template <typename TSeqCont>
60 | struct SeqHead_;
61 | 
62 | template <template <typename...> class Container, typename TH, typename...TCases>
63 | struct SeqHead_<Container<TH, TCases...>>
64 | {
65 |     using type = TH;
66 | };
67 | 
68 | template <typename TSeqCont>
69 | using SeqHead = typename SeqHead_<TSeqCont>::type;
70 | 
71 | template <typename TSeqCont>
72 | struct SeqTail_;
73 | 
74 | template <template <typename...> class Container, typename TH, typename...TCases>
75 | struct SeqTail_<Container<TH, TCases...>>
76 | {
77 |     using type = Container<TCases...>;
78 | };
79 | 
80 | template <typename TSeqCont>
81 | using SeqTail = typename SeqTail_<TSeqCont>::type;
82 | 
83 | template <typename T>
84 | constexpr static bool DependencyFalse = false;
85 | 
86 | template <typename T>
87 | using RemConstRef = std::remove_cv_t<std::remove_reference_t<T>>;
88 | }


--------------------------------------------------------------------------------
/Test/data/test_trival_matrix.cpp:
--------------------------------------------------------------------------------
 1 | #include "test_trival_matrix.h"
 2 | #include "../facilities/calculate_tags.h"
 3 | #include <iostream>
 4 | #include <cassert>
 5 | #include <MetaNN/meta_nn.h>
 6 | using namespace std;
 7 | using namespace MetaNN;
 8 | 
 9 | namespace
10 | {
11 | void TestTrivalMatrix1()
12 | {
13 |     cout << "Test trival matrix case 1...\t";
14 |     static_assert(IsMatrix<TrivalMatrix<int, CheckDevice, Scalar<int, DeviceTags::CPU>>>, "Test Error");
15 |     static_assert(IsMatrix<TrivalMatrix<int, CheckDevice, Scalar<int, DeviceTags::CPU>> &>, "Test Error");
16 |     static_assert(IsMatrix<TrivalMatrix<int, CheckDevice, Scalar<int, DeviceTags::CPU>> &&>, "Test Error");
17 |     static_assert(IsMatrix<const TrivalMatrix<int, CheckDevice, Scalar<int, DeviceTags::CPU>> &>, "Test Error");
18 |     static_assert(IsMatrix<const TrivalMatrix<int, CheckDevice, Scalar<int, DeviceTags::CPU>> &&>, "Test Error");
19 | 
20 |     auto rm = MakeTrivalMatrix<int, CheckDevice>(10, 20, 100);
21 |     assert(rm.RowNum() == 10);
22 |     assert(rm.ColNum() == 20);
23 | 
24 |     const auto& evalHandle = rm.EvalRegister();
25 |     EvalPlan<DeviceTags::CPU>::Eval();
26 | 
27 |     auto rm1 = evalHandle.Data();
28 |     for (size_t i=0; i<10; ++i)
29 |     {
30 |         for (size_t j=0; j<20; ++j)
31 |         {
32 |             assert(rm1(i, j) == 100);
33 |         }
34 |     }
35 | 
36 |     cout << "done" << endl;
37 | }
38 | 
39 | void TestTrivalMatrix2()
40 | {
41 |     cout << "Test trival matrix case 2...\t";
42 |     auto rm1 = MakeTrivalMatrix<int, CheckDevice>(100, 10, 14);
43 |     auto rm2 = MakeTrivalMatrix<int, CheckDevice>(10, 20, 35);
44 | 
45 |     const auto& evalRes1 = rm1.EvalRegister();
46 |     const auto& evalRes2 = rm2.EvalRegister();
47 | 
48 |     EvalPlan<DeviceTags::CPU>::Eval();
49 |     for (size_t j = 0; j < 100; ++j)
50 |     {
51 |         for (size_t k = 0; k < 10; ++k)
52 |         {
53 |             assert(evalRes1.Data()(j, k) == 14);
54 |         }
55 |     }
56 | 
57 |     for (size_t j = 0; j < 10; ++j)
58 |     {
59 |         for (size_t k = 0; k < 20; ++k)
60 |         {
61 |             assert(evalRes2.Data()(j, k) == 35);
62 |         }
63 |     }
64 | 
65 |     cout << "done" << endl;
66 | }
67 | }
68 | 
69 | void test_trival_matrix()
70 | {
71 |     TestTrivalMatrix1();
72 |     TestTrivalMatrix2();
73 | }
74 | 


--------------------------------------------------------------------------------
/MetaNN/layers/facilities/traits.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <MetaNN/data/dynamic.h>
 4 | #include <MetaNN/model_rel/grad_col/grad_collector.h>
 5 | #include <stack>
 6 | #include <stdexcept>
 7 | 
 8 | namespace MetaNN
 9 | {
10 | namespace LayerTraits
11 | {
12 | template <typename ElementType, typename DeviceType, typename CateType>
13 | struct LayerInternalBufType_
14 | {
15 |     using tmp2 = DynamicData<ElementType, DeviceType, CateType>;
16 |     using type = std::stack<tmp2, std::list<tmp2>>;
17 | };
18 | 
19 | template <bool triger, bool batchMode, 
20 |           typename ElementType, typename DeviceType,
21 |           typename CateTypeSingle, typename CateTypeBatch>
22 | struct LayerInternalBuf_
23 | {
24 |     using type = typename std::conditional_t<batchMode,
25 |                                         LayerTraits::LayerInternalBufType_<ElementType, DeviceType,
26 |                                                                            CateTypeBatch>,
27 |                                         LayerTraits::LayerInternalBufType_<ElementType, DeviceType,
28 |                                                                            CateTypeSingle>>::type;
29 | };
30 | 
31 | template <bool batchMode, 
32 |           typename ElementType, typename DeviceType,
33 |           typename CateTypeSingle, typename CateTypeBatch>
34 | struct LayerInternalBuf_<false, batchMode, ElementType, DeviceType, CateTypeSingle, CateTypeBatch>
35 | {
36 |     using type = NullParameter;
37 | };
38 | 
39 | template <bool triger, bool batchMode, 
40 |           typename ElementType, typename DeviceType,
41 |           typename CateTypeSingle, typename CateTypeBatch>
42 | using LayerInternalBuf = typename LayerInternalBuf_<triger, batchMode, ElementType, DeviceType, CateTypeSingle, CateTypeBatch>::type;
43 | 
44 | template <typename ElementType, typename DeviceType, typename CateType>
45 | using LayerInternalBufType = typename LayerInternalBufType_<ElementType, DeviceType, CateType>::type;    
46 | 
47 | template <typename TWeight, typename TGrad, typename TGradCollector>
48 | void MatrixGradCollect(const TWeight& weight,
49 |                        TGrad& grad,
50 |                        TGradCollector& col)
51 | {
52 |     while (!grad.empty())
53 |     {
54 |         auto g = grad.top();
55 |         grad.pop();
56 |         col.Collect(weight, g);
57 |     }
58 | }
59 | }
60 | }


--------------------------------------------------------------------------------
/Test/data/test_one_hot_vector.cpp:
--------------------------------------------------------------------------------
 1 | #include "test_one_hot_vector.h"
 2 | #include "../facilities/calculate_tags.h"
 3 | #include <iostream>
 4 | #include <cassert>
 5 | #include <MetaNN/meta_nn.h>
 6 | using namespace std;
 7 | using namespace MetaNN;
 8 | 
 9 | namespace
10 | {
11 | void test_one_hot_vector1()
12 | {
13 |     cout << "Test one-hot vector case 1...\t";
14 |     static_assert(IsMatrix<OneHotVector<int, CheckDevice>>, "Test Error");
15 |     static_assert(IsMatrix<OneHotVector<int, CheckDevice> &>, "Test Error");
16 |     static_assert(IsMatrix<OneHotVector<int, CheckDevice> &&>, "Test Error");
17 |     static_assert(IsMatrix<const OneHotVector<int, CheckDevice> &>, "Test Error");
18 |     static_assert(IsMatrix<const OneHotVector<int, CheckDevice> &&>, "Test Error");
19 | 
20 |     auto rm = OneHotVector<int, CheckDevice>(100, 37);
21 |     assert(rm.RowNum() == 1);
22 |     assert(rm.ColNum() == 100);
23 |     assert(rm.HotPos() == 37);
24 | 
25 |     auto rm1 = Evaluate(rm);
26 |     for (size_t i=0; i<1; ++i)
27 |     {
28 |         for (size_t j=0; j<100; ++j)
29 |         {
30 |             if (j != 37)
31 |             {
32 |                 assert(rm1(i, j) == 0);
33 |             }
34 |             else
35 |             {
36 |                 assert(rm1(i, j) == 1);
37 |             }
38 |         }
39 |     }
40 | 
41 |     cout << "done" << endl;
42 | }
43 | 
44 | void test_one_hot_vector2()
45 | {
46 |     cout << "Test one-hot vector case 2...\t";
47 |     auto rm1 = OneHotVector<int, CheckDevice>(100, 37);
48 |     auto rm2 = OneHotVector<int, CheckDevice>(50, 16);
49 | 
50 |     auto evalRes1 = rm1.EvalRegister();
51 |     auto evalRes2 = rm2.EvalRegister();
52 | 
53 |     EvalPlan<DeviceTags::CPU>::Eval();
54 |     for (size_t j = 0; j < 100; ++j)
55 |     {
56 |         if (j == 37)
57 |         {
58 |             assert(evalRes1.Data()(0, j) == 1);
59 |         }
60 |         else
61 |         {
62 |             assert(evalRes1.Data()(0, j) == 0);
63 |         }
64 |     }
65 | 
66 |     for (size_t j = 0; j < 50; ++j)
67 |     {
68 |         if (j == 16)
69 |         {
70 |             assert(evalRes2.Data()(0, j) == 1);
71 |         }
72 |         else
73 |         {
74 |             assert(evalRes2.Data()(0, j) == 0);
75 |         }
76 |     }
77 |     cout << "done" << endl;
78 | }
79 | }
80 | 
81 | void test_one_hot_vector()
82 | {
83 |     test_one_hot_vector1();
84 |     test_one_hot_vector2();
85 | }
86 | 


--------------------------------------------------------------------------------
/MetaNN/data/batch/scalar.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <MetaNN/data/batch/batch.h>
 4 | #include <MetaNN/data/facilities/continuous_memory.h>
 5 | #include <MetaNN/data/facilities/traits.h>
 6 | #include <MetaNN/data/facilities/tags.h>
 7 | #include <MetaNN/evaluate/facilities/eval_handle.h>
 8 | #include <cassert>
 9 | 
10 | namespace MetaNN
11 | {
12 | template <typename TElem>
13 | class Batch<TElem, DeviceTags::CPU, CategoryTags::Scalar>
14 | {
15 | public:
16 |     using ElementType = TElem;
17 |     using DeviceType = DeviceTags::CPU;
18 |     
19 |     friend LowerAccessImpl<Batch<TElem, DeviceTags::CPU, CategoryTags::Scalar>>;
20 |     
21 | public:
22 |     Batch(size_t length = 0)
23 |         : m_mem(length)
24 |         , m_len(length) {}
25 |      
26 |     size_t BatchNum() const { return m_len; }
27 | 
28 |     bool AvailableForWrite() const { return m_mem.UseCount() == 1; }
29 | 
30 |     void SetValue(size_t p_id, ElementType val)
31 |     {
32 |         assert(AvailableForWrite());
33 |         assert(p_id < m_len);
34 |         (m_mem.RawMemory())[p_id] = val;
35 |     }
36 |     
37 |     const auto operator[](size_t p_id) const
38 |     {
39 |         assert(p_id < m_len);
40 |         return (m_mem.RawMemory())[p_id];
41 |     }
42 |    
43 |     bool operator== (const Batch& val) const
44 |     {
45 |         return (m_mem == val.m_mem) && (m_len == val.m_len);
46 |     }
47 | 
48 |     template <typename TOtherType>
49 |     bool operator== (const TOtherType&) const
50 |     {
51 |         return false;
52 |     }
53 | 
54 |     template <typename TData>
55 |     bool operator!= (const TData& val) const
56 |     {
57 |         return !(operator==(val));
58 |     }
59 |     
60 |     auto EvalRegister() const
61 |     {
62 |         return MakeConstEvalHandle(*this);
63 |     }
64 |     
65 | private:
66 |     ContinuousMemory<ElementType, DeviceType> m_mem;
67 |     size_t m_len;
68 | };
69 | 
70 | template<typename TElem>
71 | struct LowerAccessImpl<Batch<TElem, DeviceTags::CPU, CategoryTags::Scalar>>
72 | {
73 |     LowerAccessImpl(Batch<TElem, DeviceTags::CPU, CategoryTags::Scalar> p)
74 |         : m_data(std::move(p))
75 |     {}
76 | 
77 |     auto MutableRawMemory()
78 |     {
79 |         return m_data.m_mem.RawMemory();
80 |     }
81 | 
82 |     const auto RawMemory() const
83 |     {
84 |         return m_data.m_mem.RawMemory();
85 |     }
86 | 
87 | private:
88 |     Batch<TElem, DeviceTags::CPU, CategoryTags::Scalar> m_data;
89 | };
90 | }


--------------------------------------------------------------------------------
/Test/operators/test_transpose.cpp:
--------------------------------------------------------------------------------
 1 | #include "test_transpose.h"
 2 | #include "../facilities/data_gen.h"
 3 | #include <MetaNN/meta_nn.h>
 4 | #include <cassert>
 5 | #include <iostream>
 6 | using namespace MetaNN;
 7 | using namespace std;
 8 | 
 9 | namespace
10 | {
11 | void test_transpose1()
12 | {
13 |     cout << "Test transpose case 1 ...\t";
14 |     auto rm1 = GenMatrix<int>(4, 5, 0, 1);
15 |     auto tr = Transpose(rm1);
16 |     auto tr_r = Evaluate(tr);
17 |     for (size_t i = 0; i < 5; ++i)
18 |     {
19 |         for (size_t j = 0; j<4; ++j)
20 |         {
21 |             assert(tr_r(i, j) == rm1(j, i));
22 |         }
23 |     }
24 | 
25 |     rm1 = GenMatrix<int>(111, 113, 2, 3);
26 |     rm1 = rm1.SubMatrix(31, 35, 17, 22);
27 |     tr = Transpose(rm1);
28 |     tr_r = Evaluate(tr);
29 |     for (size_t i = 0; i < 5; ++i)
30 |     {
31 |         for (size_t j = 0; j<4; ++j)
32 |         {
33 |             assert(tr_r(i, j) == rm1(j, i));
34 |         }
35 |     }
36 |     cout << "done" << endl;
37 | }
38 | 
39 | void test_transpose2()
40 | {
41 |     cout << "Test transpose case 2 ...\t";
42 |     {
43 |         auto rm1 = GenMatrix<int>(4, 5, 0, 1);
44 |         auto res = Transpose(rm1);
45 |         auto res2 = Transpose(rm1);
46 | 
47 |         assert(res == res2);
48 | 
49 |         auto cm1 = Evaluate(res);
50 |         auto cm2 = Evaluate(res);
51 |         assert(cm1 == cm2);
52 |     }
53 |     {
54 |         auto rm1 = GenMatrix<int>(4, 5, 0, 1);
55 |         auto res = Transpose(rm1);
56 |         auto res2 = res;
57 | 
58 |         assert(res == res2);
59 | 
60 |         const auto& evalHandle1 = res.EvalRegister();
61 |         const auto& evalHandle2 = res2.EvalRegister();
62 |         EvalPlan<DeviceTags::CPU>::Eval();
63 | 
64 |         auto cm1 = evalHandle1.Data();
65 |         auto cm2 = evalHandle2.Data();
66 |     }
67 |     cout << "done" << endl;
68 | }
69 | 
70 | void test_transpose3()
71 | {
72 |     cout << "Test transpose case 3 ...\t";
73 |     auto rm1 = GenBatchMatrix<int>(4, 5, 7, 0, 1);
74 |     auto tr = Transpose(rm1);
75 |     auto tr_r = Evaluate(tr);
76 |     for (size_t b = 0; b < 7; ++b)
77 |     {
78 |         for (size_t i = 0; i < 5; ++i)
79 |         {
80 |             for (size_t j = 0; j<4; ++j)
81 |             {
82 |                 assert(tr_r[b](i, j) == rm1[b](j, i));
83 |             }
84 |         }
85 |     }
86 |     cout << "done" << endl;
87 | }
88 | }
89 | 
90 | void test_transpose()
91 | {
92 |     test_transpose1();
93 |     test_transpose2();
94 |     test_transpose3();
95 | }
96 | 


--------------------------------------------------------------------------------
/MetaNN/layers/facilities/policies.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <MetaNN/policies/policy_macro_begin.h>
 4 | #include <MetaNN/data/facilities/tags.h>
 5 | namespace MetaNN
 6 | {
 7 | struct FeedbackPolicy
 8 | {
 9 |     using MajorClass = FeedbackPolicy;
10 |     
11 |     struct IsUpdateValueCate;
12 |     struct IsFeedbackOutputValueCate;
13 | 
14 |     static constexpr bool IsUpdate = false;
15 |     static constexpr bool IsFeedbackOutput = false;
16 | };
17 | ValuePolicyObj(PUpdate,   FeedbackPolicy, IsUpdate, true);
18 | ValuePolicyObj(PNoUpdate, FeedbackPolicy, IsUpdate, false);
19 | ValuePolicyObj(PFeedbackOutput,   FeedbackPolicy, IsFeedbackOutput, true);
20 | ValuePolicyObj(PFeedbackNoOutput, FeedbackPolicy, IsFeedbackOutput, false);
21 | 
22 | struct InputPolicy
23 | {
24 |     using MajorClass = InputPolicy;
25 |     
26 |     struct BatchModeValueCate;
27 |     static constexpr bool BatchMode = false;
28 | };
29 | ValuePolicyObj(PBatchMode,  InputPolicy, BatchMode, true);
30 | ValuePolicyObj(PNoBatchMode,InputPolicy, BatchMode, false);
31 | 
32 | struct OperandPolicy
33 | {
34 |     using MajorClass = OperandPolicy;
35 |     
36 |     struct DeviceTypeCate : public MetaNN::DeviceTags {};
37 |     using Device = DeviceTypeCate::CPU;
38 |     
39 |     struct ElementTypeCate;
40 |     using Element = float;
41 | };
42 | TypePolicyObj(PCPUDevice, OperandPolicy, Device, CPU);
43 | TypePolicyTemplate(PElementTypeIs, OperandPolicy, Element);
44 | 
45 | struct SingleLayerPolicy
46 | {
47 |     using MajorClass = SingleLayerPolicy;
48 |     
49 |     struct ActionTypeCate
50 |     {
51 |         struct Sigmoid;
52 |         struct Tanh;
53 |     };
54 |     struct HasBiasValueCate;
55 | 
56 |     using Action = ActionTypeCate::Sigmoid;
57 |     static constexpr bool HasBias = true;
58 | };
59 | TypePolicyObj(PSigmoidAction, SingleLayerPolicy, Action, Sigmoid);
60 | TypePolicyObj(PTanhAction, SingleLayerPolicy, Action, Tanh);
61 | ValuePolicyObj(PBiasSingleLayer,  SingleLayerPolicy, HasBias, true);
62 | ValuePolicyObj(PNoBiasSingleLayer, SingleLayerPolicy, HasBias, false);
63 | 
64 | struct RecurrentLayerPolicy
65 | {
66 |     using MajorClass = RecurrentLayerPolicy;
67 |     
68 |     struct StepTypeCate
69 |     {
70 |         struct GRU;
71 |     };
72 |     struct UseBpttValueCate;
73 | 
74 |     using Step = StepTypeCate::GRU;
75 |     constexpr static bool UseBptt = true;
76 | };
77 | TypePolicyObj(PRecGRUStep, RecurrentLayerPolicy, Step, GRU);
78 | ValuePolicyObj(PEnableBptt,  RecurrentLayerPolicy, UseBptt, true);
79 | ValuePolicyObj(PDisableBptt,  RecurrentLayerPolicy, UseBptt, false);
80 | }
81 | #include <MetaNN/policies/policy_macro_end.h>
82 | 


--------------------------------------------------------------------------------
/Test/operators/test_tanh.cpp:
--------------------------------------------------------------------------------
 1 | #include "test_tanh.h"
 2 | #include "../facilities/data_gen.h"
 3 | #include <MetaNN/meta_nn.h>
 4 | #include <cassert>
 5 | #include <iostream>
 6 | using namespace MetaNN;
 7 | using namespace std;
 8 | 
 9 | namespace
10 | {
11 | void test_tanh1()
12 | {
13 |     cout << "Test tanh case 1 ...\t";
14 |     auto rm1 = GenMatrix<float>(4, 5, 1, 0.0001f);
15 |     auto t = Tanh(rm1);
16 |     auto t_r = Evaluate(t);
17 |     for (size_t i = 0; i < 4; ++i)
18 |     {
19 |         for (size_t j = 0; j<5; ++j)
20 |         {
21 |             float aim = tanh(rm1(i, j));
22 |             assert(fabs(t_r(i, j) - aim) < 0.0001);
23 |         }
24 |     }
25 | 
26 |     rm1 = GenMatrix<float>(111, 113, 2, 0.001f);
27 |     rm1 = rm1.SubMatrix(31, 35, 17, 22);
28 |     t = Tanh(rm1);
29 |     t_r = Evaluate(t);
30 |     for (size_t i = 0; i < 4; ++i)
31 |     {
32 |         for (size_t j = 0; j<5; ++j)
33 |         {
34 |             float aim = tanh(rm1(i, j));
35 |             assert(fabs(t_r(i, j) - aim) < 0.0001);
36 |         }
37 |     }
38 |     cout << "done" << endl;
39 | }
40 | 
41 | void test_tanh2()
42 | {
43 |     cout << "Test tanh case 2 ...\t";
44 |     {
45 |         auto rm1 = GenMatrix<float>(4, 5, 1, 2);
46 |         auto res = Tanh(rm1);
47 |         auto res2 = Tanh(rm1);
48 | 
49 |         assert(res == res2);
50 | 
51 |         auto cm1 = Evaluate(res);
52 |         auto cm2 = Evaluate(res);
53 |         assert(cm1 == cm2);
54 |     }
55 |     {
56 |         auto rm1 = GenMatrix<float>(4, 5, 1, 2);
57 |         auto res = Tanh(rm1);
58 |         auto res2 = res;
59 | 
60 |         assert(res == res2);
61 | 
62 |         const auto& evalHandle1 = res.EvalRegister();
63 |         const auto& evalHandle2 = res2.EvalRegister();
64 |         EvalPlan<DeviceTags::CPU>::Eval();
65 | 
66 |         auto cm1 = evalHandle1.Data();
67 |         auto cm2 = evalHandle2.Data();
68 |     }
69 |     cout << "done" << endl;
70 | }
71 | 
72 | void test_tanh3()
73 | {
74 |     cout << "Test tanh case 3 ...\t";
75 |     auto rm1 = GenBatchMatrix<float>(4, 5, 7, 1, 0.0001f);
76 |     auto t = Tanh(rm1);
77 |     auto t_r = Evaluate(t);
78 |     for (size_t b = 0; b < 7; ++b)
79 |     {
80 |         for (size_t i = 0; i < 4; ++i)
81 |         {
82 |             for (size_t j = 0; j<5; ++j)
83 |             {
84 |                 float aim = tanh(rm1[b](i, j));
85 |                 assert(fabs(t_r[b](i, j) - aim) < 0.0001);
86 |             }
87 |         }        
88 |     }
89 |     cout << "done" << endl;
90 | }
91 | }
92 | 
93 | void test_tanh()
94 | {
95 |     test_tanh1();
96 |     test_tanh2();
97 |     test_tanh3();
98 | }
99 | 


--------------------------------------------------------------------------------
/MetaNN/meta_nn.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <MetaNN/policies/policy_operations.h>
 4 | #include <MetaNN/policies/policy_selector.h>
 5 | #include <MetaNN/policies/change_policy.h>
 6 | #include <MetaNN/facilities/var_type_dict.h>
 7 | #include <MetaNN/layers/facilities/policies.h>
 8 | 
 9 | #include <MetaNN/data/dynamic.h>
10 | #include <MetaNN/data/scalar.h>
11 | #include <MetaNN/data/matrices/cpu_matrix.h>
12 | #include <MetaNN/data/matrices/trival_matrix.h>
13 | #include <MetaNN/data/matrices/one_hot_vector.h>
14 | #include <MetaNN/data/matrices/zero_matrix.h>
15 | 
16 | #include <MetaNN/data/batch/array.h>
17 | #include <MetaNN/data/batch/duplicate.h>
18 | #include <MetaNN/data/batch/scalar.h>
19 | #include <MetaNN/data/batch/matrix.h>
20 | 
21 | #include <MetaNN/operators/abs.h>
22 | #include <MetaNN/operators/add.h>
23 | #include <MetaNN/operators/collapse.h>
24 | #include <MetaNN/operators/divide.h>
25 | #include <MetaNN/operators/dot.h>
26 | #include <MetaNN/operators/element_mul.h>
27 | #include <MetaNN/operators/interpolate.h>
28 | #include <MetaNN/operators/negative_log_likelihood.h>
29 | #include <MetaNN/operators/negative_log_likelihood_derivative.h>
30 | #include <MetaNN/operators/sigmoid.h>
31 | #include <MetaNN/operators/sigmoid_derivative.h>
32 | #include <MetaNN/operators/sign.h>
33 | #include <MetaNN/operators/softmax.h>
34 | #include <MetaNN/operators/softmax_derivative.h>
35 | #include <MetaNN/operators/substract.h>
36 | #include <MetaNN/operators/tanh.h>
37 | #include <MetaNN/operators/tanh_derivative.h>
38 | #include <MetaNN/operators/transpose.h>
39 | 
40 | #include <MetaNN/data_copy/data_copy.h>
41 | 
42 | #include <MetaNN/policies/inject_policy.h>
43 | #include <MetaNN/layers/elementary/abs_layer.h>
44 | #include <MetaNN/layers/elementary/add_layer.h>
45 | #include <MetaNN/layers/elementary/bias_layer.h>
46 | #include <MetaNN/layers/elementary/element_mul_layer.h>
47 | #include <MetaNN/layers/elementary/interpolate_layer.h>
48 | #include <MetaNN/layers/elementary/sigmoid_layer.h>
49 | #include <MetaNN/layers/elementary/softmax_layer.h>
50 | #include <MetaNN/layers/elementary/tanh_layer.h>
51 | #include <MetaNN/layers/elementary/weight_layer.h>
52 | 
53 | #include <MetaNN/layers/compose/linear_layer.h>
54 | #include <MetaNN/layers/compose/single_layer.h>
55 | 
56 | #include <MetaNN/layers/recurrent/recurrent_layer.h>
57 | 
58 | #include <MetaNN/layers/cost/negative_log_likelihood_layer.h>
59 | 
60 | #include <MetaNN/model_rel/param_initializer/constant_filler.h>
61 | #include <MetaNN/model_rel/param_initializer/gaussian_filler.h>
62 | #include <MetaNN/model_rel/param_initializer/uniform_filler.h>
63 | #include <MetaNN/model_rel/param_initializer/var_scale_filler.h>
64 | #include <MetaNN/model_rel/param_initializer/param_initializer.h>


--------------------------------------------------------------------------------
/Test/data/test_duplicate.cpp:
--------------------------------------------------------------------------------
 1 | #include "test_duplicate.h"
 2 | #include "../facilities/calculate_tags.h"
 3 | #include <iostream>
 4 | #include <cassert>
 5 | #include <set>
 6 | #include <MetaNN/meta_nn.h>
 7 | using namespace std;
 8 | using namespace MetaNN;
 9 | 
10 | namespace
11 | {
12 | void TestDuplicate1()
13 | {
14 |     cout << "Test duplicate case 1 (matrix)...\t";
15 |     static_assert(IsBatchMatrix<Duplicate<Matrix<CheckElement, CheckDevice>>>, "Test Error");
16 |     static_assert(IsBatchMatrix<Duplicate<Matrix<CheckElement, CheckDevice>> &>, "Test Error");
17 |     static_assert(IsBatchMatrix<Duplicate<Matrix<CheckElement, CheckDevice>> &&>, "Test Error");
18 |     static_assert(IsBatchMatrix<const Duplicate<Matrix<CheckElement, CheckDevice>> &>, "Test Error");
19 |     static_assert(IsBatchMatrix<const Duplicate<Matrix<CheckElement, CheckDevice>> &&>, "Test Error");
20 | 
21 |     auto me1 = Matrix<CheckElement, CheckDevice>(10, 20);
22 |     int c = 0;
23 |     for (size_t i = 0; i < 10; ++i)
24 |     {
25 |         for (size_t j = 0; j < 20; ++j)
26 |         {
27 |             me1.SetValue(i, j, (float)(c++));
28 |         }
29 |     }
30 |     auto rm1 = MakeDuplicate(13, me1);
31 |     assert(rm1.BatchNum() == 13);
32 |     assert(rm1.RowNum() == 10);
33 |     assert(rm1.ColNum() == 20);
34 |     
35 |     auto rm2 = Evaluate(rm1);
36 |     for (size_t i = 0; i < 13; ++i)
37 |     {
38 |         for (size_t j = 0; j < 10; ++j)
39 |         {
40 |             for (size_t k = 0; k < 20; ++k)
41 |             {
42 |                 assert(rm2[i](j, k) == me1(j, k));
43 |             }
44 |         }
45 |     }
46 |     cout << "done" << endl;
47 | }
48 | 
49 | void TestDuplicate2()
50 | {
51 |     cout << "Test duplicate case 2 (scalar)...\t";
52 |     static_assert(IsBatchScalar<Duplicate<Scalar<CheckElement, CheckDevice>>>, "Test Error");
53 |     static_assert(IsBatchScalar<Duplicate<Scalar<CheckElement, CheckDevice>> &>, "Test Error");
54 |     static_assert(IsBatchScalar<Duplicate<Scalar<CheckElement, CheckDevice>> &&>, "Test Error");
55 |     static_assert(IsBatchScalar<const Duplicate<Scalar<CheckElement, CheckDevice>> &>, "Test Error");
56 |     static_assert(IsBatchScalar<const Duplicate<Scalar<CheckElement, CheckDevice>> &&>, "Test Error");
57 | 
58 |     auto rm1 = Duplicate<Scalar<CheckElement, CheckDevice>>(3, 13);
59 |     assert(rm1.Size() == 13);
60 | 
61 |     auto evalHandle = rm1.EvalRegister();
62 |     EvalPlan<DeviceTags::CPU>::Eval();
63 |     auto rm2 = evalHandle.Data();
64 | 
65 |     for (size_t i = 0; i < 13; ++i)
66 |     {
67 |         assert(rm2[i] == 3);
68 |     }
69 |     cout << "done" << endl;
70 | }
71 | }
72 | 
73 | void test_duplicate()
74 | {
75 |     TestDuplicate1();
76 |     TestDuplicate2();
77 | }


--------------------------------------------------------------------------------
/Test/data/test_general_matrix.cpp:
--------------------------------------------------------------------------------
 1 | #include "test_general_matrix.h"
 2 | #include "../facilities/calculate_tags.h"
 3 | #include <iostream>
 4 | #include <cassert>
 5 | #include <set>
 6 | #include <MetaNN/meta_nn.h>
 7 | using namespace std;
 8 | using namespace MetaNN;
 9 | 
10 | namespace
11 | {
12 | void TestMatrix1()
13 | {
14 |     cout << "Test general matrix case 1...\t";
15 |     static_assert(IsMatrix<Matrix<CheckElement, CheckDevice>>, "Test Error");
16 |     static_assert(IsMatrix<Matrix<CheckElement, CheckDevice>&>, "Test Error");
17 |     static_assert(IsMatrix<Matrix<CheckElement, CheckDevice>&&>, "Test Error");
18 |     static_assert(IsMatrix<const Matrix<CheckElement, CheckDevice>&>, "Test Error");
19 |     static_assert(IsMatrix<const Matrix<CheckElement, CheckDevice>&&>, "Test Error");
20 | 
21 |     Matrix<CheckElement, CheckDevice> rm;
22 |     assert(rm.RowNum() == 0);
23 |     assert(rm.ColNum() == 0);
24 | 
25 |     rm = Matrix<CheckElement, CheckDevice>(10, 20);
26 |     assert(rm.RowNum() == 10);
27 |     assert(rm.ColNum() == 20);
28 | 
29 |     int c = 0;
30 |     for (size_t i=0; i<10; ++i)
31 |     {
32 |         for (size_t j=0; j<20; ++j)
33 |         {
34 |             rm.SetValue(i, j, (float)(c++));
35 |         }
36 |     }
37 | 
38 |     const Matrix<CheckElement, CheckDevice> rm2 = rm;
39 |     c = 0;
40 |     for (size_t i=0; i<10; ++i)
41 |     {
42 |         for (size_t j=0; j<20; ++j)
43 |             assert(rm2(i, j) == c++);
44 |     }
45 | 
46 |     auto rm3 = rm.SubMatrix(3, 7, 5, 15);
47 |     for (size_t i=0; i<rm3.RowNum(); ++i)
48 |     {
49 |         for (size_t j = 0; j<rm3.ColNum(); ++j)
50 |         {
51 |             assert(rm3(i, j) == rm(i+3, j+5));
52 |         }
53 |     }
54 | 
55 |     auto evalHandle = rm.EvalRegister();
56 |     auto cm = evalHandle.Data();
57 | 
58 |     for (size_t i=0; i<cm.RowNum(); ++i)
59 |     {
60 |         for (size_t j = 0; j<cm.ColNum(); ++j)
61 |         {
62 |             assert(cm(i, j) == rm(i, j));
63 |         }
64 |     }
65 |     cout << "done" << endl;
66 | }
67 | 
68 | void TestMatrix2()
69 | {
70 |     cout << "Test general matrix case 2...\t";
71 |     auto rm1 = Matrix<CheckElement, CheckDevice>(10, 20);
72 |     int c = 0;
73 |     for (size_t i = 0; i < 10; ++i)
74 |     {
75 |         for (size_t j = 0; j < 20; ++j)
76 |         {
77 |             rm1.SetValue(i, j, (float)(c++));
78 |         }
79 |     }
80 | 
81 |     auto rm2 = Matrix<CheckElement, CheckDevice>(3, 7);
82 |     for (size_t i = 0; i < 3; ++i)
83 |     {
84 |         for (size_t j = 0; j < 7; ++j)
85 |         {
86 |             rm2.SetValue(i, j, (float)(c++));
87 |         }
88 |     }
89 |     cout << "done" << endl;
90 | }
91 | }
92 | 
93 | void test_general_matrix()
94 | {
95 |     TestMatrix1();
96 |     TestMatrix2();
97 | }


--------------------------------------------------------------------------------
/MetaNN/layers/facilities/interface_fun.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <type_traits>
 4 | namespace MetaNN
 5 | {
 6 | template <typename TLayer, typename TIn>
 7 | auto LayerFeedForward(TLayer& layer, TIn&& p_in)
 8 | {
 9 |     return layer.FeedForward(std::forward<TIn>(p_in));
10 | }
11 | 
12 | template <typename TLayer, typename TGrad>
13 | auto LayerFeedBackward(TLayer& layer, TGrad&& p_grad)
14 | {
15 |     return layer.FeedBackward(std::forward<TGrad>(p_grad));
16 | }
17 | 
18 | /// init interface ========================================
19 | namespace NSLayerInterface
20 | {
21 | template <typename L, typename TInitializer, typename TBuffer, typename TInitPolicies>
22 | std::true_type InitTest(decltype(&L::template Init<TInitializer, TBuffer, TInitPolicies>));
23 | 
24 | template <typename L, typename TInitPolicies, typename TInitContainer, typename TLoad>
25 | std::false_type InitTest(...);
26 | 
27 | template <typename L, typename TGradCollector>
28 | std::true_type GradCollectTest(decltype(&L::template GradCollect<TGradCollector>));
29 | 
30 | template <typename L, typename TGradCollector>
31 | std::false_type GradCollectTest(...);
32 | 
33 | template <typename L, typename TSave>
34 | std::true_type SaveWeightsTest(decltype(&L::template SaveWeights<TSave>));
35 | 
36 | template <typename L, typename TSave>
37 | std::false_type SaveWeightsTest(...);
38 | 
39 | template <typename L>
40 | std::true_type NeutralInvariantTest(decltype(&L::NeutralInvariant));
41 | 
42 | template <typename L>
43 | std::false_type NeutralInvariantTest(...);
44 | }
45 | 
46 | template <typename TLayer, typename TInitializer, typename TBuffer, 
47 |           typename TInitPolicies = typename TInitializer::PolicyCont>
48 | void LayerInit(TLayer& layer, TInitializer& initializer, TBuffer& loadBuffer, std::ostream* log = nullptr)
49 | {
50 |     if constexpr (decltype(NSLayerInterface::InitTest<TLayer, TInitializer, TBuffer, TInitPolicies>(nullptr))::value)
51 |         layer.template Init<TInitializer, TBuffer, TInitPolicies>(initializer, loadBuffer, log);
52 | }
53 |     
54 | template <typename TLayer, typename TGradCollector>
55 | void LayerGradCollect(TLayer& layer, TGradCollector& gc)
56 | {
57 |     if constexpr(decltype(NSLayerInterface::GradCollectTest<TLayer, TGradCollector>(nullptr))::value)
58 |         layer.GradCollect(gc);
59 | }
60 | 
61 | template <typename TLayer, typename TSave>
62 | void LayerSaveWeights(const TLayer& layer, TSave& saver)
63 | {
64 |     if constexpr (decltype(NSLayerInterface::SaveWeightsTest<TLayer, TSave>(nullptr))::value)
65 |         layer.SaveWeights(saver);
66 | }
67 | 
68 | template <typename TLayer>
69 | void LayerNeutralInvariant(TLayer& layer)
70 | {
71 |     if constexpr (decltype(NSLayerInterface::NeutralInvariantTest<TLayer>(nullptr))::value)
72 |         layer.NeutralInvariant();
73 | }
74 | }
75 | 


--------------------------------------------------------------------------------
/MetaNN/layers/elementary/tanh_layer.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | #include <MetaNN/layers/facilities/common_io.h>
 3 | #include <MetaNN/layers/facilities/policies.h>
 4 | #include <MetaNN/policies/policy_operations.h>
 5 | 
 6 | namespace MetaNN
 7 | {
 8 | template <typename TPolicies>
 9 | class TanhLayer
10 | {
11 |     static_assert(IsPolicyContainer<TPolicies>, "TPolicies is not a policy container.");
12 |     using CurLayerPolicy = PlainPolicy<TPolicies>;
13 | 
14 | public:
15 |     static constexpr bool IsFeedbackOutput = PolicySelect<FeedbackPolicy, CurLayerPolicy>::IsFeedbackOutput;
16 |     static constexpr bool IsUpdate = false;
17 |     using InputType = LayerIO;
18 |     using OutputType = LayerIO;
19 | 
20 | public:
21 |     template <typename TIn>
22 |     auto FeedForward(const TIn& p_in)
23 |     {
24 |         const auto& val = p_in.template Get<LayerIO>();
25 | 
26 |         using rawType = std::decay_t<decltype(val)>;
27 |         static_assert(!std::is_same<rawType, NullParameter>::value, "parameter is invalid");
28 | 
29 |         auto tmp = Tanh(val);
30 |         if constexpr (IsFeedbackOutput)
31 |         {
32 |             m_data.push(MakeDynamic(tmp));
33 |         }
34 |         return LayerIO::Create().template Set<LayerIO>(std::move(tmp));
35 |     }
36 | 
37 |     template <typename TGrad>
38 |     auto FeedBackward(const TGrad& p_grad)
39 |     {
40 |         if constexpr (IsFeedbackOutput)
41 |         {
42 |             if (m_data.empty())
43 |             {
44 |                 throw std::runtime_error("Cannot feed back in SigmoidLayer");
45 |             }
46 |             auto tmp = p_grad.template Get<LayerIO>();
47 |             auto& tmp2 = m_data.top();
48 |             auto res = LayerIO::Create().template Set<LayerIO>(TanhDerivative(tmp, tmp2));
49 |             m_data.pop();
50 |             return res;
51 |         }
52 |         else
53 |         {
54 |             return LayerIO::Create();
55 |         }
56 |     }
57 | 
58 |     void NeutralInvariant()
59 |     {
60 |         if constexpr(IsFeedbackOutput)
61 |         {
62 |             if (!m_data.empty())
63 |             {
64 |                 throw std::runtime_error("NeutralInvariant Fail!");
65 |             }
66 |         }
67 |     }
68 | 
69 | private:
70 |     using InternalType = LayerTraits::LayerInternalBuf<IsFeedbackOutput,
71 |                                                        PolicySelect<InputPolicy, CurLayerPolicy>::BatchMode,
72 |                                                        typename PolicySelect<OperandPolicy, CurLayerPolicy>::Element,
73 |                                                        typename PolicySelect<OperandPolicy, CurLayerPolicy>::Device,
74 |                                                        CategoryTags::Matrix, CategoryTags::BatchMatrix>;
75 |     InternalType m_data;
76 | };
77 | }
78 | 


--------------------------------------------------------------------------------
/MetaNN/layers/elementary/sigmoid_layer.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | #include <MetaNN/layers/facilities/common_io.h>
 3 | #include <MetaNN/layers/facilities/policies.h>
 4 | #include <MetaNN/policies/policy_operations.h>
 5 | 
 6 | namespace MetaNN
 7 | {
 8 | template <typename TPolicies>
 9 | class SigmoidLayer
10 | {
11 |     static_assert(IsPolicyContainer<TPolicies>, "TPolicies is not a policy container.");
12 |     using CurLayerPolicy = PlainPolicy<TPolicies>;
13 | 
14 | public:
15 |     static constexpr bool IsFeedbackOutput = PolicySelect<FeedbackPolicy, CurLayerPolicy>::IsFeedbackOutput;
16 |     static constexpr bool IsUpdate = false;
17 |     using InputType = LayerIO;
18 |     using OutputType = LayerIO;
19 | 
20 | public:
21 |     template <typename TIn>
22 |     auto FeedForward(const TIn& p_in)
23 |     {
24 |         const auto& val = p_in.template Get<LayerIO>();
25 | 
26 |         using rawType = std::decay_t<decltype(val)>;
27 |         static_assert(!std::is_same<rawType, NullParameter>::value, "parameter is invalid");
28 | 
29 |         auto tmp = Sigmoid(val);
30 |         
31 |         if constexpr (IsFeedbackOutput)
32 |         {
33 |             m_data.push(MakeDynamic(tmp));
34 |         }
35 |         return LayerIO::Create().template Set<LayerIO>(tmp);
36 |     }
37 | 
38 |     template <typename TGrad>
39 |     auto FeedBackward(const TGrad& p_grad)
40 |     {
41 |         if constexpr (IsFeedbackOutput)
42 |         {
43 |             if (m_data.empty())
44 |             {
45 |                 throw std::runtime_error("Cannot feed back in SigmoidLayer");
46 |             }
47 |             auto tmp = p_grad.template Get<LayerIO>();
48 |             auto& tmp2 = m_data.top();
49 |             auto res = LayerIO::Create().template Set<LayerIO>(SigmoidDerivative(tmp, tmp2));
50 |             m_data.pop();
51 |             return res;
52 |         }
53 |         else
54 |         {
55 |             return LayerIO::Create();
56 |         }
57 |     }
58 | 
59 |     void NeutralInvariant()
60 |     {
61 |         if constexpr(IsFeedbackOutput)
62 |         {
63 |             if (!m_data.empty())
64 |             {
65 |                 throw std::runtime_error("NeutralInvariant Fail!");
66 |             }
67 |         }
68 |     }
69 | 
70 | private:
71 |     using InternalType = LayerTraits::LayerInternalBuf<IsFeedbackOutput,
72 |                                                        PolicySelect<InputPolicy, CurLayerPolicy>::BatchMode,
73 |                                                        typename PolicySelect<OperandPolicy, CurLayerPolicy>::Element,
74 |                                                        typename PolicySelect<OperandPolicy, CurLayerPolicy>::Device,
75 |                                                        CategoryTags::Matrix, CategoryTags::BatchMatrix>;
76 |     InternalType m_data;
77 | };
78 | }
79 | 


--------------------------------------------------------------------------------
/MetaNN/MetaNN.mk:
--------------------------------------------------------------------------------
  1 | ##
  2 | ## Auto Generated makefile by CodeLite IDE
  3 | ## any manual changes will be erased      
  4 | ##
  5 | ## Release
  6 | ProjectName            :=MetaNN
  7 | ConfigurationName      :=Release
  8 | WorkspacePath          :=/home/liwei/MetaNN
  9 | ProjectPath            :=/home/liwei/MetaNN/MetaNN
 10 | IntermediateDirectory  :=./Release
 11 | OutDir                 := $(IntermediateDirectory)
 12 | CurrentFileName        :=
 13 | CurrentFilePath        :=
 14 | CurrentFileFullPath    :=
 15 | User                   :=liwei
 16 | Date                   :=28/06/18
 17 | CodeLitePath           :=/home/liwei/.codelite
 18 | LinkerName             :=g++
 19 | SharedObjectLinkerName :=g++ -shared -fPIC
 20 | ObjectSuffix           :=.o
 21 | DependSuffix           :=.o.d
 22 | PreprocessSuffix       :=.o.i
 23 | DebugSwitch            :=-gstab
 24 | IncludeSwitch          :=-I
 25 | LibrarySwitch          :=-l
 26 | OutputSwitch           :=-o 
 27 | LibraryPathSwitch      :=-L
 28 | PreprocessorSwitch     :=-D
 29 | SourceSwitch           :=-c 
 30 | OutputFile             :=$(IntermediateDirectory)/lib$(ProjectName).a
 31 | Preprocessors          :=
 32 | ObjectSwitch           :=-o 
 33 | ArchiveOutputSwitch    := 
 34 | PreprocessOnlySwitch   :=-E 
 35 | ObjectsFileList        :="MetaNN.txt"
 36 | PCHCompileFlags        :=
 37 | MakeDirCommand         :=mkdir -p
 38 | LinkOptions            :=  
 39 | IncludePath            :=  $(IncludeSwitch). $(IncludeSwitch). 
 40 | IncludePCH             := 
 41 | RcIncludePath          := 
 42 | Libs                   := 
 43 | ArLibs                 :=  
 44 | LibPath                := $(LibraryPathSwitch). 
 45 | 
 46 | ##
 47 | ## Common variables
 48 | ## AR, CXX, CC, AS, CXXFLAGS and CFLAGS can be overriden using an environment variables
 49 | ##
 50 | AR       := ar rcus
 51 | CXX      := g++
 52 | CC       := gcc
 53 | CXXFLAGS :=   $(Preprocessors)
 54 | CFLAGS   :=   $(Preprocessors)
 55 | ASFLAGS  := 
 56 | AS       := as
 57 | 
 58 | 
 59 | ##
 60 | ## User defined environment variables
 61 | ##
 62 | CodeLiteDir:=/usr/share/codelite
 63 | 
 64 | 
 65 | Objects=
 66 | 
 67 | ##
 68 | ## Main Build Targets 
 69 | ##
 70 | .PHONY: all clean PreBuild PrePreBuild PostBuild MakeIntermediateDirs
 71 | all: $(IntermediateDirectory) $(OutputFile)
 72 | 
 73 | $(OutputFile): $(Objects)
 74 | 	@$(MakeDirCommand) $(@D)
 75 | 	@echo "" > $(IntermediateDirectory)/.d
 76 | 	$(AR) $(ArchiveOutputSwitch)$(OutputFile) @$(ObjectsFileList) $(ArLibs)
 77 | 	@$(MakeDirCommand) "/home/liwei/MetaNN/.build-release"
 78 | 	@echo rebuilt > "/home/liwei/MetaNN/.build-release/MetaNN"
 79 | 
 80 | MakeIntermediateDirs:
 81 | 	@test -d ./Release || $(MakeDirCommand) ./Release
 82 | 
 83 | 
 84 | ./Release:
 85 | 	@test -d ./Release || $(MakeDirCommand) ./Release
 86 | 
 87 | PreBuild:
 88 | 
 89 | 
 90 | ##
 91 | ## Objects
 92 | ##
 93 | 
 94 | -include $(IntermediateDirectory)/*$(DependSuffix)
 95 | ##
 96 | ## Clean
 97 | ##
 98 | clean:
 99 | 	$(RM) -r ./Release/
100 | 
101 | 
102 | 


--------------------------------------------------------------------------------
/Test/operators/test_softmax.cpp:
--------------------------------------------------------------------------------
  1 | #include "test_softmax.h"
  2 | 
  3 | #include "../facilities/data_gen.h"
  4 | #include <MetaNN/meta_nn.h>
  5 | #include <cassert>
  6 | #include <iostream>
  7 | using namespace MetaNN;
  8 | using namespace std;
  9 | 
 10 | namespace
 11 | {
 12 | void test_softmax1()
 13 | {
 14 |     cout << "Test softmax case 1 ...\t";
 15 |     auto rm1 = GenMatrix<float>(1, 20, 0, 0.001f);
 16 |     auto t = VecSoftmax(rm1);
 17 |     auto t_r = Evaluate(t);
 18 | 
 19 |     float sum = 0;
 20 |     for (size_t i = 0; i < 20; ++i)
 21 |     {
 22 |         sum += exp(rm1(0, i));
 23 |     }
 24 | 
 25 |     for (size_t i = 0; i < 20; ++i)
 26 |     {
 27 |         assert(fabs(t_r(0, i) - exp(rm1(0, i)) / sum) < 0.0001);
 28 |     }
 29 | 
 30 |     rm1 = GenMatrix<float>(111, 113, 2, 0.001f);
 31 |     rm1 = rm1.SubMatrix(17, 18, 31, 51);
 32 |     t = VecSoftmax(rm1);
 33 |     t_r = Evaluate(t);
 34 | 
 35 |     sum = 0;
 36 |     for (size_t i = 0; i < 20; ++i)
 37 |     {
 38 |         sum += exp(rm1(0, i));
 39 |     }
 40 | 
 41 |     for (size_t i = 0; i < 20; ++i)
 42 |     {
 43 |         assert(fabs(t_r(0, i) - exp(rm1(0, i)) / sum) < 0.0001);
 44 |     }
 45 |     cout << "done" << endl;
 46 | }
 47 | 
 48 | void test_softmax2()
 49 | {
 50 |     cout << "Test softmax case 2 ...\t";
 51 |     {
 52 |         auto rm1 = GenMatrix<float>(1, 20, 0, 0.001f);
 53 |         auto res = VecSoftmax(rm1);
 54 |         auto res2 = VecSoftmax(rm1);
 55 | 
 56 |         assert(res == res2);
 57 | 
 58 |         auto cm1 = Evaluate(res);
 59 |         auto cm2 = Evaluate(res);
 60 |         assert(cm1 == cm2);
 61 |     }
 62 |     {
 63 |         auto rm1 = GenMatrix<float>(1, 20, 0, 0.001f);
 64 |         auto res = VecSoftmax(rm1);
 65 |         auto res2 = res;
 66 | 
 67 |         assert(res == res2);
 68 | 
 69 |         const auto& evalHandle1 = res.EvalRegister();
 70 |         const auto& evalHandle2 = res2.EvalRegister();
 71 |         EvalPlan<DeviceTags::CPU>::Eval();
 72 | 
 73 |         auto cm1 = evalHandle1.Data();
 74 |         auto cm2 = evalHandle2.Data();
 75 |         assert(cm1 == cm2);
 76 |     }
 77 |     cout << "done" << endl;
 78 | }
 79 | 
 80 | void test_softmax3()
 81 | {
 82 |     cout << "Test softmax case 3 ...\t";
 83 |     auto rm1 = GenBatchMatrix<float>(1, 20, 7, 0, 0.001f);
 84 |     auto t = VecSoftmax(rm1);
 85 |     auto t_r = Evaluate(t);
 86 | 
 87 |     for (size_t b = 0; b < 7; ++b)
 88 |     {
 89 |         float sum = 0;
 90 |         for (size_t i = 0; i < 20; ++i)
 91 |         {
 92 |             sum += exp(rm1[b](0, i));
 93 |         }
 94 | 
 95 |         for (size_t i = 0; i < 20; ++i)
 96 |         {
 97 |             assert(fabs(t_r[b](0, i) - exp(rm1[b](0, i)) / sum) < 0.0001);
 98 |         }
 99 |     }
100 |     cout << "done" << endl;
101 | }
102 | }
103 | 
104 | void test_softmax()
105 | {
106 |     test_softmax1();
107 |     test_softmax2();
108 |     test_softmax3();
109 | }
110 | 


--------------------------------------------------------------------------------
/Test/operators/test_abs.cpp:
--------------------------------------------------------------------------------
  1 | #include "test_abs.h"
  2 | #include "../facilities/data_gen.h"
  3 | #include <MetaNN/meta_nn.h>
  4 | #include <cassert>
  5 | #include <iostream>
  6 | using namespace MetaNN;
  7 | using namespace std;
  8 | 
  9 | namespace
 10 | {
 11 | void test_abs1()
 12 | {
 13 |     cout << "Test abs case 1 ...\t";
 14 |     auto rm1 = GenMatrix<float>(4, 5, -3.3f, 0.1f);
 15 |     auto t = Abs(rm1);
 16 |     
 17 |     auto handle = t.EvalRegister();
 18 |     EvalPlan<DeviceTags::CPU>::Eval();
 19 |     auto t_r = handle.Data();
 20 | 
 21 |     for (size_t i = 0; i < 4; ++i)
 22 |     {
 23 |         for (size_t j = 0; j<5; ++j)
 24 |         {
 25 |             float aim = fabs(rm1(i, j));
 26 |             assert(fabs(t_r(i, j) - aim) < 0.0001);
 27 |         }
 28 |     }
 29 | 
 30 |     rm1 = GenMatrix<float>(111, 113, -22, 0.0001f);
 31 |     rm1 = rm1.SubMatrix(31, 35, 17, 22);
 32 |     t = Abs(rm1);
 33 |     t_r = Evaluate(t);
 34 | 
 35 |     for (size_t i = 0; i < 4; ++i)
 36 |     {
 37 |         for (size_t j = 0; j<5; ++j)
 38 |         {
 39 |             float aim = fabs(rm1(i, j));
 40 |             assert(fabs(t_r(i, j) - aim) < 0.0001);
 41 |         }
 42 |     }
 43 |     cout << "done" << endl;
 44 | }
 45 | 
 46 | void test_abs2()
 47 | {
 48 |     cout << "Test abs case 2 ...\t";
 49 |     {
 50 |         auto rm1 = GenMatrix<float>(4, 5, 0, 0.0001f);
 51 |         auto res = Abs(rm1);
 52 |         auto res2 = Abs(rm1);
 53 | 
 54 |         assert(res == res2);
 55 | 
 56 |         auto cm1 = Evaluate(res);
 57 |         auto cm2 = Evaluate(res);
 58 |         assert(cm1 == cm2);
 59 |     }
 60 |     {
 61 |         auto rm1 = GenMatrix<float>(4, 5, 0, 0.0001f);
 62 |         auto res = Abs(rm1);
 63 |         auto res2 = res;
 64 | 
 65 |         assert(res == res2);
 66 | 
 67 |         const auto& evalHandle1 = res.EvalRegister();
 68 |         const auto& evalHandle2 = res2.EvalRegister();
 69 |         EvalPlan<DeviceTags::CPU>::Eval();
 70 | 
 71 |         auto cm1 = evalHandle1.Data();
 72 |         auto cm2 = evalHandle2.Data();
 73 |         assert(cm1 == cm2);
 74 |     }
 75 |     cout << "done" << endl;
 76 | }
 77 | 
 78 | void test_abs3()
 79 | {
 80 |     cout << "Test abs case 3 ...\t";
 81 |     auto rm1 = GenBatchMatrix<float>(4, 5, 7, -3.3f, 0.1f);
 82 |     auto t = Abs(rm1);
 83 |     
 84 |     auto handle = t.EvalRegister();
 85 |     EvalPlan<DeviceTags::CPU>::Eval();
 86 |     auto t_r = handle.Data();
 87 | 
 88 |     for (size_t b = 0; b < 7; ++b)
 89 |     {
 90 |         for (size_t i = 0; i < 4; ++i)
 91 |         {
 92 |             for (size_t j = 0; j<5; ++j)
 93 |             {
 94 |                 float aim = fabs(rm1[b](i, j));
 95 |                 assert(fabs(t_r[b](i, j) - aim) < 0.0001);
 96 |             }
 97 |         }
 98 |     }
 99 |     cout << "done" << endl;
100 | }
101 | }
102 | 
103 | void test_abs()
104 | {
105 |     test_abs1();
106 |     test_abs2();
107 |     test_abs3();
108 | }
109 | 


--------------------------------------------------------------------------------
/MetaNN/model_rel/param_initializer/param_initializer.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | #include <map>
 3 | #include <string>
 4 | #include <stdexcept>
 5 | 
 6 | namespace MetaNN
 7 | {
 8 | template <typename TElem, typename TPolicyCont, typename TFillers>
 9 | class ParamInitializer
10 | {
11 | public:
12 |     using PolicyCont = TPolicyCont;
13 |     
14 |     ParamInitializer(TFillers&& filler)
15 |         : m_filler(std::move(filler))
16 |     {}
17 |     
18 |     template <typename TTag, typename TVal>
19 |     auto SetFiller(TVal&& val) &&
20 |     {
21 |         auto newFiller = std::move(m_filler).template Set<TTag, TVal>(std::forward<TVal>(val));
22 |         using newFillerType = RemConstRef<decltype(newFiller)>;
23 |         return ParamInitializer<TElem, TPolicyCont, newFillerType>(std::move(newFiller));
24 |     }
25 |     
26 |     template <typename TTag>
27 |     auto& GetFiller()
28 |     {
29 |         return m_filler.template Get<TTag>();
30 |     }
31 |     
32 |     template <typename TElem2, typename TDevice2>
33 |     void SetMatrix(const std::string& name, const Matrix<TElem2, TDevice2>& param)
34 |     {
35 |         if (m_params.find(name) != m_params.end())
36 |         {
37 |             throw std::runtime_error("Duplicate parameter matrix: " + name);
38 |         }
39 |         
40 |         if constexpr (std::is_same<TElem2, TElem>::value &&
41 |                       std::is_same<TDevice2, DeviceTags::CPU>::value)
42 |         {
43 |             m_params.insert({name, param});
44 |         }
45 |         else
46 |         {
47 |             Matrix<TElem, DeviceTags::CPU> mat(param.RowNum(), param.ColNum());
48 |             DataCopy(param, mat);
49 |             m_params.insert({name, std::move(mat)});
50 |         }
51 |     }
52 |     
53 |     template <typename TElem2, typename TDevice2>
54 |     void GetMatrix(const std::string& name, Matrix<TElem2, TDevice2>& res) const
55 |     {
56 |         auto it = m_params.find(name);
57 |         if (it == m_params.end())
58 |         {
59 |             throw std::runtime_error("Parameter not exist: " + name);
60 |         }
61 |         const auto& oriMat = it->second;
62 |         if ((oriMat.RowNum() != res.RowNum()) || (oriMat.ColNum() != res.ColNum()))
63 |         {
64 |             throw std::runtime_error("Matrices with different dimensions.");
65 |         }
66 |         
67 |         DataCopy(oriMat, res);
68 |     }
69 |     
70 |     bool IsMatrixExist(const std::string& name) const
71 |     {
72 |         auto it = m_params.find(name);
73 |         return it != m_params.end();
74 |     }
75 |     
76 | private:
77 |     TFillers m_filler;
78 |     std::map<std::string, Matrix<TElem, DeviceTags::CPU>> m_params;
79 | };
80 | 
81 | template <typename TElem, typename...TPolicies>
82 | auto MakeInitializer()
83 | {
84 |     using npType = FillerTags2NamedParams<TPolicies...>;
85 |     using FilDictType = RemConstRef<decltype(npType::Create())>;
86 |     return ParamInitializer<TElem, PolicyContainer<TPolicies...>, FilDictType>(npType::Create());
87 | }
88 | }


--------------------------------------------------------------------------------
/Test/operators/test_interpolate.cpp:
--------------------------------------------------------------------------------
 1 | #include "test_interpolate.h"
 2 | #include "../facilities/data_gen.h"
 3 | #include <MetaNN/meta_nn.h>
 4 | #include <cassert>
 5 | #include <cmath>
 6 | #include <iostream>
 7 | using namespace MetaNN;
 8 | using namespace std;
 9 | 
10 | namespace
11 | {
12 | void test_interpolate1()
13 | {
14 |     cout << "Test interpolate case 1 ...\t";
15 |     auto rm1 = GenMatrix<float>(4, 5, 0, 0.0001f);
16 |     auto rm2 = GenMatrix<float>(4, 5, 1, 0.0002f);
17 |     auto rm3 = GenMatrix<float>(4, 5, 2, 0.0007f);
18 |     auto res = Interpolate(rm1, rm2, rm3);
19 |     auto res_r = Evaluate(res);
20 |     for (size_t i = 0; i < 4; ++i)
21 |     {
22 |         for (size_t j = 0; j<5; ++j)
23 |         {
24 |             float h = rm1(i, j) * rm3(i, j) + rm2(i, j) * (1 - rm3(i, j));
25 |             assert(fabs(res_r(i, j) - h) < 0.0001);
26 |         }
27 |     }
28 | 
29 |     rm1 = GenMatrix<float>(111, 113, 0, 0.0001f);
30 |     rm2 = GenMatrix<float>(111, 113, 1, 0.0002f);
31 |     rm3 = GenMatrix<float>(111, 113, 3, 0.0007f);
32 | 
33 |     rm1 = rm1.SubMatrix(31, 35, 17, 22);
34 |     rm2 = rm2.SubMatrix(41, 45, 27, 32);
35 |     rm3 = rm3.SubMatrix(51, 55, 37, 42);
36 | 
37 |     res = Interpolate(rm1, rm2, rm3);
38 |     res_r = Evaluate(res);
39 | 
40 |     auto rm4 = GenMatrix<float>(111, 113, 5, 0.001f);
41 |     for (size_t i = 0; i < 4; ++i)
42 |     {
43 |         for (size_t j = 0; j<5; ++j)
44 |         {
45 |             float h = rm1(i, j) * rm3(i, j) + rm2(i, j) * (1 - rm3(i, j));
46 |             assert(fabs(res_r(i, j) - h) < 0.0001);
47 |         }
48 |     }
49 |     cout << "done" << endl;
50 | }
51 | 
52 | void test_interpolate2()
53 | {
54 |     cout << "Test interpolate case 2 ...\t";
55 |     {
56 |         auto rm1 = GenMatrix<float>(4, 5, 0, 0.0001f);
57 |         auto rm2 = GenMatrix<float>(4, 5, 1, 0.002f);
58 |         auto rm3 = GenMatrix<float>(4, 5, 0, 0.0001f);
59 |         auto res = Interpolate(rm1, rm2, rm3);
60 |         auto res2 = Interpolate(rm1, rm2, rm3);
61 | 
62 |         assert(res == res2);
63 | 
64 |         auto handle1 = res.EvalRegister();
65 |         auto handle2 = res.EvalRegister();
66 |         EvalPlan<DeviceTags::CPU>::Eval();
67 | 
68 |         auto cm1 = handle1.Data();
69 |         auto cm2 = handle2.Data();
70 |         assert(cm1 == cm2);
71 |     }
72 |     {
73 |         auto rm1 = GenMatrix<float>(4, 5, 0, 0.0001f);
74 |         auto rm2 = GenMatrix<float>(4, 5, 1, 0.002f);
75 |         auto rm3 = GenMatrix<float>(4, 5, 0, 0.0001f);
76 |         auto res = Interpolate(rm1, rm2, rm3);
77 |         auto res2 = res;
78 | 
79 |         assert(res == res2);
80 | 
81 |         auto handle1 = res.EvalRegister();
82 |         auto handle2 = res2.EvalRegister();
83 |         EvalPlan<DeviceTags::CPU>::Eval();
84 | 
85 |         auto cm1 = handle1.Data();
86 |         auto cm2 = handle2.Data();
87 |         assert(cm1 == cm2);
88 |     }
89 |     cout << "done" << endl;
90 | }
91 | }
92 | 
93 | void test_interpolate()
94 | {
95 |     test_interpolate1();
96 |     test_interpolate2();
97 | }
98 | 


--------------------------------------------------------------------------------
/Test/operators/test_sigmoid.cpp:
--------------------------------------------------------------------------------
  1 | #include "test_sigmoid.h"
  2 | #include "../facilities/data_gen.h"
  3 | #include <MetaNN/meta_nn.h>
  4 | #include <cassert>
  5 | #include <iostream>
  6 | using namespace MetaNN;
  7 | using namespace std;
  8 | 
  9 | namespace
 10 | {
 11 | void test_sigmoid1()
 12 | {
 13 |     cout << "Test sigmoid case 1 ...\t";
 14 |     auto rm1 = GenMatrix<float>(4, 5, 0, 0.0001f);
 15 |     auto t = Sigmoid(rm1);
 16 |     
 17 |     auto handle = t.EvalRegister();
 18 |     EvalPlan<DeviceTags::CPU>::Eval();
 19 |     auto t_r = handle.Data();
 20 | 
 21 |     for (size_t i = 0; i < 4; ++i)
 22 |     {
 23 |         for (size_t j = 0; j<5; ++j)
 24 |         {
 25 |             float aim = 1/(1+exp(-rm1(i, j)));
 26 |             assert(fabs(t_r(i, j) - aim) < 0.0001);
 27 |         }
 28 |     }
 29 | 
 30 |     rm1 = GenMatrix<float>(111, 113, 2, 0.0001f);
 31 |     rm1 = rm1.SubMatrix(31, 35, 17, 22);
 32 |     t = Sigmoid(rm1);
 33 |     t_r = Evaluate(t);
 34 | 
 35 |     for (size_t i = 0; i < 4; ++i)
 36 |     {
 37 |         for (size_t j = 0; j<5; ++j)
 38 |         {
 39 |             float aim = 1/(1+exp(-rm1(i, j)));
 40 |             assert(fabs(t_r(i, j) - aim) < 0.0001);
 41 |         }
 42 |     }
 43 |     cout << "done" << endl;
 44 | }
 45 | 
 46 | void test_sigmoid2()
 47 | {
 48 |     cout << "Test sigmoid case 2 ...\t";
 49 |     {
 50 |         auto rm1 = GenMatrix<float>(4, 5, 0, 0.0001f);
 51 |         auto res = Sigmoid(rm1);
 52 |         auto res2 = Sigmoid(rm1);
 53 | 
 54 |         assert(res == res2);
 55 | 
 56 |         auto cm1 = Evaluate(res);
 57 |         auto cm2 = Evaluate(res);
 58 |         assert(cm1 == cm2);
 59 |     }
 60 |     {
 61 |         auto rm1 = GenMatrix<float>(4, 5, 0, 0.0001f);
 62 |         auto res = Sigmoid(rm1);
 63 |         auto res2 = res;
 64 | 
 65 |         assert(res == res2);
 66 | 
 67 |         const auto& evalHandle1 = res.EvalRegister();
 68 |         const auto& evalHandle2 = res2.EvalRegister();
 69 |         EvalPlan<DeviceTags::CPU>::Eval();
 70 | 
 71 |         auto cm1 = evalHandle1.Data();
 72 |         auto cm2 = evalHandle2.Data();
 73 |         assert(cm1 == cm2);
 74 |     }
 75 |     cout << "done" << endl;
 76 | }
 77 | 
 78 | void test_sigmoid3()
 79 | {
 80 |     cout << "Test sigmoid case 3 ...\t";
 81 |     auto rm1 = GenBatchMatrix<float>(4, 5, 7, 0, 0.0001f);
 82 |     auto t = Sigmoid(rm1);
 83 |     
 84 |     auto handle = t.EvalRegister();
 85 |     EvalPlan<DeviceTags::CPU>::Eval();
 86 |     auto t_r = handle.Data();
 87 | 
 88 |     for (size_t b = 0; b < 7; ++b)
 89 |     {
 90 |         for (size_t i = 0; i < 4; ++i)
 91 |         {
 92 |             for (size_t j = 0; j<5; ++j)
 93 |             {
 94 |                 float aim = 1/(1+exp(-rm1[b](i, j)));
 95 |                 assert(fabs(t_r[b](i, j) - aim) < 0.0001);
 96 |             }
 97 |         }
 98 |     }
 99 |     cout << "done" << endl;
100 | }
101 | }
102 | 
103 | void test_sigmoid()
104 | {
105 |     test_sigmoid1();
106 |     test_sigmoid2();
107 |     test_sigmoid3();
108 | }
109 | 


--------------------------------------------------------------------------------
/Test/operators/test_sign.cpp:
--------------------------------------------------------------------------------
  1 | #include "test_sign.h"
  2 | #include "../facilities/data_gen.h"
  3 | #include <MetaNN/meta_nn.h>
  4 | #include <cassert>
  5 | #include <iostream>
  6 | using namespace MetaNN;
  7 | using namespace std;
  8 | 
  9 | namespace
 10 | {
 11 | void test_sign1()
 12 | {
 13 |     cout << "Test sign case 1 ...\t";
 14 |     auto rm1 = GenMatrix<float>(4, 5, -3.3f, 0.1f);
 15 |     auto t = Sign(rm1);
 16 |     
 17 |     auto handle = t.EvalRegister();
 18 |     EvalPlan<DeviceTags::CPU>::Eval();
 19 |     auto t_r = handle.Data();
 20 | 
 21 |     for (size_t i = 0; i < 4; ++i)
 22 |     {
 23 |         for (size_t j = 0; j<5; ++j)
 24 |         {
 25 |             float aim = 0;
 26 |             if (rm1(i, j) != 0)
 27 |                 aim = (rm1(i, j) > 0) ? 1 : -1;
 28 |             assert(fabs(t_r(i, j) - aim) < 0.0001);
 29 |         }
 30 |     }
 31 | 
 32 |     rm1 = GenMatrix<float>(111, 113, -22, 0.0001f);
 33 |     rm1 = rm1.SubMatrix(31, 35, 17, 22);
 34 |     t = Sign(rm1);
 35 |     t_r = Evaluate(t);
 36 | 
 37 |     for (size_t i = 0; i < 4; ++i)
 38 |     {
 39 |         for (size_t j = 0; j<5; ++j)
 40 |         {
 41 |             float aim = 0;
 42 |             if (rm1(i, j) != 0)
 43 |                 aim = (rm1(i, j) > 0) ? 1 : -1;
 44 |             assert(fabs(t_r(i, j) - aim) < 0.0001);
 45 |         }
 46 |     }
 47 |     cout << "done" << endl;
 48 | }
 49 | 
 50 | void test_sign2()
 51 | {
 52 |     cout << "Test sign case 2 ...\t";
 53 |     {
 54 |         auto rm1 = GenMatrix<float>(4, 5, 0, 0.0001f);
 55 |         auto res = Sign(rm1);
 56 |         auto res2 = Sign(rm1);
 57 | 
 58 |         assert(res == res2);
 59 | 
 60 |         auto cm1 = Evaluate(res);
 61 |         auto cm2 = Evaluate(res);
 62 |         assert(cm1 == cm2);
 63 |     }
 64 |     {
 65 |         auto rm1 = GenMatrix<float>(4, 5, 0, 0.0001f);
 66 |         auto res = Sign(rm1);
 67 |         auto res2 = res;
 68 | 
 69 |         assert(res == res2);
 70 | 
 71 |         const auto& evalHandle1 = res.EvalRegister();
 72 |         const auto& evalHandle2 = res2.EvalRegister();
 73 |         EvalPlan<DeviceTags::CPU>::Eval();
 74 | 
 75 |         auto cm1 = evalHandle1.Data();
 76 |         auto cm2 = evalHandle2.Data();
 77 |         assert(cm1 == cm2);
 78 |     }
 79 |     cout << "done" << endl;
 80 | }
 81 | 
 82 | void test_sign3()
 83 | {
 84 |     cout << "Test sign case 3 ...\t";
 85 |     auto rm1 = GenBatchMatrix<float>(4, 5, 7, -3.3f, 0.1f);
 86 |     auto t = Sign(rm1);
 87 |     
 88 |     auto handle = t.EvalRegister();
 89 |     EvalPlan<DeviceTags::CPU>::Eval();
 90 |     auto t_r = handle.Data();
 91 | 
 92 |     for (size_t b = 0; b < 7; ++b)
 93 |     {
 94 |         for (size_t i = 0; i < 4; ++i)
 95 |         {
 96 |             for (size_t j = 0; j<5; ++j)
 97 |             {
 98 |                 float aim = 0;
 99 |                 if (rm1[b](i, j) != 0)
100 |                     aim = (rm1[b](i, j) > 0) ? 1 : -1;
101 |                 assert(fabs(t_r[b](i, j) - aim) < 0.0001);
102 |             }
103 |         }
104 |     }
105 |     cout << "done" << endl;
106 | }
107 | }
108 | 
109 | void test_sign()
110 | {
111 |     test_sign1();
112 |     test_sign2();
113 |     test_sign3();
114 | }
115 | 


--------------------------------------------------------------------------------
/Test/policies/test_policy_selector.cpp:
--------------------------------------------------------------------------------
  1 | #include "test_policy_selector.h"
  2 | #include <iostream>
  3 | #include <cassert>
  4 | #include <cmath>
  5 | #include <set>
  6 | #include <MetaNN/meta_nn.h>
  7 | using namespace std;
  8 | using namespace MetaNN;
  9 | 
 10 | namespace
 11 | {
 12 | struct AccPolicy
 13 | {
 14 |     using MajorClass = AccPolicy;
 15 |     
 16 |     struct AccuTypeCate
 17 |     {
 18 |         struct Add;
 19 |         struct Mul;
 20 |     };
 21 |     using Accu = AccuTypeCate::Add;
 22 | 
 23 |     struct IsAveValueCate;
 24 |     static constexpr bool IsAve = false;
 25 | 
 26 |     struct ValueTypeCate;
 27 |     using Value = float;
 28 | };
 29 | 
 30 | #include <MetaNN/policies/policy_macro_begin.h>
 31 | TypePolicyObj (PAddAccu,     AccPolicy, Accu,  Add);
 32 | TypePolicyObj (PMulAccu,     AccPolicy, Accu,  Mul);
 33 | ValuePolicyObj(PAve,         AccPolicy, IsAve, true);
 34 | ValuePolicyObj(PNoAve,       AccPolicy, IsAve, false);
 35 | TypePolicyTemplate (PValueTypeIs,  AccPolicy, Value);
 36 | ValuePolicyTemplate(PAvePolicyIs, AccPolicy, IsAve);
 37 | #include <MetaNN/policies/policy_macro_end.h>
 38 | 
 39 | template <typename...TPolicies>
 40 | struct Accumulator
 41 | {
 42 |     using TPoliCont = PolicyContainer<TPolicies...>;
 43 |     using TPolicyRes = PolicySelect<AccPolicy, TPoliCont>;
 44 | 
 45 |     using ValueType = typename TPolicyRes::Value;
 46 |     static constexpr bool is_ave = TPolicyRes::IsAve;
 47 |     using AccuType = typename TPolicyRes::Accu;
 48 | 
 49 | public:
 50 |     template <typename TIn>
 51 |     static auto Eval(const TIn& in)
 52 |     {
 53 |         if constexpr(std::is_same<AccuType, AccPolicy::AccuTypeCate::Add>::value)
 54 |         {
 55 |             ValueType count = 0;
 56 |             ValueType res = 0;
 57 |             for (const auto& x : in)
 58 |             {
 59 |                 res += x;
 60 |                 count += 1;
 61 |             }
 62 | 
 63 |             if constexpr (is_ave)
 64 |                 return res / count;
 65 |             else
 66 |                 return res;
 67 |         }
 68 |         else if constexpr (std::is_same<AccuType, AccPolicy::AccuTypeCate::Mul>::value)
 69 |         {
 70 |             ValueType res = 1;
 71 |             ValueType count = 0;
 72 |             for (const auto& x : in)
 73 |             {
 74 |                 res *= x;
 75 |                 count += 1;
 76 |             }
 77 |             if constexpr (is_ave)
 78 |                 return pow(res, 1.0 / count);
 79 |             else
 80 |                 return res;
 81 |         }
 82 |         else
 83 |         {
 84 |             static_assert(DependencyFalse<AccuType>);
 85 |         }
 86 |     }
 87 | };
 88 | }
 89 | 
 90 | void test_policy_selector()
 91 | {
 92 |     cout << "Test policy selector...\t";
 93 | 
 94 |     const int a[] = {1, 2, 3, 4, 5};
 95 |     assert(fabs(Accumulator<>::Eval(a) - 15) < 0.0001);
 96 |     assert(fabs(Accumulator<PMulAccu>::Eval(a) - 120) < 0.0001);
 97 |     assert(fabs(Accumulator<PMulAccu, PAve>::Eval(a) - pow(120.0, 0.2)) < 0.0001);
 98 |     assert(fabs(Accumulator<PAve, PMulAccu>::Eval(a) - pow(120.0, 0.2)) < 0.0001);
 99 |     assert(fabs(Accumulator<PAvePolicyIs<true>, PMulAccu>::Eval(a) - pow(120.0, 0.2)) < 0.0001);
100 |     cout << "done" << endl;
101 | }


--------------------------------------------------------------------------------
/Test/operators/test_tanh_derivative.cpp:
--------------------------------------------------------------------------------
  1 | #include "test_tanh_derivative.h"
  2 | #include "../facilities/data_gen.h"
  3 | #include <MetaNN/meta_nn.h>
  4 | #include <cassert>
  5 | #include <iostream>
  6 | using namespace MetaNN;
  7 | using namespace std;
  8 | 
  9 | namespace
 10 | {
 11 | void test_tanh_derivative1()
 12 | {
 13 |     cout << "Test tanh derivative case 1 ...\t";
 14 |     auto rm1 = GenMatrix<float>(4, 5, 0, 0.0001f);
 15 |     auto rm2 = GenMatrix<float>(4, 5, 1, 0.0003f);
 16 |     auto t = TanhDerivative(rm1, rm2);
 17 |     auto t_r = Evaluate(t);
 18 |     for (size_t i = 0; i < 4; ++i)
 19 |     {
 20 |         for (size_t j = 0; j<5; ++j)
 21 |         {
 22 |             float aim = rm1(i, j) * (1 - rm2(i, j) * rm2(i, j));
 23 |             assert(fabs(t_r(i, j) - aim) < 0.0001);
 24 |         }
 25 |     }
 26 | 
 27 |     rm1 = GenMatrix<float>(111, 113, 1, 0.001f);
 28 |     rm2 = GenMatrix<float>(111, 113, 0, 0.003f);
 29 |     rm1 = rm1.SubMatrix(31, 35, 17, 22);
 30 |     rm2 = rm2.SubMatrix(30, 34, 18, 23);
 31 |     t = TanhDerivative(rm1, rm2);
 32 |     t_r = Evaluate(t);
 33 |     for (size_t i = 0; i < 4; ++i)
 34 |     {
 35 |         for (size_t j = 0; j<5; ++j)
 36 |         {
 37 |             float aim = rm1(i, j) * (1 - rm2(i, j) * rm2(i, j));
 38 |             assert(fabs(t_r(i, j) - aim) < 0.0001);
 39 |         }
 40 |     }
 41 |     cout << "done" << endl;
 42 | }
 43 | 
 44 | void test_tanh_derivative2()
 45 | {
 46 |     cout << "Test tanh derivative case 2 ...\t";
 47 |     {
 48 |         auto rm1 = GenMatrix<float>(4, 5, 0, 0.0001f);
 49 |         auto rm2 = GenMatrix<float>(4, 5, 1, 0.0003f);
 50 |         auto res = TanhDerivative(rm1, rm2);
 51 |         auto res2 = TanhDerivative(rm1, rm2);
 52 | 
 53 |         assert(res == res2);
 54 | 
 55 |         auto cm1 = Evaluate(res);
 56 |         auto cm2 = Evaluate(res);
 57 |         assert(cm1 == cm2);
 58 |     }
 59 |     {
 60 |         auto rm1 = GenMatrix<float>(4, 5, 0, 0.0001f);
 61 |         auto rm2 = GenMatrix<float>(4, 5, 1, 0.0003f);
 62 |         auto res = TanhDerivative(rm1, rm2);
 63 |         auto res2 = res;
 64 | 
 65 |         assert(res == res2);
 66 | 
 67 |         auto handle1 = res.EvalRegister();
 68 |         auto handle2 = res2.EvalRegister();
 69 |         EvalPlan<DeviceTags::CPU>::Eval();
 70 | 
 71 |         auto cm1 = handle1.Data();
 72 |         auto cm2 = handle2.Data();
 73 |         assert(cm1 == cm2);
 74 |     }
 75 |     cout << "done" << endl;
 76 | }
 77 | 
 78 | void test_tanh_derivative3()
 79 | {
 80 |     cout << "Test tanh derivative case 3 ...\t";
 81 |     auto rm1 = GenBatchMatrix<float>(4, 5, 7, 0, 0.0001f);
 82 |     auto rm2 = GenBatchMatrix<float>(4, 5, 7, 1, 0.0003f);
 83 |     auto t = TanhDerivative(rm1, rm2);
 84 |     auto t_r = Evaluate(t);
 85 |     for (size_t b = 0; b < 7; ++b)
 86 |     {
 87 |         for (size_t i = 0; i < 4; ++i)
 88 |         {
 89 |             for (size_t j = 0; j<5; ++j)
 90 |             {
 91 |                 float aim = rm1[b](i, j) * (1 - rm2[b](i, j) * rm2[b](i, j));
 92 |                 assert(fabs(t_r[b](i, j) - aim) < 0.0001);
 93 |             }
 94 |         }
 95 |     }
 96 |     cout << "done" << endl;
 97 | }
 98 | }
 99 | 
100 | void test_tanh_derivative()
101 | {
102 |     test_tanh_derivative1();
103 |     test_tanh_derivative2();
104 |     test_tanh_derivative3();
105 | }
106 | 
107 | 


--------------------------------------------------------------------------------
/Test/operators/test_dot.cpp:
--------------------------------------------------------------------------------
  1 | #include "test_dot.h"
  2 | #include "../facilities/data_gen.h"
  3 | #include <MetaNN/meta_nn.h>
  4 | #include <iostream>
  5 | #include <cassert>
  6 | using namespace std;
  7 | using namespace MetaNN;
  8 | 
  9 | namespace
 10 | {
 11 | void test_dot_1()
 12 | {
 13 |     cout << "Test dot case 1 ...\t";
 14 |     auto rm = GenMatrix<int>(4, 5, 0, 1);
 15 |     auto cm = GenMatrix<int>(5, 3, 3, 2);
 16 |     auto mul = Dot(rm, cm);
 17 |     auto mul_r = Evaluate(mul);
 18 |     for (size_t i = 0; i < 4; ++i)
 19 |     {
 20 |         for (size_t j = 0; j<3; ++j)
 21 |         {
 22 |             int h = 0;
 23 |             for (size_t k = 0; k < 5; ++k)
 24 |             {
 25 |                 h += rm(i,k) * cm(k, j);
 26 |             }
 27 |             assert(h == mul_r(i, j));
 28 |         }
 29 |     }
 30 | 
 31 |     auto rm2 = GenMatrix<int>(111, 113, 0, 1);
 32 |     auto cm2 = GenMatrix<int>(111, 113, 2, 3);
 33 |     rm2 = rm2.SubMatrix(31, 35, 17, 22);
 34 |     cm2 = cm2.SubMatrix(31, 36, 41, 44);
 35 |     for (size_t i = 0; i < 4; ++i)
 36 |     {
 37 |         for (size_t j = 0; j < 5; ++j)
 38 |         {
 39 |             rm2.SetValue(i, j, rm(i, j));
 40 |         }
 41 |     }
 42 |     for (size_t i = 0; i < 5; ++i)
 43 |     {
 44 |         for (size_t j = 0; j < 3; ++j)
 45 |         {
 46 |             cm2.SetValue(i, j, cm(i, j));
 47 |         }
 48 |     }
 49 |     auto mul2 = Dot(rm2, cm2);
 50 |     auto mul2_r = Evaluate(mul2);
 51 | 
 52 |     for (size_t i = 0; i < 4; ++i)
 53 |     {
 54 |         for (size_t j = 0; j<3; ++j)
 55 |         {
 56 |             assert(mul2_r(i, j) == mul_r(i, j));
 57 |         }
 58 |     }
 59 |     cout << "done" << endl;
 60 | }
 61 | 
 62 | void test_dot_2()
 63 | {
 64 |     cout << "Test dot case 2 ...\t";
 65 |     auto rm = GenBatchMatrix<int>(4, 5, 3, 0, 1);
 66 |     auto cm = GenMatrix<int>(5, 3, 3, 2);
 67 |     auto mul = Dot(rm, cm);
 68 |     auto mul_r = Evaluate(mul);
 69 |     for (size_t b = 0; b < 3; ++b)
 70 |     {
 71 |         auto rm1 = rm[b];
 72 |         for (size_t i = 0; i < 4; ++i)
 73 |         {
 74 |             for (size_t j = 0; j<3; ++j)
 75 |             {
 76 |                 int h = 0;
 77 |                 for (size_t k = 0; k < 5; ++k)
 78 |                 {
 79 |                     h += rm1(i,k) * cm(k, j);
 80 |                 }
 81 |                 assert(h == mul_r[b](i, j));
 82 |             }
 83 |         }
 84 |     }
 85 |     cout << "done" << endl;
 86 | }
 87 | 
 88 | void test_dot_3()
 89 | {
 90 |     cout << "Test dot case 3 ...\t";
 91 |     auto rm = GenMatrix<int>(4, 5, 0, 1);
 92 |     auto cm = GenBatchMatrix<int>(5, 3, 3, 3, 2);
 93 |     auto mul = Dot(rm, cm);
 94 |     auto mul_r = Evaluate(mul);
 95 |     for (size_t b = 0; b < 3; ++b)
 96 |     {
 97 |         auto cm1 = cm[b];
 98 |         for (size_t i = 0; i < 4; ++i)
 99 |         {
100 |             for (size_t j = 0; j<3; ++j)
101 |             {
102 |                 int h = 0;
103 |                 for (size_t k = 0; k < 5; ++k)
104 |                 {
105 |                     h += rm(i,k) * cm1(k, j);
106 |                 }
107 |                 assert(h == mul_r[b](i, j));
108 |             }
109 |         }
110 |     }
111 |     cout << "done" << endl;
112 | }
113 | }
114 | 
115 | void test_dot()
116 | {
117 |     test_dot_1();
118 |     test_dot_2();   // BatchMatrix dot matrix
119 |     test_dot_3();
120 | }
121 | 


--------------------------------------------------------------------------------
/MetaNN/policies/policy_selector.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <type_traits>
 4 | #include <MetaNN/facilities/traits.h>
 5 | #include <MetaNN/policies/policy_container.h>
 6 | 
 7 | namespace MetaNN
 8 | {
 9 | namespace NSPolicySelect
10 | {
11 | template <typename TPolicyCont>
12 | struct PolicySelRes;
13 | 
14 | template <typename TPolicy>
15 | struct PolicySelRes<PolicyContainer<TPolicy>> : public TPolicy {};
16 | 
17 | template <typename TCurPolicy, typename... TOtherPolicies>
18 | struct PolicySelRes<PolicyContainer<TCurPolicy, TOtherPolicies...>>
19 |         : public TCurPolicy, public PolicySelRes<PolicyContainer<TOtherPolicies...>> {};
20 |         
21 | /// ====================== Split Major Class ============================
22 | template <typename MCO, typename TMajorClass, typename... TP>
23 | struct MajorFilter_
24 | {
25 |     using type = MCO;
26 | };
27 | 
28 | template <typename... TFilteredPolicies, typename TMajorClass,
29 |           typename TCurPolicy, typename... TP>
30 | struct MajorFilter_<PolicyContainer<TFilteredPolicies...>, TMajorClass,
31 |                     TCurPolicy, TP...>
32 | {
33 |     template <typename CurMajor, typename TDummy = void>
34 |     struct _impl
35 |     {
36 |         using type = typename MajorFilter_<PolicyContainer<TFilteredPolicies...>, TMajorClass, TP...>::type;
37 |     };
38 | 
39 |     template <typename TDummy>
40 |     struct _impl<TMajorClass, TDummy>
41 |     {
42 |         using type = typename MajorFilter_<PolicyContainer<TFilteredPolicies..., TCurPolicy>,
43 |                                            TMajorClass, TP...>::type;
44 |     };
45 |     using type = typename _impl<typename TCurPolicy::MajorClass>::type;
46 | };
47 | 
48 | /// ================= Minor Check ===============================
49 | template <typename TMinorClass, typename... TP>
50 | struct MinorDedup_
51 | {
52 |     static constexpr bool value = true;
53 | };
54 | 
55 | template <typename TMinorClass, typename TCurPolicy, typename... TP>
56 | struct MinorDedup_<TMinorClass, TCurPolicy, TP...>
57 | {
58 |     using TCurMirror = typename TCurPolicy::MinorClass;
59 |     constexpr static bool cur_check = !(std::is_same<TMinorClass, TCurMirror>::value);
60 |     constexpr static bool value = AndValue<cur_check,
61 |                                            MinorDedup_<TMinorClass, TP...>>;
62 | };
63 | 
64 | template <typename TPolicyCont>
65 | struct MinorCheck_
66 | {
67 |     static constexpr bool value = true;
68 | };
69 | 
70 | template <typename TCurPolicy, typename... TP>
71 | struct MinorCheck_<PolicyContainer<TCurPolicy, TP...>>
72 | {
73 |     static constexpr bool cur_check = MinorDedup_<typename TCurPolicy::MinorClass, TP...>::value;
74 | 
75 |     static constexpr bool value
76 |         = AndValue<cur_check, MinorCheck_<PolicyContainer<TP...>>>;
77 | };
78 | 
79 | template <typename TMajorClass, typename TPolicyContainer>
80 | struct Selector_;
81 | 
82 | template <typename TMajorClass, typename... TPolicies>
83 | struct Selector_<TMajorClass, PolicyContainer<TPolicies...>>
84 | {
85 |     using TMF = typename MajorFilter_<PolicyContainer<>, TMajorClass, TPolicies...>::type;
86 |     static_assert(MinorCheck_<TMF>::value, "Minor class set conflict!");
87 | 
88 |     using type = std::conditional_t<IsArrayEmpty<TMF>, TMajorClass, PolicySelRes<TMF>>;
89 | };
90 | }
91 | 
92 | template <typename TMajorClass, typename TPolicyContainer>
93 | using PolicySelect = typename NSPolicySelect::Selector_<TMajorClass, TPolicyContainer>::type;
94 | }


--------------------------------------------------------------------------------
/Test/operators/test_negative_log_likelihood.cpp:
--------------------------------------------------------------------------------
  1 | #include "test_negative_log_likelihood.h"
  2 | #include "../facilities/data_gen.h"
  3 | #include <MetaNN/meta_nn.h>
  4 | #include <cassert>
  5 | #include <iostream>
  6 | using namespace MetaNN;
  7 | using namespace std;
  8 | 
  9 | namespace
 10 | {
 11 | void test_negative_log_likelihood1()
 12 | {
 13 |     cout << "Test negative log likelihood case 1 ...\t";
 14 |     auto rm1 = GenMatrix<float>(4, 5, 1.0f, 0.0001f);
 15 |     auto rm2 = GenMatrix<float>(4, 5, 0.1f, 0.2f);
 16 |     auto t = NegativeLogLikelihood(rm1, rm2);
 17 |     auto t_r = Evaluate(t);
 18 | 
 19 |     float check = 0;
 20 |     for (size_t i = 0; i < 4; ++i)
 21 |     {
 22 |         for (size_t j = 0; j<5; ++j)
 23 |         {
 24 |             check -= rm1(i, j) * log(rm2(i, j));
 25 |         }
 26 |     }
 27 |     assert(fabs(t_r.Value() - check) < 0.0001);
 28 | 
 29 | 
 30 |     rm1 = GenMatrix<float>(111, 113, 1.1f, 0.0001f);
 31 |     rm1 = rm1.SubMatrix(31, 35, 17, 22);
 32 |     t = NegativeLogLikelihood(rm1, rm2);
 33 |     t_r = Evaluate(t);
 34 | 
 35 |     check = 0;
 36 |     for (size_t i = 0; i < 4; ++i)
 37 |     {
 38 |         for (size_t j = 0; j<5; ++j)
 39 |         {
 40 |             check -= rm1(i, j) * log(rm2(i, j));
 41 |         }
 42 |     }
 43 |     assert(fabs(t_r.Value() - check) < 0.0001);
 44 | 
 45 |     cout << "done" << endl;
 46 | }
 47 | 
 48 | void test_negative_log_likelihood2()
 49 | {
 50 |     cout << "Test negative log likelihood case 2 ...\t";
 51 |     {
 52 |         auto rm1 = GenMatrix<float>(4, 5, 1.0f, 0.0001f);
 53 |         auto rm2 = GenMatrix<float>(4, 5, 0.1f, 0.2f);
 54 |         auto res = NegativeLogLikelihood(rm1, rm2);
 55 |         auto res2 = NegativeLogLikelihood(rm1, rm2);
 56 | 
 57 |         assert(res == res2);
 58 | 
 59 |         auto cm1 = Evaluate(res);
 60 |         auto cm2 = Evaluate(res);
 61 |         assert(cm1 == cm2);
 62 |     }
 63 |     {
 64 |         auto rm1 = GenMatrix<float>(4, 5, 1.0f, 0.0001f);
 65 |         auto rm2 = GenMatrix<float>(4, 5, 0.1f, 0.2f);
 66 |         auto res = NegativeLogLikelihood(rm1, rm2);
 67 |         auto res2 = res;
 68 | 
 69 |         assert(res == res2);
 70 | 
 71 |         auto handle1 = res.EvalRegister();
 72 |         auto handle2 = res2.EvalRegister();
 73 |         EvalPlan<DeviceTags::CPU>::Eval();
 74 | 
 75 |         auto cm1 = handle1.Data();
 76 |         auto cm2 = handle2.Data();
 77 |         assert(cm1 == cm2);
 78 |     }
 79 |     cout << "done" << endl;
 80 | }
 81 | 
 82 | void test_negative_log_likelihood3()
 83 | {
 84 |     cout << "Test negative log likelihood case 3 ...\t";
 85 |     auto rm1 = GenBatchMatrix<float>(4, 5, 7, 1.0f, 0.0001f);
 86 |     auto rm2 = GenBatchMatrix<float>(4, 5, 7, 0.1f, 0.2f);
 87 |     auto t = NegativeLogLikelihood(rm1, rm2);
 88 |     auto t_r = Evaluate(t);
 89 | 
 90 |     for (size_t b = 0; b < 7; ++b)
 91 |     {
 92 |         float check = 0;
 93 |         for (size_t i = 0; i < 4; ++i)
 94 |         {
 95 |             for (size_t j = 0; j<5; ++j)
 96 |             {
 97 |                 check -= rm1[b](i, j) * log(rm2[b](i, j));
 98 |             }
 99 |         }
100 |         assert(fabs(t_r[b] - check) < 0.0001);
101 |     }
102 |     cout << "done" << endl;
103 | }
104 | }
105 | 
106 | void test_negative_log_likelihood()
107 | {
108 |     test_negative_log_likelihood1();
109 |     test_negative_log_likelihood2();
110 |     test_negative_log_likelihood3();
111 | }
112 | 


--------------------------------------------------------------------------------
/Test/model_rel/param_initializer/test_var_scale_filter.cpp:
--------------------------------------------------------------------------------
  1 | #include "test_var_scale_filter.h"
  2 | #include <MetaNN/meta_nn.h>
  3 | #include <cassert>
  4 | #include <iostream>
  5 | using namespace MetaNN;
  6 | using namespace std;
  7 | 
  8 | namespace
  9 | {
 10 | void test_xavier_filler1()
 11 | {
 12 |     cout << "test xavier filler case 1 ...";
 13 |     
 14 |     XavierFiller<PolicyContainer<PUniformVarScale>> filler;
 15 |     Matrix<float, DeviceTags::CPU> mat (100, 200);
 16 |     filler.Fill(mat, 100, 200);
 17 |     
 18 |     float mean = 0;
 19 |     for (size_t i = 0; i < mat.RowNum(); ++i)
 20 |     {
 21 |         for (size_t j = 0; j < mat.ColNum(); ++j)
 22 |         {
 23 |             mean += mat(i, j);
 24 |         }
 25 |     }
 26 |     mean /= mat.RowNum() * mat.ColNum();
 27 |     
 28 |     float var = 0;
 29 |     for (size_t i = 0; i < mat.RowNum(); ++i)
 30 |     {
 31 |         for (size_t j = 0; j < mat.ColNum(); ++j)
 32 |         {
 33 |             var += (mat(i, j) - mean) * (mat(i, j) - mean);
 34 |         }
 35 |     }
 36 |     var /= mat.RowNum() * mat.ColNum();
 37 |     
 38 |     // std = 0.0816 (sqrt(1/150))
 39 |     cout << "mean-delta = " << fabs(mean) << " std-delta = " << fabs(sqrt(var)-0.0816) << ' ';
 40 |     cout << "done" << endl;
 41 | }
 42 | 
 43 | void test_xavier_filler2()
 44 | {
 45 |     cout << "test xavier filler case 2 ...";
 46 |     
 47 |     XavierFiller<PolicyContainer<PNormVarScale>> filler;
 48 |     Matrix<float, DeviceTags::CPU> mat (100, 200);
 49 |     filler.Fill(mat, 100, 200);
 50 |     
 51 |     float mean = 0;
 52 |     for (size_t i = 0; i < mat.RowNum(); ++i)
 53 |     {
 54 |         for (size_t j = 0; j < mat.ColNum(); ++j)
 55 |         {
 56 |             mean += mat(i, j);
 57 |         }
 58 |     }
 59 |     mean /= mat.RowNum() * mat.ColNum();
 60 |     
 61 |     float var = 0;
 62 |     for (size_t i = 0; i < mat.RowNum(); ++i)
 63 |     {
 64 |         for (size_t j = 0; j < mat.ColNum(); ++j)
 65 |         {
 66 |             var += (mat(i, j) - mean) * (mat(i, j) - mean);
 67 |         }
 68 |     }
 69 |     var /= mat.RowNum() * mat.ColNum();
 70 |     
 71 |     // std = 0.0816 (sqrt(1/150))
 72 |     cout << "mean-delta = " << fabs(mean) << " std = " << fabs(sqrt(var)-0.0816) << ' ';
 73 |     cout << "done" << endl;
 74 | }
 75 | 
 76 | void test_msra_filler1()
 77 | {
 78 |     cout << "test msra filler case 1 ...";
 79 |     
 80 |     MSRAFiller<> filler;
 81 |     Matrix<float, DeviceTags::CPU> mat (100, 200);
 82 |     filler.Fill(mat, 100, 200);
 83 |     
 84 |     float mean = 0;
 85 |     for (size_t i = 0; i < mat.RowNum(); ++i)
 86 |     {
 87 |         for (size_t j = 0; j < mat.ColNum(); ++j)
 88 |         {
 89 |             mean += mat(i, j);
 90 |         }
 91 |     }
 92 |     mean /= mat.RowNum() * mat.ColNum();
 93 |     
 94 |     float var = 0;
 95 |     for (size_t i = 0; i < mat.RowNum(); ++i)
 96 |     {
 97 |         for (size_t j = 0; j < mat.ColNum(); ++j)
 98 |         {
 99 |             var += (mat(i, j) - mean) * (mat(i, j) - mean);
100 |         }
101 |     }
102 |     var /= mat.RowNum() * mat.ColNum();
103 |     
104 |     // std = 0.1414 (sqrt(2/50))
105 |     cout << "mean-delta = " << fabs(mean) << " std-delta = " << fabs(sqrt(var)-0.1414) << ' ';
106 |     cout << "done" << endl;
107 | }
108 | }
109 | 
110 | void test_var_scale_filter()
111 | {
112 |     test_xavier_filler1();
113 |     test_xavier_filler2();
114 |     test_msra_filler1();
115 | }


--------------------------------------------------------------------------------
/MetaNN/data/matrices/zero_matrix.h:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include <MetaNN/data/facilities/tags.h>
  4 | #include <MetaNN/data/matrices/matrices.h>
  5 | #include <MetaNN/evaluate/facilities/eval_buffer.h>
  6 | #include <MetaNN/evaluate/facilities/eval_group.h>
  7 | #include <MetaNN/evaluate/facilities/eval_handle.h>
  8 | #include <MetaNN/evaluate/facilities/eval_plan.h>
  9 | #include <MetaNN/evaluate/facilities/eval_unit.h>
 10 | #include <cassert>
 11 | #include <stdexcept>
 12 | 
 13 | namespace MetaNN
 14 | {
 15 | namespace NSZeroMatrix
 16 | {
 17 | template <typename TElem, typename TDevice>
 18 | class EvalUnit;
 19 | 
 20 | template <typename TElement>
 21 | class EvalUnit<TElement, DeviceTags::CPU>
 22 |     : public BaseEvalUnit<DeviceTags::CPU>
 23 | {
 24 | public:
 25 |     EvalUnit(EvalHandle<Matrix<TElement, DeviceTags::CPU>> resBuf,
 26 |              size_t rowNum, size_t colNum)
 27 |         : m_resHandle(std::move(resBuf))
 28 |         , m_rowNum(rowNum)
 29 |         , m_colNum(colNum) {}
 30 | 
 31 |     void Eval() override
 32 |     {
 33 |         m_resHandle.Allocate(m_rowNum, m_colNum);
 34 |         auto lowLayer = LowerAccess(m_resHandle.MutableData());
 35 |         const size_t rowLen = lowLayer.RowLen();
 36 |         auto mem = lowLayer.MutableRawMemory();
 37 |         if (rowLen != m_colNum)
 38 |         {
 39 |             throw std::runtime_error("Gap among matrix rows");
 40 |         }
 41 |         
 42 |         memset(mem, 0, sizeof(TElement) * m_colNum * m_rowNum);
 43 |         m_resHandle.SetEval();
 44 |     }
 45 | 
 46 | private:
 47 |     EvalHandle<Matrix<TElement, DeviceTags::CPU>> m_resHandle;
 48 |     size_t m_rowNum;
 49 |     size_t m_colNum;
 50 | };
 51 | }
 52 | 
 53 | template <typename TElem, typename TDevice>
 54 | class ZeroMatrix
 55 | {
 56 | public:
 57 |     using ElementType = TElem;
 58 |     using DeviceType = TDevice;
 59 |     
 60 | public:
 61 |     ZeroMatrix(size_t p_rowNum, size_t p_colNum)
 62 |         : m_rowNum(p_rowNum)
 63 |         , m_colNum(p_colNum) {}
 64 | 
 65 |     bool operator== (const ZeroMatrix& val) const
 66 |     {
 67 |         return (m_rowNum == val.m_rowNum) &&
 68 |                (m_colNum == val.m_colNum);
 69 |     }
 70 | 
 71 |     template <typename TOtherType>
 72 |     bool operator== (const TOtherType&) const
 73 |     {
 74 |         return false;
 75 |     }
 76 | 
 77 |     template <typename TData>
 78 |     bool operator!= (const TData& val) const
 79 |     {
 80 |         return !(operator==(val));
 81 |     }
 82 | 
 83 |     size_t RowNum() const { return m_rowNum; }
 84 | 
 85 |     size_t ColNum() const { return m_colNum; }
 86 | 
 87 |     auto EvalRegister() const
 88 |     {
 89 |         using TEvalUnit = NSZeroMatrix::EvalUnit<ElementType, DeviceType>;
 90 |         using TEvalGroup = TrivalEvalGroup<TEvalUnit>;
 91 |         if (!m_evalBuf.IsEvaluated())
 92 |         {
 93 |             auto evalHandle = m_evalBuf.Handle();
 94 |             decltype(auto) outPtr = evalHandle.DataPtr();
 95 |             TEvalUnit unit(std::move(evalHandle), m_rowNum, m_colNum);
 96 |             EvalPlan<DeviceType>::template Register<TEvalGroup>(std::move(unit), outPtr, {});
 97 |         }
 98 |         return m_evalBuf.ConstHandle();
 99 |     }
100 |     
101 | private:
102 |     size_t m_rowNum;
103 |     size_t m_colNum;
104 |     EvalBuffer<Matrix<ElementType, DeviceType>> m_evalBuf;
105 | };
106 | 
107 | template <typename TElem, typename TDevice>
108 | constexpr bool IsMatrix<ZeroMatrix<TElem, TDevice>> = true;
109 | }


--------------------------------------------------------------------------------
/MetaNN/layers/elementary/element_mul_layer.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <MetaNN/facilities/var_type_dict.h>
 4 | #include <MetaNN/layers/facilities/common_io.h>
 5 | #include <MetaNN/layers/facilities/policies.h>
 6 | #include <MetaNN/policies/policy_operations.h>
 7 | #include <stdexcept>
 8 | 
 9 | namespace MetaNN
10 | {
11 | using ElementMulLayerInput = VarTypeDict<struct ElementMulLayerIn1,
12 |                                          struct ElementMulLayerIn2>;
13 | 
14 | template <typename TPolicies>
15 | class ElementMulLayer
16 | {
17 |     static_assert(IsPolicyContainer<TPolicies>, "TPolicies is not a policy container.");
18 |     using CurLayerPolicy = PlainPolicy<TPolicies>;
19 | public:
20 |     static constexpr bool IsFeedbackOutput = PolicySelect<FeedbackPolicy, CurLayerPolicy>::IsFeedbackOutput;
21 |     static constexpr bool IsUpdate = false;
22 |     using InputType = ElementMulLayerInput;
23 |     using OutputType = LayerIO;
24 | 
25 | public:
26 |     template <typename TIn>
27 |     auto FeedForward(const TIn& p_in)
28 |     {
29 |         const auto& val1 = p_in.template Get<ElementMulLayerIn1>();
30 |         const auto& val2 = p_in.template Get<ElementMulLayerIn2>();
31 | 
32 |         using rawType1 = std::decay_t<decltype(val1)>;
33 |         using rawType2 = std::decay_t<decltype(val2)>;
34 | 
35 |         static_assert(!std::is_same<rawType1, NullParameter>::value, "parameter1 is invalid");
36 |         static_assert(!std::is_same<rawType2, NullParameter>::value, "parameter2 is invalid");
37 | 
38 |         if constexpr (IsFeedbackOutput)
39 |         {
40 |             m_data1.push(MakeDynamic(val1));
41 |             m_data2.push(MakeDynamic(val2));
42 |         }
43 |         return LayerIO::Create().template Set<LayerIO>(val1 * val2);
44 |     }
45 | 
46 |     template <typename TGrad>
47 |     auto FeedBackward(const TGrad& p_grad)
48 |     {
49 |         if constexpr (IsFeedbackOutput)
50 |         {
51 |             if ((m_data1.empty()) || (m_data2.empty()))
52 |             {
53 |                 throw std::runtime_error("Cannot do FeedBackward for ElementMulLayer.");
54 |             }
55 | 
56 |             auto top1 = m_data1.top();
57 |             auto top2 = m_data2.top();
58 |             m_data1.pop();
59 |             m_data2.pop();
60 | 
61 |             auto grad_eval = p_grad.template Get<LayerIO>();
62 | 
63 |             return ElementMulLayerInput::Create()
64 |                             .template Set<ElementMulLayerIn1>(grad_eval * top2)
65 |                             .template Set<ElementMulLayerIn2>(grad_eval * top1);
66 |         }
67 |         else
68 |         {
69 |             return ElementMulLayerInput::Create();
70 |         }
71 |     }
72 | 
73 |     void NeutralInvariant()
74 |     {
75 |         if constexpr(IsFeedbackOutput)
76 |         {
77 |             if ((!m_data1.empty()) || (!m_data2.empty()))
78 |             {
79 |                 throw std::runtime_error("NeutralInvariant Fail!");
80 |             }
81 |         }
82 |     }
83 | 
84 | private:
85 |     using DataType = LayerTraits::LayerInternalBuf<IsFeedbackOutput,
86 |                                                    PolicySelect<InputPolicy, CurLayerPolicy>::BatchMode,
87 |                                                    typename PolicySelect<OperandPolicy, CurLayerPolicy>::Element,
88 |                                                    typename PolicySelect<OperandPolicy, CurLayerPolicy>::Device,
89 |                                                    CategoryTags::Matrix, CategoryTags::BatchMatrix>;
90 |     DataType m_data1;
91 |     DataType m_data2;
92 | };
93 | }
94 | 


--------------------------------------------------------------------------------
/Test/operators/test_sigmoid_derivative.cpp:
--------------------------------------------------------------------------------
  1 | #include "test_sigmoid_derivative.h"
  2 | #include "../facilities/data_gen.h"
  3 | #include <MetaNN/meta_nn.h>
  4 | #include <cassert>
  5 | #include <iostream>
  6 | using namespace MetaNN;
  7 | using namespace std;
  8 | 
  9 | namespace
 10 | {
 11 | void test_sigmoid_derivative1()
 12 | {
 13 |     cout << "Test sigmoid derivative case 1 ...\t";
 14 |     auto rm1 = GenMatrix<float>(4, 5, 0, 0.001f);
 15 |     auto rm2 = GenMatrix<float>(4, 5, 1, 0.003f);
 16 | 
 17 |     auto t = SigmoidDerivative(rm1, rm2);
 18 |     auto t_r = Evaluate(t);
 19 |     for (size_t i = 0; i < 4; ++i)
 20 |     {
 21 |         for (size_t j = 0; j<5; ++j)
 22 |         {
 23 |             float aim = rm1(i, j) * rm2(i, j) * (1 - rm2(i, j));
 24 | 
 25 |             assert(fabs(t_r(i, j) - aim) < 0.0001);
 26 |         }
 27 |     }
 28 | 
 29 |     rm1 = GenMatrix<float>(111, 113, 1, 0.002f);
 30 |     rm2 = GenMatrix<float>(111, 113, 0, 0.001f);
 31 |     rm1 = rm1.SubMatrix(31, 35, 17, 22);
 32 |     rm2 = rm2.SubMatrix(30, 34, 18, 23);
 33 |     t = SigmoidDerivative(rm1, rm2);
 34 |     t_r = Evaluate(t);
 35 |     for (size_t i = 0; i < 4; ++i)
 36 |     {
 37 |         for (size_t j = 0; j<5; ++j)
 38 |         {
 39 |             float aim = rm1(i, j) * rm2(i, j) * (1 - rm2(i, j));
 40 |             assert(fabs(t_r(i, j) - aim) < 0.0001);
 41 |         }
 42 |     }
 43 |     cout << "done" << endl;
 44 | }
 45 | 
 46 | void test_sigmoid_derivative2()
 47 | {
 48 |     cout << "Test sigmoid derivative case 2 ...\t";
 49 |     {
 50 |         auto rm1 = GenMatrix<float>(4, 5, 0, 0.001f);
 51 |         auto rm2 = GenMatrix<float>(4, 5, 1, 0.003f);
 52 |         auto res = SigmoidDerivative(rm1, rm2);
 53 |         auto res2 = SigmoidDerivative(rm1, rm2);
 54 | 
 55 |         assert(res == res2);
 56 | 
 57 |         auto handle1 = res.EvalRegister();
 58 |         auto handle2 = res.EvalRegister();
 59 |         EvalPlan<DeviceTags::CPU>::Eval();
 60 | 
 61 |         auto cm1 = handle1.Data();
 62 |         auto cm2 = handle2.Data();
 63 |         assert(cm1 == cm2);
 64 |     }
 65 |     {
 66 |         auto rm1 = GenMatrix<float>(4, 5, 0, 0.001f);
 67 |         auto rm2 = GenMatrix<float>(4, 5, 1, 0.003f);
 68 |         auto res = SigmoidDerivative(rm1, rm2);
 69 |         auto res2 = res;
 70 | 
 71 |         assert(res == res2);
 72 | 
 73 |         auto handle1 = res.EvalRegister();
 74 |         auto handle2 = res2.EvalRegister();
 75 |         EvalPlan<DeviceTags::CPU>::Eval();
 76 | 
 77 |         auto cm1 = handle1.Data();
 78 |         auto cm2 = handle2.Data();
 79 |         assert(cm1 == cm2);
 80 |     }
 81 |     cout << "done" << endl;
 82 | }
 83 | 
 84 | void test_sigmoid_derivative3()
 85 | {
 86 |     cout << "Test sigmoid derivative case 3 ...\t";
 87 |     auto rm1 = GenBatchMatrix<float>(4, 5, 7, 0, 0.001f);
 88 |     auto rm2 = GenBatchMatrix<float>(4, 5, 7, 1, 0.003f);
 89 | 
 90 |     auto t = SigmoidDerivative(rm1, rm2);
 91 |     auto t_r = Evaluate(t);
 92 |     
 93 |     for (size_t b = 0; b < 7; ++b)
 94 |     {
 95 |         for (size_t i = 0; i < 4; ++i)
 96 |         {
 97 |             for (size_t j = 0; j<5; ++j)
 98 |             {
 99 |                 float aim = rm1[b](i, j) * rm2[b](i, j) * (1 - rm2[b](i, j));
100 | 
101 |                 assert(fabs(t_r[b](i, j) - aim) < 0.0001);
102 |             }
103 |         }
104 |     }
105 |     cout << "done" << endl;
106 | }
107 | }
108 | 
109 | void test_sigmoid_derivative()
110 | {
111 |     test_sigmoid_derivative1();
112 |     test_sigmoid_derivative2();
113 |     test_sigmoid_derivative3();
114 | }
115 | 


--------------------------------------------------------------------------------
/MetaNN/data/matrices/one_hot_vector.h:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include <MetaNN/data/matrices/matrices.h>
  4 | #include <MetaNN/evaluate/facilities/eval_buffer.h>
  5 | #include <MetaNN/evaluate/facilities/eval_group.h>
  6 | #include <MetaNN/evaluate/facilities/eval_handle.h>
  7 | #include <MetaNN/evaluate/facilities/eval_plan.h>
  8 | #include <MetaNN/evaluate/facilities/eval_unit.h>
  9 | 
 10 | #include <cassert>
 11 | #include <memory>
 12 | 
 13 | namespace MetaNN
 14 | {
 15 | namespace NSOneHotVector
 16 | {
 17 | template <typename TElem, typename TDevice>
 18 | class EvalUnit;
 19 | 
 20 | template <typename TElement>
 21 | class EvalUnit<TElement, DeviceTags::CPU>
 22 |     : public BaseEvalUnit<DeviceTags::CPU>
 23 | {
 24 | public:
 25 |     EvalUnit(EvalHandle<Matrix<TElement, DeviceTags::CPU>> resBuf,
 26 |              size_t rowNum, size_t colNum, size_t val)
 27 |         : m_resHandle(std::move(resBuf))
 28 |         , m_rowNum(rowNum)
 29 |         , m_colNum(colNum)
 30 |         , m_val(val)
 31 |     {
 32 |         assert(m_val < (m_rowNum * m_colNum));
 33 |     }
 34 | 
 35 |     void Eval() override
 36 |     {
 37 |         auto& mutableData = m_resHandle.MutableData();
 38 |         m_resHandle.Allocate(m_rowNum, m_colNum);
 39 |         auto lowLayer = LowerAccess(mutableData);
 40 |         auto mem = lowLayer.MutableRawMemory();
 41 |         memset(mem, 0, sizeof(TElement) * m_rowNum * m_colNum);
 42 |         mem[m_val] = 1;
 43 |         m_resHandle.SetEval();
 44 |     }
 45 | 
 46 | private:
 47 |     EvalHandle<Matrix<TElement, DeviceTags::CPU>> m_resHandle;
 48 |     size_t m_rowNum;
 49 |     size_t m_colNum;
 50 |     size_t m_val;
 51 | };
 52 | }
 53 | 
 54 | template <typename TElem, typename TDevice>
 55 | class OneHotVector
 56 | {
 57 | public:
 58 |     using ElementType = TElem;
 59 |     using DeviceType = TDevice;
 60 | 
 61 | public:
 62 |     OneHotVector(size_t p_colNum, size_t p_hotPos)
 63 |         : m_colNum(p_colNum)
 64 |         , m_hotPos(p_hotPos)
 65 |     {
 66 |         assert(p_hotPos < m_colNum);
 67 |     }
 68 | 
 69 |     bool operator== (const OneHotVector& val) const
 70 |     {
 71 |         return (m_hotPos == val.m_hotPos) &&
 72 |                (m_colNum == val.m_colNum);
 73 |     }
 74 | 
 75 |     template <typename TOtherType>
 76 |     bool operator== (const TOtherType&) const
 77 |     {
 78 |         return false;
 79 |     }
 80 | 
 81 |     template <typename TData>
 82 |     bool operator!= (const TData& val) const
 83 |     {
 84 |         return !(operator==(val));
 85 |     }
 86 | 
 87 |     size_t RowNum() const { return 1; }
 88 |     size_t ColNum() const { return m_colNum; }
 89 | 
 90 |     auto EvalRegister() const
 91 |     {
 92 |         using TEvalUnit = NSOneHotVector::EvalUnit<ElementType, DeviceType>;
 93 |         using TEvalGroup = TrivalEvalGroup<TEvalUnit>;
 94 |         if (!m_evalBuf.IsEvaluated())
 95 |         {
 96 |             auto evalHandle = m_evalBuf.Handle();
 97 |             decltype(auto) outputPtr = evalHandle.DataPtr();
 98 |             TEvalUnit unit(std::move(evalHandle), 1, m_colNum, m_hotPos);
 99 |             EvalPlan<DeviceType>::template Register<TEvalGroup>(std::move(unit), outputPtr, {});
100 |         }
101 |         return m_evalBuf.ConstHandle();
102 |     }
103 | 
104 |     auto HotPos() const
105 |     {
106 |         return m_hotPos;
107 |     }
108 | 
109 | private:
110 |     size_t m_colNum;
111 |     size_t m_hotPos;
112 |     EvalBuffer<Matrix<ElementType, DeviceType>> m_evalBuf;
113 | };
114 | 
115 | template <typename TElem, typename TDevice>
116 | constexpr bool IsMatrix<OneHotVector<TElem, TDevice>> = true;
117 | }


--------------------------------------------------------------------------------
/Test/operators/test_negative_log_likelihood_derivative.cpp:
--------------------------------------------------------------------------------
  1 | #include "test_negative_log_likelihood_derivative.h"
  2 | #include "../facilities/data_gen.h"
  3 | 
  4 | #include <MetaNN/meta_nn.h>
  5 | #include <cassert>
  6 | #include <iostream>
  7 | using namespace MetaNN;
  8 | using namespace std;
  9 | 
 10 | namespace
 11 | {
 12 | void test_negative_log_likelihood_derivative1()
 13 | {
 14 |     cout << "Test negative log likelihood derivative case 1 ...\t";
 15 |     auto rm1 = GenMatrix<float>(4, 5, 1, 1);
 16 |     auto rm2 = GenMatrix<float>(4, 5, 2, 2);
 17 |     auto div = NegativeLogLikelihoodDerivative(Scalar<float>(0.5), rm1, rm2);
 18 |     auto div_r = Evaluate(div);
 19 | 
 20 |     for (size_t i = 0; i < 4; ++i)
 21 |     {
 22 |         for (size_t j = 0; j<5; ++j)
 23 |         {
 24 |             assert(fabs(div_r(i, j) + 0.5 * rm1(i, j) / rm2(i, j)) < 0.001);
 25 |         }
 26 |     }
 27 | 
 28 |     rm1 = GenMatrix<float>(111, 113, 1, 1);
 29 |     rm2 = GenMatrix<float>(111, 113, 2, 3);
 30 |     rm1 = rm1.SubMatrix(31, 35, 17, 22);
 31 |     rm2 = rm2.SubMatrix(41, 45, 27, 32);
 32 |     div = NegativeLogLikelihoodDerivative(Scalar<float>(0.3), rm1, rm2);
 33 |     div_r = Evaluate(div);
 34 |     for (size_t i = 0; i < 4; ++i)
 35 |     {
 36 |         for (size_t j = 0; j<5; ++j)
 37 |         {
 38 |             assert(fabs(div_r(i, j) + 0.3 * rm1(i, j) / rm2(i, j)) < 0.001);
 39 |         }
 40 |     }
 41 |     cout << "done" << endl;
 42 | }
 43 | 
 44 | void test_negative_log_likelihood_derivative2()
 45 | {
 46 |     cout << "Test negative log likelihood derivative case 2 ...\t";
 47 |     {
 48 |         auto rm1 = GenMatrix<float>(4, 5, 1.0f, 0.0001f);
 49 |         auto rm2 = GenMatrix<float>(4, 5, 2, 2);
 50 |         auto res = NegativeLogLikelihoodDerivative(Scalar<float>(1), rm1, rm2);
 51 |         auto res2 = NegativeLogLikelihoodDerivative(Scalar<float>(1), rm1, rm2);
 52 | 
 53 |         assert(res == res2);
 54 | 
 55 |         auto cm1 = Evaluate(res);
 56 |         auto cm2 = Evaluate(res);
 57 |         assert(cm1 == cm2);
 58 |     }
 59 |     {
 60 |         auto rm1 = GenMatrix<float>(4, 5, 1.0f, 0.0001f);
 61 |         auto rm2 = GenMatrix<float>(4, 5, 2, 2);
 62 |         auto res = NegativeLogLikelihoodDerivative(Scalar<float>(1), rm1, rm2);
 63 |         auto res2 = res;
 64 | 
 65 |         assert(res == res2);
 66 | 
 67 |         auto handle1 = res.EvalRegister();
 68 |         auto handle2 = res2.EvalRegister();
 69 |         EvalPlan<DeviceTags::CPU>::Eval();
 70 | 
 71 |         auto cm1 = handle1.Data();
 72 |         auto cm2 = handle2.Data();
 73 |         assert(cm1 == cm2);
 74 |     }
 75 |     cout << "done" << endl;
 76 | }
 77 | 
 78 | void test_negative_log_likelihood_derivative3()
 79 | {
 80 |     cout << "Test negative log likelihood derivative case 3 ...\t";
 81 |     auto rm1 = GenBatchMatrix<float>(4, 5, 7, 1, 1);
 82 |     auto rm2 = GenBatchMatrix<float>(4, 5, 7, 2, 2);
 83 |     auto div = NegativeLogLikelihoodDerivative(MakeDuplicate(7, Scalar<float>(0.5)), rm1, rm2);
 84 |     auto div_r = Evaluate(div);
 85 | 
 86 |     for (size_t b = 0; b < 7; ++b)
 87 |     {
 88 |         for (size_t i = 0; i < 4; ++i)
 89 |         {
 90 |             for (size_t j = 0; j<5; ++j)
 91 |             {
 92 |                 assert(fabs(div_r[b](i, j) + 0.5 * rm1[b](i, j) / rm2[b](i, j)) < 0.001);
 93 |             }
 94 |         }
 95 |     }
 96 |     cout << "done" << endl;
 97 | }
 98 | }
 99 | 
100 | void test_negative_log_likelihood_derivative()
101 | {
102 |     test_negative_log_likelihood_derivative1();
103 |     test_negative_log_likelihood_derivative2();
104 |     test_negative_log_likelihood_derivative3();
105 | }
106 | 


--------------------------------------------------------------------------------
/MetaNN/layers/elementary/abs_layer.h:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | #include <MetaNN/layers/facilities/common_io.h>
  3 | #include <MetaNN/layers/facilities/policies.h>
  4 | #include <MetaNN/layers/facilities/traits.h>
  5 | #include <MetaNN/policies/policy_operations.h>
  6 | 
  7 | namespace MetaNN
  8 | {
  9 | namespace NSAbsLayer
 10 | {
 11 | template <bool isFeedback>
 12 | struct FeedbackOut_
 13 | {
 14 |     template <typename ElementType, typename DeviceType>
 15 |     using InternalType = LayerTraits::LayerInternalBufType<ElementType, DeviceType,
 16 |                                                            CategoryTags::Matrix>;
 17 | 
 18 |     template <typename T, typename DataType>
 19 |     static void RecordData(const T& p_in, DataType& data)
 20 |     {
 21 |         data.push(MakeDynamic(p_in));
 22 |     }
 23 | 
 24 |     template <typename TGrad, typename DataType>
 25 |     static auto FeedBack(DataType& data, const TGrad& grad)
 26 |     {
 27 |         if (data.empty())
 28 |         {
 29 |             throw std::runtime_error("Cannot feed back in SigmoidLayer");
 30 |         }
 31 |         auto tmp = grad.template Get<LayerIO>();
 32 |         auto& tmp2 = data.top();
 33 |         auto res = LayerIO::Create().template Set<LayerIO>(Sign(tmp2) * tmp);
 34 |         data.pop();
 35 |         return res;
 36 |     }
 37 | };
 38 | 
 39 | template <>
 40 | struct FeedbackOut_<false>
 41 | {
 42 |     template <typename ElementType, typename DeviceType>
 43 |     using InternalType = NullParameter;
 44 | 
 45 |     template <typename T, typename DataType>
 46 |     static void RecordData(T&&, DataType&)
 47 |     { }
 48 | 
 49 |     template <typename TGrad, typename DataType>
 50 |     static auto FeedBack(const DataType&, const TGrad&)
 51 |     {
 52 |         return LayerIO::Create();
 53 |     }
 54 | };
 55 | }
 56 | 
 57 | template <typename TPolicies>
 58 | class AbsLayer
 59 | {
 60 |     static_assert(IsPolicyContainer<TPolicies>, "TPolicies is not a policy container.");
 61 |     using CurLayerPolicy = PlainPolicy<TPolicies>;
 62 | 
 63 | public:
 64 |     static constexpr bool IsFeedbackOutput = PolicySelect<FeedbackPolicy, CurLayerPolicy>::IsFeedbackOutput;
 65 |     static constexpr bool IsUpdate = false;
 66 |     using InputType = LayerIO;
 67 |     using OutputType = LayerIO;
 68 | 
 69 | private:
 70 |     using ElementType = typename PolicySelect<OperandPolicy, CurLayerPolicy>::Element;
 71 |     using DeviceType = typename PolicySelect<OperandPolicy, CurLayerPolicy>::Device;
 72 | 
 73 | private:
 74 |     using FeedbackOut_ = NSAbsLayer::FeedbackOut_<IsFeedbackOutput>;
 75 |     using InternalType = typename FeedbackOut_::template InternalType<ElementType, DeviceType>;
 76 | 
 77 | public:
 78 |     template <typename TIn>
 79 |     auto FeedForward(const TIn& p_in)
 80 |     {
 81 |         const auto& val = p_in.template Get<LayerIO>();
 82 | 
 83 |         using rawType = std::decay_t<decltype(val)>;
 84 |         static_assert(!std::is_same<rawType, NullParameter>::value, "parameter is invalid");
 85 | 
 86 |         auto tmp = Abs(val);
 87 |         FeedbackOut_::RecordData(val, m_data);
 88 |         return LayerIO::Create().template Set<LayerIO>(tmp);
 89 |     }
 90 | 
 91 |     template <typename TGrad>
 92 |     auto FeedBackward(const TGrad& p_grad)
 93 |     {
 94 |         return FeedbackOut_::FeedBack(m_data, p_grad);
 95 |     }
 96 | 
 97 |     void NeutralInvariant() const
 98 |     {
 99 |         if constexpr(IsFeedbackOutput)
100 |         {
101 |             if (!m_data.empty())
102 |             {
103 |                 throw std::runtime_error("NeutralInvariant Fail!");
104 |             }
105 |         }
106 |     }
107 | 
108 | private:
109 |     InternalType m_data;
110 | };
111 | }
112 | 


--------------------------------------------------------------------------------
/MetaNN/operators/collapse.h:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | namespace MetaNN
  4 | {
  5 | template <>
  6 | struct OperCategory_<UnaryOpTags::Collapse, CategoryTags::BatchMatrix>
  7 | {
  8 |     using type = CategoryTags::Matrix;
  9 | };
 10 | 
 11 | namespace NSCollapse
 12 | {
 13 | namespace NSCaseGen
 14 | {
 15 | template <typename TOperand, typename TElem, typename TDevice>
 16 | class EvalUnit;
 17 | 
 18 | template <typename TOperand, typename TElem>
 19 | class EvalUnit<TOperand, TElem, DeviceTags::CPU>
 20 |     : public BaseEvalUnit<DeviceTags::CPU>
 21 | {
 22 | public:
 23 |     using ElementType = TElem;
 24 |     using DeviceType = DeviceTags::CPU;
 25 | 
 26 |     EvalUnit(TOperand evalInput,
 27 |              EvalHandle<Matrix<ElementType, DeviceType>> evalOutput)
 28 |         : m_evalInput(std::move(evalInput))
 29 |         , m_evalOutput(std::move(evalOutput)) {}
 30 | 
 31 |     void Eval() override
 32 |     {
 33 |         const auto& p_v = m_evalInput.Data();
 34 | 
 35 |         const size_t rowNum = p_v.RowNum();
 36 |         const size_t colNum = p_v.ColNum();
 37 |         const size_t batchNum = p_v.BatchNum();
 38 |         m_evalOutput.Allocate(rowNum, colNum);
 39 |         
 40 |         auto& res = m_evalOutput.MutableData();
 41 | 
 42 |         for (size_t j = 0; j < rowNum; ++j)
 43 |         {
 44 |             for (size_t k = 0; k < colNum; ++k)
 45 |             {
 46 |                 TElem tmp = 0;
 47 |                 for (size_t i = 0; i < batchNum; ++i)
 48 |                 {
 49 |                     tmp += p_v[i](j, k);
 50 |                 }
 51 |                 res.SetValue(j, k, tmp);
 52 |             }
 53 |         }
 54 |         m_evalOutput.SetEval();
 55 |     }
 56 | 
 57 | private:
 58 |     TOperand m_evalInput;
 59 |     EvalHandle<Matrix<ElementType, DeviceType>> m_evalOutput;
 60 | };
 61 | 
 62 | struct Calculator
 63 | {
 64 |     template <typename TCaseTail, typename TEvalRes, typename TOperand>
 65 |     static void EvalRegister(TEvalRes& evalRes, const TOperand& operand)
 66 |     {
 67 |         static_assert(std::is_same<TCaseTail, OperSeqContainer<>>::value,
 68 |                       "General Case is not the last one");
 69 |                       
 70 |         using ElementType = typename TEvalRes::DataType::ElementType;
 71 |         using DeviceType = typename TEvalRes::DataType::DeviceType;
 72 |         
 73 |         auto handle = operand.EvalRegister();
 74 |         using UnitType = EvalUnit<decltype(handle), ElementType, DeviceType>;
 75 |         using GroupType = TrivalEvalGroup<UnitType>;
 76 | 
 77 |         auto outHandle = evalRes.Handle();
 78 |         const void* dataPtr = outHandle.DataPtr();
 79 |         const void* depVec = handle.DataPtr();
 80 |         
 81 |         UnitType unit(std::move(handle), std::move(outHandle));
 82 |         EvalPlan<DeviceType>::template Register<GroupType>(std::move(unit), dataPtr, {depVec});
 83 |     }
 84 | };
 85 | }
 86 | }
 87 | 
 88 | template <>
 89 | struct OperSeq_<UnaryOpTags::Collapse>
 90 | {
 91 |     using type = OperSeqContainer<NSCollapse::NSCaseGen::Calculator>;
 92 | };
 93 | 
 94 | template <typename TP>
 95 | struct OperCollapse_
 96 | {
 97 | // valid check
 98 | private:
 99 |     using rawM = std::decay_t<TP>;
100 | 
101 | public:
102 |     static constexpr bool valid = IsBatchMatrix<rawM>;
103 | 
104 | public:
105 |     static auto Eval(TP&& p_m)
106 |     {
107 |         using ResType = UnaryOp<UnaryOpTags::Collapse, rawM>;
108 |         return ResType(std::forward<TP>(p_m));
109 |     }
110 | };
111 | 
112 | template <typename TP,
113 |           std::enable_if_t<OperCollapse_<TP>::valid>* = nullptr>
114 | auto Collapse(TP&& p_m)
115 | {
116 |     return OperCollapse_<TP>::Eval(std::forward<TP>(p_m));
117 | }
118 | }


--------------------------------------------------------------------------------
/MetaNN/operators/facilities/organizer.h:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include <MetaNN/data/facilities/traits.h>
  4 | 
  5 | namespace MetaNN
  6 | {
  7 | template <typename TOpTag, typename TCate>
  8 | class OperOrganizer;
  9 | 
 10 | template <typename TOpTag>
 11 | class OperOrganizer<TOpTag, CategoryTags::Scalar>
 12 | {
 13 | public:
 14 |     template <typename THead, typename...TRemain>
 15 |     OperOrganizer(const THead&, const TRemain&...)
 16 |     {}
 17 | };
 18 | 
 19 | template <typename TOpTag>
 20 | class OperOrganizer<TOpTag, CategoryTags::Matrix>
 21 | {
 22 | private:
 23 |     template <typename THead, typename...TRemain>
 24 |     bool SameDim(const THead&, const TRemain&...)
 25 |     {
 26 |         return true;
 27 |     }
 28 | 
 29 |     template <typename THead, typename TCur, typename...TRemain>
 30 |     bool SameDim(const THead& head, const TCur& cur, const TRemain&...rem)
 31 |     {
 32 |         const bool tmp = (head.RowNum() == cur.RowNum()) &&
 33 |                          (head.ColNum() == cur.ColNum());
 34 |         return tmp && SameDim(cur, rem...);
 35 |     }
 36 | 
 37 | public:
 38 |     template <typename THead, typename...TRemain>
 39 |     OperOrganizer(const THead& head, const TRemain&... rem)
 40 |         : m_rowNum(head.RowNum())
 41 |         , m_colNum(head.ColNum())
 42 |     {
 43 |         assert(SameDim(head, rem...));
 44 |     }
 45 | 
 46 |     size_t RowNum() const { return m_rowNum; }
 47 |     size_t ColNum() const { return m_colNum; }
 48 | 
 49 | private:
 50 |     size_t m_rowNum;
 51 |     size_t m_colNum;
 52 | };
 53 | 
 54 | template <typename TOpTag>
 55 | class OperOrganizer<TOpTag, CategoryTags::BatchMatrix>
 56 | {
 57 | private:
 58 |     template <typename THead, typename...TRemain>
 59 |     bool SameDim(const THead&, const TRemain&...)
 60 |     {
 61 |         return true;
 62 |     }
 63 | 
 64 |     template <typename THead, typename TCur, typename...TRemain>
 65 |     bool SameDim(const THead& head, const TCur& cur, const TRemain&...rem)
 66 |     {
 67 |         const bool tmp = (head.RowNum() == cur.RowNum()) &&
 68 |                          (head.ColNum() == cur.ColNum()) &&
 69 |                          (head.BatchNum() == cur.BatchNum());
 70 |         return tmp && SameDim(cur, rem...);
 71 |     }
 72 | 
 73 | public:
 74 |     template <typename THead, typename...TRemain>
 75 |     OperOrganizer(const THead& head, const TRemain&... rem)
 76 |         : m_rowNum(head.RowNum())
 77 |         , m_colNum(head.ColNum())
 78 |         , m_batchNum(head.BatchNum())
 79 |     {
 80 |         assert(SameDim(head, rem...));
 81 |     }
 82 | 
 83 |     size_t RowNum() const { return m_rowNum; }
 84 |     size_t ColNum() const { return m_colNum; }
 85 |     size_t BatchNum() const { return m_batchNum; }
 86 | 
 87 | private:
 88 |     size_t m_rowNum;
 89 |     size_t m_colNum;
 90 |     size_t m_batchNum;
 91 | };
 92 | 
 93 | template <typename TOpTag>
 94 | class OperOrganizer<TOpTag, CategoryTags::BatchScalar>
 95 | {
 96 | private:
 97 |     template <typename THead, typename...TRemain>
 98 |     bool SameDim(const THead&, const TRemain&...)
 99 |     {
100 |         return true;
101 |     }
102 | 
103 |     template <typename THead, typename TCur, typename...TRemain>
104 |     bool SameDim(const THead& head, const TCur& cur, const TRemain&...rem)
105 |     {
106 |         const bool tmp = (head.BatchNum() == cur.BatchNum());
107 |         return tmp && SameDim(cur, rem...);
108 |     }
109 | 
110 | public:
111 |     template <typename THead, typename...TRemain>
112 |     OperOrganizer(const THead& head, const TRemain&... rem)
113 |         : m_batchNum(head.BatchNum())
114 |     {
115 |         assert(SameDim(head, rem...));
116 |     }
117 | 
118 |     size_t BatchNum() const { return m_batchNum; }
119 | 
120 | private:
121 |     size_t m_batchNum;
122 | };
123 | }


--------------------------------------------------------------------------------
/MetaNN/layers/elementary/softmax_layer.h:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | #include <MetaNN/layers/facilities/common_io.h>
  3 | #include <MetaNN/layers/facilities/policies.h>
  4 | #include <MetaNN/policies/policy_operations.h>
  5 | 
  6 | namespace MetaNN
  7 | {
  8 | namespace NSSoftmaxLayer
  9 | {
 10 | template <bool isFeedback>
 11 | struct FeedbackOut_
 12 | {
 13 |     template <typename T, typename DataType>
 14 |     static auto RecordData(const T& p_in, DataType& data)
 15 |     {
 16 |         auto res = MakeDynamic(p_in);
 17 |         data.push(res);
 18 |         return res;
 19 |     }
 20 | 
 21 |     template <typename TGrad, typename DataType>
 22 |     static auto FeedBack(DataType& data, const TGrad& grad)
 23 |     {
 24 |         if (data.empty())
 25 |         {
 26 |             throw std::runtime_error("Cannot do FeedBackward for Softmax Layer");
 27 |         }
 28 | 
 29 |         auto tmp = VecSoftmaxDerivative(grad.template Get<LayerIO>(), data.top());
 30 |         auto res = LayerIO::Create().template Set<LayerIO>(tmp);
 31 |         data.pop();
 32 |         return res;
 33 |     }
 34 | };
 35 | 
 36 | template <>
 37 | struct FeedbackOut_<false>
 38 | {
 39 |     template <typename T, typename DataType>
 40 |     static auto RecordData(T&& val, DataType&)
 41 |     {
 42 |         return std::forward<T>(val);
 43 |     }
 44 | 
 45 |     template <typename TGrad, typename DataType>
 46 |     static auto FeedBack(const DataType&, const TGrad&)
 47 |     {
 48 |         return LayerIO::Create();
 49 |     }
 50 | };
 51 | }
 52 | 
 53 | template <typename TPolicies>
 54 | class SoftmaxLayer
 55 | {
 56 |     static_assert(IsPolicyContainer<TPolicies>, "TPolicies is not a policy container.");
 57 |     using CurLayerPolicy = PlainPolicy<TPolicies>;
 58 | 
 59 | public:
 60 |     static constexpr bool IsFeedbackOutput = PolicySelect<FeedbackPolicy, CurLayerPolicy>::IsFeedbackOutput;
 61 |     static constexpr bool IsUpdate = false;
 62 |     using InputType = LayerIO;
 63 |     using OutputType = LayerIO;
 64 | 
 65 | private:
 66 |     using ElementType = typename PolicySelect<OperandPolicy, CurLayerPolicy>::Element;
 67 |     using DeviceType = typename PolicySelect<OperandPolicy, CurLayerPolicy>::Device;
 68 | 
 69 | private:
 70 |     using FeedbackOut_ = NSSoftmaxLayer::FeedbackOut_<IsFeedbackOutput>;
 71 |     
 72 | public:
 73 |     template <typename TIn>
 74 |     auto FeedForward(const TIn& p_in)
 75 |     {
 76 |         const auto& val = p_in.template Get<LayerIO>();
 77 | 
 78 |         using rawType = std::decay_t<decltype(val)>;
 79 |         static_assert(!std::is_same<rawType, NullParameter>::value, "parameter is invalid");
 80 | 
 81 |         auto tmp = VecSoftmax(val);
 82 |         auto tmp2 = FeedbackOut_::RecordData(tmp, m_data);
 83 |         return LayerIO::Create().template Set<LayerIO>(tmp2);
 84 |     }
 85 | 
 86 |     template <typename TGrad>
 87 |     auto FeedBackward(const TGrad& p_grad)
 88 |     {
 89 |         return FeedbackOut_::FeedBack(m_data, p_grad);
 90 |     }
 91 | 
 92 |     void NeutralInvariant()
 93 |     {
 94 |         if constexpr(IsFeedbackOutput)
 95 |         {
 96 |             if (!m_data.empty())
 97 |             {
 98 |                 throw std::runtime_error("NeutralInvariant Fail!");
 99 |             }
100 |         }
101 |     }
102 |     
103 | private:
104 |     using DataType = LayerTraits::LayerInternalBuf<IsFeedbackOutput,
105 |                                                    PolicySelect<InputPolicy, CurLayerPolicy>::BatchMode,
106 |                                                    typename PolicySelect<OperandPolicy, CurLayerPolicy>::Element,
107 |                                                    typename PolicySelect<OperandPolicy, CurLayerPolicy>::Device,
108 |                                                    CategoryTags::Matrix, CategoryTags::BatchMatrix>;
109 |     DataType m_data;
110 | };
111 | }
112 | 


--------------------------------------------------------------------------------
/MetaNN/model_rel/param_initializer/var_scale_filler.h:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include <random>
  4 | #include <stdexcept>
  5 | #include <type_traits>
  6 | #include <MetaNN/model_rel/param_initializer/facilities/fill_with_spec_dist.h>
  7 | 
  8 | namespace MetaNN
  9 | {
 10 | template <typename TPolicyCont = PolicyContainer<>>
 11 | class VarScaleFiller
 12 | {
 13 |     using TRandomEngine = typename PolicySelect<InitPolicy, TPolicyCont>::RandEngine;
 14 | public:
 15 |     VarScaleFiller(double factor = 1,
 16 |                    unsigned seed = std::random_device{}())
 17 |         : m_factor(factor)
 18 |         , m_engine(seed)
 19 |     {}
 20 |     
 21 |     template <typename TData>
 22 |     void Fill(TData& data, size_t fanIn, size_t fanOut)
 23 |     {
 24 |         using ScaleMode = typename PolicySelect<VarScaleFillerPolicy, TPolicyCont>::ScaleMode;        
 25 |         double fan_factor = 0;
 26 |         if constexpr (std::is_same<ScaleMode, VarScaleFillerPolicy::ScaleModeTypeCate::FanIn>::value)
 27 |         {
 28 |             fan_factor = fanIn;
 29 |         }
 30 |         else if constexpr (std::is_same<ScaleMode, VarScaleFillerPolicy::ScaleModeTypeCate::FanOut>::value)
 31 |         {
 32 |             fan_factor = fanOut;
 33 |         }
 34 |         else if constexpr (std::is_same<ScaleMode, VarScaleFillerPolicy::ScaleModeTypeCate::FanAvg>::value)
 35 |         {
 36 |             fan_factor = (fanIn + fanOut) / 2;
 37 |         }
 38 |         else
 39 |         {
 40 |             static_assert(DependencyFalse<ScaleMode>);
 41 |         }
 42 |         
 43 |         using DistType = typename PolicySelect<VarScaleFillerPolicy, TPolicyCont>::Distribute;                                            
 44 |         using ElementType = typename TData::ElementType;
 45 |         if constexpr (std::is_same<DistType, VarScaleFillerPolicy::DistributeTypeCate::Uniform>::value)
 46 |         {
 47 |             double limit = sqrt(3.0 * m_factor / fan_factor);
 48 |             std::uniform_real_distribution<ElementType> dist(-limit, limit);
 49 |             NSInitializer::FillWithDist(data, dist, m_engine);
 50 |         }
 51 |         else if constexpr (std::is_same<DistType, VarScaleFillerPolicy::DistributeTypeCate::Norm>::value)
 52 |         {
 53 |             double stddev = sqrt(m_factor / fan_factor);
 54 |             std::normal_distribution<ElementType> dist(0, stddev);
 55 |             NSInitializer::FillWithDist(data, dist, m_engine);
 56 |         }
 57 |         else
 58 |         {
 59 |             static_assert(DependencyFalse<DistType>);
 60 |         }
 61 |     }
 62 |     
 63 | private:
 64 |     double m_factor; 
 65 |     TRandomEngine m_engine;
 66 | };
 67 | 
 68 | 
 69 | namespace NSVarScaleFiller
 70 | {
 71 | template <typename TPolicyCont>
 72 | struct MSRAFillerPolicy_
 73 | {
 74 |     using type1 = ChangePolicy<PNormVarScale, TPolicyCont>;
 75 |     using type = ChangePolicy<PVarScaleFanIn, type1>;
 76 | };
 77 | 
 78 | template <typename TPolicyCont>
 79 | using MSRAFillerPolicy = typename MSRAFillerPolicy_<TPolicyCont>::type;
 80 | }
 81 | 
 82 | // MSRA Filler, use Norm Dist and FanIn Mode
 83 | template <typename TPolicyCont = PolicyContainer<>>
 84 | class MSRAFiller : public VarScaleFiller<NSVarScaleFiller::MSRAFillerPolicy<TPolicyCont>>
 85 | {
 86 |     using BaseType = VarScaleFiller<NSVarScaleFiller::MSRAFillerPolicy<TPolicyCont>>;
 87 | public:
 88 |     MSRAFiller(unsigned seed = std::random_device{}())
 89 |         : BaseType(2, seed) {}
 90 | };
 91 | 
 92 | // Xavier Filler, use FanAvg Mode
 93 | template <typename TPolicyCont = PolicyContainer<>>
 94 | class XavierFiller : public VarScaleFiller<ChangePolicy<PVarScaleFanAvg, TPolicyCont>>
 95 | {
 96 |     using BaseType = VarScaleFiller<ChangePolicy<PVarScaleFanAvg, TPolicyCont>>;
 97 | public:
 98 |     XavierFiller(unsigned seed = std::random_device{}())
 99 |         : BaseType(1, seed) {}
100 | };
101 | }


--------------------------------------------------------------------------------
/Test/layers/elementary/test_softmax_layer.cpp:
--------------------------------------------------------------------------------
  1 | #include <MetaNN/meta_nn.h>
  2 | #include "../../facilities/data_gen.h"
  3 | #include <cassert>
  4 | #include <iostream>
  5 | using namespace MetaNN;
  6 | using namespace std;
  7 | 
  8 | namespace
  9 | {
 10 | void test_softmax_layer1()
 11 | {
 12 |     cout << "Test softmax layer case 1 ...\t";
 13 |     using RootLayer = InjectPolicy<SoftmaxLayer, PFeedbackOutput>;
 14 |     static_assert(RootLayer::IsFeedbackOutput, "Test Error");
 15 |     static_assert(!RootLayer::IsUpdate, "Test Error");
 16 | 
 17 |     RootLayer layer;
 18 | 
 19 |     Matrix<float, DeviceTags::CPU> in(1, 2);
 20 |     in.SetValue(0, 0, -0.27f);
 21 |     in.SetValue(0, 1, -0.41f);
 22 | 
 23 |     auto input = LayerIO::Create().Set<LayerIO>(in);
 24 | 
 25 |     LayerNeutralInvariant(layer);
 26 | 
 27 |     auto out = layer.FeedForward(input);
 28 |     auto check = VecSoftmax(in);
 29 | 
 30 |     auto handle1 = out.Get<LayerIO>().EvalRegister();
 31 |     auto handle2 = check.EvalRegister();
 32 |     EvalPlan<DeviceTags::CPU>::Eval();
 33 | 
 34 |     auto res = handle1.Data();
 35 |     auto c = handle2.Data();
 36 | 
 37 |     assert(fabs(res(0, 0) - c(0, 0)) < 0.001);
 38 |     assert(fabs(res(0, 1) - c(0, 1)) < 0.001);
 39 | 
 40 |     Matrix<float, DeviceTags::CPU> grad(1, 2);
 41 |     grad.SetValue(0, 0, 0.1f);
 42 |     grad.SetValue(0, 1, 0.3f);
 43 | 
 44 |     auto out_grad = layer.FeedBackward(LayerIO::Create().Set<LayerIO>(grad));
 45 |     auto fb = Evaluate(out_grad.Get<LayerIO>());
 46 | 
 47 |     c = Evaluate(VecSoftmaxDerivative(grad, c));
 48 |     assert(fabs(fb(0, 0) - c(0, 0)) < 0.001);
 49 |     assert(fabs(fb(0, 1) - c(0, 1)) < 0.001);
 50 | 
 51 |     LayerNeutralInvariant(layer);
 52 | 
 53 |     cout << "done" << endl;
 54 | }
 55 | 
 56 | void test_softmax_layer2()
 57 | {
 58 |     cout << "Test softmax layer case 2 ...\t";
 59 |     using RootLayer = InjectPolicy<SoftmaxLayer, PFeedbackOutput>;
 60 |     static_assert(RootLayer::IsFeedbackOutput, "Test Error");
 61 |     static_assert(!RootLayer::IsUpdate, "Test Error");
 62 | 
 63 |     RootLayer layer;
 64 | 
 65 |     vector<Matrix<float, DeviceTags::CPU>> op;
 66 | 
 67 |     LayerNeutralInvariant(layer);
 68 |     for (size_t loop_count = 1; loop_count < 10; ++loop_count)
 69 |     {
 70 |         auto in = GenMatrix<float>(1, loop_count, 0.1f, 0.13f);
 71 | 
 72 |         auto input = LayerIO::Create().Set<LayerIO>(in);
 73 | 
 74 |         auto out = layer.FeedForward(input);
 75 |         auto check = VecSoftmax(in);
 76 | 
 77 |         auto handle1 = out.Get<LayerIO>().EvalRegister();
 78 |         auto handle2 = check.EvalRegister();
 79 |         EvalPlan<DeviceTags::CPU>::Eval();
 80 | 
 81 |         auto res = handle1.Data();
 82 |         auto c = handle2.Data();
 83 | 
 84 |         op.push_back(c);
 85 |         for (size_t i = 0; i < loop_count; ++i)
 86 |         {
 87 |             assert(fabs(res(0, i) - c(0, i)) < 0.0001);
 88 |         }
 89 |     }
 90 | 
 91 |     for (size_t loop_count = 9; loop_count >= 1; --loop_count)
 92 |     {
 93 |         auto grad = GenMatrix<float>(1, loop_count, 1.3f, 1.1f);
 94 |         auto out_grad = layer.FeedBackward(LayerIO::Create().Set<LayerIO>(grad));
 95 |         auto check = VecSoftmaxDerivative(grad, op.back());
 96 | 
 97 |         auto handle1 = out_grad.Get<LayerIO>().EvalRegister();
 98 |         auto handle2 = check.EvalRegister();
 99 |         EvalPlan<DeviceTags::CPU>::Eval();
100 | 
101 |         auto fb = handle1.Data();
102 |         auto c = handle2.Data();
103 |         op.pop_back();
104 | 
105 |         for (size_t i = 0; i < loop_count; ++i)
106 |         {
107 |             assert(fabs(fb(0, i) - c(0, i)) < 0.0001);
108 |         }
109 |     }
110 | 
111 |     LayerNeutralInvariant(layer);
112 | 
113 |     cout << "done" << endl;
114 | }
115 | }
116 | 
117 | void test_softmax_layer()
118 | {
119 |     test_softmax_layer1();
120 |     test_softmax_layer2();
121 | }
122 | 


--------------------------------------------------------------------------------
/Test/main.cpp:
--------------------------------------------------------------------------------
  1 | #include "policies/test_change_policy.h"
  2 | #include "policies/test_policy_operations.h"
  3 | #include "policies/test_policy_selector.h"
  4 | #include "facilities/test_var_type_dict.h"
  5 | #include "evaluate/test_eval_plan.h"
  6 | #include "data/test_array.h"
  7 | #include "data/test_duplicate.h"
  8 | #include "data/test_scalar.h"
  9 | #include "data/test_general_matrix.h"
 10 | #include "data/test_one_hot_vector.h"
 11 | #include "data/test_trival_matrix.h"
 12 | #include "data/test_zero_matrix.h"
 13 | #include "data/test_batch_scalar.h"
 14 | #include "data/test_batch_matrix.h"
 15 | #include "operators/test_abs.h"
 16 | #include "operators/test_add.h"
 17 | #include "operators/test_collapse.h"
 18 | #include "operators/test_divide.h"
 19 | #include "operators/test_dot.h"
 20 | #include "operators/test_element_mul.h"
 21 | #include "operators/test_interpolate.h"
 22 | #include "operators/test_negative_log_likelihood.h"
 23 | #include "operators/test_negative_log_likelihood_derivative.h"
 24 | #include "operators/test_sigmoid.h"
 25 | #include "operators/test_sigmoid_derivative.h"
 26 | #include "operators/test_sign.h"
 27 | #include "operators/test_softmax.h"
 28 | #include "operators/test_softmax_derivative.h"
 29 | #include "operators/test_substract.h"
 30 | #include "operators/test_tanh.h"
 31 | #include "operators/test_tanh_derivative.h"
 32 | #include "operators/test_transpose.h"
 33 | #include "layers/elementary/test_abs_layer.h"
 34 | #include "layers/elementary/test_add_layer.h"
 35 | #include "layers/elementary/test_bias_layer.h"
 36 | #include "layers/elementary/test_element_mul_layer.h"
 37 | #include "layers/elementary/test_interpolate_layer.h"
 38 | #include "layers/elementary/test_sigmoid_layer.h"
 39 | #include "layers/elementary/test_softmax_layer.h"
 40 | #include "layers/elementary/test_tanh_layer.h"
 41 | #include "layers/elementary/test_weight_layer.h"
 42 | #include "layers/cost/test_negative_log_likelihood_layer.h"
 43 | #include "layers/compose/test_compose_kernel.h"
 44 | #include "layers/compose/test_linear_layer.h"
 45 | #include "layers/compose/test_single_layer.h"
 46 | #include "layers/recurrent/test_gru.h"
 47 | #include "layers/recurrent/test_gru_2.h"
 48 | #include "model_rel/param_initializer/test_constant_filler.h"
 49 | #include "model_rel/param_initializer/test_gaussian_filler.h"
 50 | #include "model_rel/param_initializer/test_var_scale_filter.h"
 51 | 
 52 | int main(int argc, char **argv)
 53 | {
 54 |     test_change_policy();
 55 |     test_policy_operations();
 56 |     test_policy_selector();
 57 | 
 58 |     test_var_type_dict();
 59 |     test_eval_plan();
 60 | 
 61 | 	test_scalar();
 62 |     test_general_matrix();
 63 |     test_one_hot_vector();
 64 |     test_trival_matrix();
 65 |     test_zero_matrix();
 66 |     test_array();
 67 |     test_duplicate();
 68 |     test_batch_scalar();
 69 |     test_batch_matrix();
 70 | 
 71 |     test_abs();
 72 |     test_add();
 73 |     test_collapse();
 74 |     test_divide();
 75 |     test_dot();
 76 |     test_element_mul();
 77 |     test_interpolate();
 78 |     test_negative_log_likelihood();
 79 |     test_negative_log_likelihood_derivative();
 80 |     test_sigmoid();
 81 |     test_sigmoid_derivative();
 82 |     test_sign();
 83 |     test_softmax();
 84 |     test_softmax_derivative();
 85 |     test_substract();
 86 |     test_tanh();
 87 |     test_tanh_derivative();
 88 |     test_transpose();
 89 | 
 90 |     test_abs_layer();
 91 |     test_add_layer();
 92 |     test_bias_layer();
 93 |     test_element_mul_layer();
 94 |     test_interpolate_layer();
 95 |     test_sigmoid_layer();
 96 |     test_softmax_layer();
 97 |     test_tanh_layer();
 98 |     test_weight_layer();
 99 | 
100 |     test_negative_log_likelihood_layer();
101 | 
102 |     test_compose_kernel();
103 |     test_linear_layer();
104 |     test_single_layer();
105 | 
106 |     test_gru();
107 |     test_gru_2();
108 | 
109 |     test_constant_filler();
110 |     test_gaussian_filler();
111 |     test_var_scale_filter();
112 | 	return 0;
113 | }
114 | 
115 | 


--------------------------------------------------------------------------------
/code_index.txt:
--------------------------------------------------------------------------------
  1 | 《C++模板元编程实战》一书分若干章节对本分支的代码进行了讨论。这些代码组合起来形成了 MetaNN 的框架，具有一定的整体性——很难按照书中所讨论的顺序划分章节进行组织。而这也就给需要对照代码来阅读本书的读者造成了一些困难：读者不得不通过搜索书中出现的关键字来定位所讨论的内容在源代码中的相应位置。为了解决这个问题，我在这里引入了这篇文档，描述了书中每一章所讨论的主要内容与源代码之间的对应关系。
  2 | 
  3 | 源代码包含了两个目录： MetaNN 与 Test ，前者包含了框架的全部逻辑；而后者则是基于 MetaNN 中的逻辑所编写的测试代码。这个文档会同时说明框架逻辑与测试代码的位置，供读者参考。
  4 | 
  5 | 第一章：基本技巧
  6 |   这一章所讨论的内容相对基本、零散，没有对应的源码。
  7 | 
  8 | 第二章：异类词典与 Policy 模板
  9 |   异类词典
 10 |     框架逻辑：
 11 |       MetaNN/facilities/var_type_dict.h
 12 |       MetaNN/facilities/null_param.h
 13 |     测试代码：
 14 |       Test/facilities/test_var_type_dict.cpp
 15 |   Policy 模板
 16 |     框架逻辑：
 17 |       MetaNN/policies/policy_container.h
 18 |       MetaNN/policies/policy_macro_begin.h
 19 |       MetaNN/policies/policy_macro_end.h
 20 |       MetaNN/policies/policy_selector.h
 21 |     测试代码：
 22 |       Test/policies/test_policy_selector.cpp
 23 | 
 24 | 第三章：深度学习概述
 25 |   这一章讨论深度学习的基本概念，不涉及代码。
 26 | 
 27 | 第四章：类型体系与基本数据类型
 28 |   MetaNN 的类型体系相关元函数
 29 |     框架逻辑：
 30 |       MetaNN/data/facilities/tags.h
 31 |       MetaNN/data/facilities/traits.h
 32 |   空间的分配与维护
 33 |     框架逻辑：
 34 |       MetaNN/data/facilities/allocators.h
 35 |       MetaNN/data/facilities/continuous_memory.h
 36 |   标量
 37 |     框架逻辑：
 38 |       MetaNN/data/scalar.h
 39 |     测试代码：
 40 |       Test/data/test_scalar.cpp
 41 |   矩阵
 42 |     框架逻辑：
 43 |       MetaNN/data/matrices （目录）
 44 |     测试代码：
 45 |       Test/data/test_general_matrix.cpp
 46 |       Test/data/test_one_hot_vector.cpp
 47 |       Test/data/test_trival_matrix.cpp
 48 |       Test/data/test_zero_matrix.cpp
 49 |   列表
 50 |     框架逻辑：
 51 |       MetaNN/data/batch （目录）
 52 |     测试代码：
 53 |       Test/data/test_array.cpp
 54 |       Test/data/test_batch_matrix.cpp
 55 |       Test/data/test_batch_scalar.cpp
 56 |       Test/data/test_duplicate.cpp
 57 | 
 58 | 第五章：运算与表达式模板
 59 |   运算分类
 60 |     框架逻辑：
 61 |       MetaNN/operators/facilities/tags.h
 62 |   辅助模板
 63 |     框架逻辑：
 64 |       MetaNN/operators/facilities/category_cal.h
 65 |       MetaNN/operators/facilities/oper_seq.h
 66 |       MetaNN/operators/facilities/organizer.h
 67 |       MetaNN/operators/facilities/traits.h
 68 |   运算模板框架
 69 |     框架逻辑：
 70 |       MetaNN/operators/operators.h
 71 |   运算模板实现示例
 72 |     框架逻辑：
 73 |       MetaNN/operators	（目录，其中的每个文件对应了一个运算）
 74 |     测试代码：
 75 |       Test/operators	（目录，其中的每个 cpp 文件对应了一个运算的测试）
 76 | 
 77 | 第六章：基本层
 78 |   初始化模块
 79 |     框架逻辑：
 80 |       MetaNN/model_rel/param_initializer	（目录）
 81 |     测试代码：
 82 |       Test/model_rel/param_initializer	（目录）
 83 |   DynamicData 类模板
 84 |     框架逻辑：
 85 |       MetaNN/data/dynamic.h
 86 |   常用 Policy 对象
 87 |     框架逻辑：
 88 |       MetaNN/layers/facilities/policies.h
 89 |   InjectPolicy 元函数
 90 |     框架逻辑：
 91 |       MetaNN/policies/inject_policy.h
 92 |   通用 IO 结构
 93 |     框架逻辑：
 94 |       MetaNN/layers/facilities/common_io.h
 95 |   通用操作函数
 96 |     框架逻辑：
 97 |       MetaNN/layers/facilities/interface_fun.h
 98 |   层的具体实现
 99 |     框架逻辑：
100 |       MetaNN/layers/elementary	（目录）
101 |     测试代码：
102 |       Test/layers/elementary	（目录）
103 | 
104 | 第七章：复合层与循环层
105 |   Policy 的继承与修正逻辑
106 |     框架逻辑：
107 |       MetaNN/policies/change_policy.h
108 |       MetaNN/policies/policy_operations.h
109 |     测试代码：
110 |       Test/policies/test_change_policy.cpp
111 |       Test/policies/test_policy_operations.cpp
112 |   ComposeTopology & ComposeKernel
113 |     框架逻辑：
114 |       MetaNN/layers/compose/compose_kernel.h
115 |     测试代码：
116 |       Test/layers/test_compose_kernel.cpp
117 |   复合层实现示例
118 |     框架逻辑：
119 |       MetaNN/layers/compose/linear_layer.h
120 |       MetaNN/layers/compose/single_layer.h
121 |     测试代码：
122 |       Test/layers/compose/test_linear_layer.cpp
123 |       Test/layers/compose/test_single_layer.cpp
124 |   循环层
125 |     框架逻辑：
126 |       MetaNN/layers/recurrent	（目录）
127 |     测试代码：
128 |       Test/layers/recurrent	（目录）
129 | 
130 | 第八章：求值与优化
131 |   求值子系统
132 |     框架逻辑：
133 |       MetaNN/evaluate	（目录）
134 |     测试代码：
135 |       Test/evaluate/test_eval_plan.cpp
136 |   具体求值实现与优化
137 |     不同的数据、运算实现了不同的求值逻辑，请参考相应的目录以获取具体的求值逻辑与测试代码：
138 |     框架逻辑：
139 |       MetaNN/data	（目录）
140 |       MetaNN/operators	（目录）
141 |     测试代码：
142 |       Test/data	（目录）
143 |       Test/operators	（目录）
144 | 


--------------------------------------------------------------------------------
/MetaNN/data/matrices/cpu_matrix.h:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include <MetaNN/data/batch/batch.h>
  4 | #include <MetaNN/data/facilities/continuous_memory.h>
  5 | #include <MetaNN/data/facilities/lower_access.h>
  6 | #include <MetaNN/data/matrices/matrices.h>
  7 | #include <MetaNN/data/scalar.h>
  8 | #include <MetaNN/evaluate/facilities/eval_handle.h>
  9 | #include <cassert>
 10 | #include <cstring>
 11 | #include <type_traits>
 12 | 
 13 | namespace MetaNN
 14 | {
 15 | template <typename TElem>
 16 | struct LowerAccessImpl<Matrix<TElem, DeviceTags::CPU>>;
 17 | 
 18 | template <typename TElem>
 19 | class Matrix<TElem, DeviceTags::CPU>
 20 | {    
 21 | public:
 22 |     static_assert(std::is_same<RemConstRef<TElem>, TElem>::value,
 23 |                   "TElem is not an available type");
 24 |                   
 25 |     using ElementType = TElem;
 26 |     using DeviceType = DeviceTags::CPU;
 27 |     
 28 |     friend struct LowerAccessImpl<Matrix<TElem, DeviceTags::CPU>>;
 29 |     friend class Batch<TElem, DeviceTags::CPU, CategoryTags::Matrix>;
 30 | 
 31 | public:
 32 |     Matrix(size_t p_rowNum = 0, size_t p_colNum = 0)
 33 |         : m_mem(p_rowNum * p_colNum)
 34 |         , m_rowNum(p_rowNum)
 35 |         , m_colNum(p_colNum)
 36 |         , m_rowLen(p_colNum)
 37 |     {}
 38 | 
 39 |     bool operator== (const Matrix& val) const
 40 |     {
 41 |         return (m_mem == val.m_mem) &&
 42 |                (m_rowNum == val.m_rowNum) &&
 43 |                (m_colNum == val.m_colNum) &&
 44 |                (m_rowLen == val.m_rowLen);
 45 |     }
 46 | 
 47 |     template <typename TOtherType>
 48 |     bool operator== (const TOtherType&) const
 49 |     {
 50 |         return false;
 51 |     }
 52 | 
 53 |     template <typename TData>
 54 |     bool operator!= (const TData& val) const
 55 |     {
 56 |         return !(operator==(val));
 57 |     }
 58 | 
 59 |     size_t RowNum() const { return m_rowNum; }
 60 |     size_t ColNum() const { return m_colNum; }
 61 | 
 62 |     bool AvailableForWrite() const { return m_mem.UseCount() == 1; }
 63 | 
 64 |     void SetValue(size_t p_rowId, size_t p_colId, ElementType val)
 65 |     {
 66 |         assert(AvailableForWrite());
 67 |         assert((p_rowId < m_rowNum) && (p_colId < m_colNum));
 68 |         (m_mem.RawMemory())[p_rowId * m_rowLen + p_colId] = val;
 69 |     }
 70 | 
 71 |     const auto operator () (size_t p_rowId, size_t p_colId) const
 72 |     {
 73 |         assert((p_rowId < m_rowNum) && (p_colId < m_colNum));
 74 |         return (m_mem.RawMemory())[p_rowId * m_rowLen + p_colId];
 75 |     }
 76 | 
 77 |     Matrix SubMatrix(size_t p_rowB, size_t p_rowE, size_t p_colB, size_t p_colE) const
 78 |     {
 79 |         assert((p_rowB < m_rowNum) && (p_colB < m_colNum));
 80 |         assert((p_rowE <= m_rowNum) && (p_colE <= m_colNum));
 81 |         auto pos = m_mem.RawMemory() + p_rowB * m_rowLen + p_colB;
 82 |         return Matrix(m_mem.SharedPtr(), pos,
 83 |                       p_rowE - p_rowB, p_colE - p_colB,
 84 |                       m_rowLen);
 85 |     }
 86 | 
 87 |     auto EvalRegister() const
 88 |     {
 89 |         return MakeConstEvalHandle(*this);
 90 |     }
 91 |     
 92 | private:
 93 |     Matrix(std::shared_ptr<ElementType> p_mem,
 94 |             ElementType* p_memStart,
 95 |             size_t p_rowNum,
 96 |             size_t p_colNum,
 97 |             size_t p_rowLen)
 98 |         : m_mem(p_mem, p_memStart)
 99 |         , m_rowNum(p_rowNum)
100 |         , m_colNum(p_colNum)
101 |         , m_rowLen(p_rowLen)
102 |     {}
103 | 
104 | private:
105 |     ContinuousMemory<ElementType, DeviceType> m_mem;
106 |     size_t m_rowNum;
107 |     size_t m_colNum;
108 |     size_t m_rowLen;
109 | };
110 | 
111 | template<typename TElem>
112 | struct LowerAccessImpl<Matrix<TElem, DeviceTags::CPU>>
113 | {
114 |     LowerAccessImpl(Matrix<TElem, DeviceTags::CPU> p)
115 |         : m_matrix(std::move(p))
116 |     {}
117 | 
118 |     auto MutableRawMemory()
119 |     {
120 |         return m_matrix.m_mem.RawMemory();
121 |     }
122 | 
123 |     const auto RawMemory() const
124 |     {
125 |         return m_matrix.m_mem.RawMemory();
126 |     }
127 | 
128 |     size_t RowLen() const
129 |     {
130 |         return m_matrix.m_rowLen;
131 |     }
132 | 
133 | private:
134 |     Matrix<TElem, DeviceTags::CPU> m_matrix;
135 | };
136 | }
137 | 


--------------------------------------------------------------------------------
/Test/data/test_array.cpp:
--------------------------------------------------------------------------------
  1 | #include "test_general_matrix.h"
  2 | #include "../facilities/calculate_tags.h"
  3 | #include <iostream>
  4 | #include <cassert>
  5 | #include <set>
  6 | #include <MetaNN/meta_nn.h>
  7 | using namespace std;
  8 | using namespace MetaNN;
  9 | 
 10 | namespace
 11 | {
 12 | void TestArray1()
 13 | {
 14 |     cout << "Test array case 1 (matrix array)...\t";
 15 |     static_assert(IsBatchMatrix<Array<Matrix<CheckElement, CheckDevice>>>, "Test Error");
 16 |     static_assert(IsBatchMatrix<Array<Matrix<CheckElement, CheckDevice>> &>, "Test Error");
 17 |     static_assert(IsBatchMatrix<Array<Matrix<CheckElement, CheckDevice>> &&>, "Test Error");
 18 |     static_assert(IsBatchMatrix<const Array<Matrix<CheckElement, CheckDevice>> &>, "Test Error");
 19 |     static_assert(IsBatchMatrix<const Array<Matrix<CheckElement, CheckDevice>> &&>, "Test Error");
 20 | 
 21 |     auto rm1 = Array<Matrix<CheckElement, CheckDevice>>(10, 20);
 22 |     assert(rm1.BatchNum() == 0);
 23 |     assert(rm1.empty());
 24 | 
 25 |     int c = 0;
 26 |     auto me1 = Matrix<CheckElement, CheckDevice>(10, 20);
 27 |     auto me2 = Matrix<CheckElement, CheckDevice>(10, 20);
 28 |     auto me3 = Matrix<CheckElement, CheckDevice>(10, 20);
 29 |     for (size_t i = 0; i < 10; ++i)
 30 |     {
 31 |         for (size_t j = 0; j < 20; ++j)
 32 |         {
 33 |             me1.SetValue(i, j, (float)(c++));
 34 |             me2.SetValue(i, j, (float)(c++));
 35 |             me3.SetValue(i, j, (float)(c++));
 36 |         }
 37 |     }
 38 |     rm1.push_back(me1);
 39 |     rm1.push_back(me2);
 40 |     rm1.push_back(me3);
 41 |     assert(rm1.BatchNum() == 3);
 42 |     assert(!rm1.empty());
 43 | 
 44 |     auto evalHandle = rm1.EvalRegister();
 45 |     EvalPlan<DeviceTags::CPU>::Eval();
 46 |     auto rm2 = evalHandle.Data();
 47 |     
 48 |     for (size_t i = 0; i < 10; ++i)
 49 |     {
 50 |         for (size_t j = 0; j < 20; ++j)
 51 |         {
 52 |             assert(rm1[0](i, j) == me1(i, j));
 53 |             assert(rm1[1](i, j) == me2(i, j));
 54 |             assert(rm1[2](i, j) == me3(i, j));
 55 |         }
 56 |     }
 57 |     cout << "done" << endl;
 58 | }
 59 | 
 60 | void TestArray2()
 61 | {
 62 |     cout << "Test array case 2 (scalar array)...\t";
 63 |     static_assert(IsBatchScalar<Array<Scalar<CheckElement, CheckDevice>>>, "Test Error");
 64 |     static_assert(IsBatchScalar<Array<Scalar<CheckElement, CheckDevice>> &>, "Test Error");
 65 |     static_assert(IsBatchScalar<Array<Scalar<CheckElement, CheckDevice>> &&>, "Test Error");
 66 |     static_assert(IsBatchScalar<const Array<Scalar<CheckElement, CheckDevice>> &>, "Test Error");
 67 |     static_assert(IsBatchScalar<const Array<Scalar<CheckElement, CheckDevice>> &&>, "Test Error");
 68 | 
 69 |     auto rm1 = Array<Scalar<CheckElement, CheckDevice>>();
 70 |     assert(rm1.BatchNum() == 0);
 71 |     assert(rm1.empty());
 72 | 
 73 |     rm1.push_back(3);
 74 |     rm1.push_back(8);
 75 |     rm1.push_back(2);
 76 |     assert(rm1.BatchNum() == 3);
 77 |     assert(!rm1.empty());
 78 | 
 79 |     auto evalHandle = rm1.EvalRegister();
 80 |     EvalPlan<DeviceTags::CPU>::Eval();
 81 |     auto rm2 = evalHandle.Data();
 82 |     
 83 |     assert(rm2[0] == 3);
 84 |     assert(rm2[1] == 8);
 85 |     assert(rm2[2] == 2);
 86 |     cout << "done" << endl;
 87 | }
 88 | 
 89 | void TestArray3()
 90 | {
 91 |     cout << "Test array case 3 ...\t";
 92 | 
 93 |     {
 94 |         vector<Matrix<CheckElement, CheckDevice>> check;
 95 |         check.emplace_back(10, 16);
 96 |         check.emplace_back(10, 16);
 97 |         check.emplace_back(10, 16);
 98 |     
 99 |         auto tmp = MakeArray(check.begin(), check.end());
100 |         assert(tmp.RowNum() == 10);
101 |         assert(tmp.ColNum() == 16);
102 |         assert(tmp.BatchNum() == 3);
103 |     }
104 |     {
105 |         vector<Scalar<int, CheckDevice>> check{3, 5, 8, 6};    
106 |         auto tmp = MakeArray(check.begin(), check.end());
107 |         assert(tmp.BatchNum() == 4);
108 |         assert(tmp[0].Value() == 3);
109 |         assert(tmp[1].Value() == 5);
110 |         assert(tmp[2].Value() == 8);
111 |         assert(tmp[3].Value() == 6);
112 |     }
113 |     cout << "done" << endl;
114 | }
115 | }
116 | 
117 | void test_array()
118 | {
119 |     TestArray1();
120 |     TestArray2();
121 |     TestArray3();
122 | }


--------------------------------------------------------------------------------
/MetaNN/layers/cost/negative_log_likelihood_layer.h:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | #include <MetaNN/layers/facilities/common_io.h>
  3 | #include <MetaNN/layers/facilities/policies.h>
  4 | #include <MetaNN/policies/policy_operations.h>
  5 | 
  6 | namespace MetaNN
  7 | {
  8 | namespace NSNegativeLogLikelihoodLayer
  9 | {
 10 | template <bool isFeedback>
 11 | struct Feedback_
 12 | {
 13 |     template <typename TLabel, typename TIn, typename TData>
 14 |     static void Record(const TLabel& label, const TIn& in,
 15 |                        TData& label_stack, TData& pred_stack)
 16 |     {
 17 |         label_stack.push(MakeDynamic(label));
 18 |         pred_stack.push(MakeDynamic(in));
 19 |     }
 20 | 
 21 |     template <typename TGrad, typename TData>
 22 |     static auto Feedback(const TGrad& p_grad, TData& label, TData& pred)
 23 |     {
 24 |         if ((label.empty()) || (pred.empty()))
 25 |         {
 26 |             throw std::runtime_error("Cannot do FeedBackward for Negative Log-likelihood Layer");
 27 |         }
 28 |         auto l = label.top();
 29 |         auto p = pred.top();
 30 |         label.pop();
 31 |         pred.pop();
 32 | 
 33 |         auto g = p_grad.template Get<LayerIO>();
 34 |         auto res = NegativeLogLikelihoodDerivative(g, std::move(l), std::move(p));
 35 |         return CostLayerIn::Create().template Set<CostLayerIn>(std::move(res));
 36 |     }
 37 | };
 38 | 
 39 | template <>
 40 | struct Feedback_<false>
 41 | {
 42 |     template <typename TLabel, typename TIn, typename TData>
 43 |     static void Record(TLabel&&, TIn&& p_in, TData&&, TData&&) { }
 44 | 
 45 |     template <typename TGrad, typename TData>
 46 |     static auto Feedback(TGrad&&, TData&&, TData&&)
 47 |     {
 48 |         return CostLayerIn::Create();
 49 |     }
 50 | };
 51 | }
 52 | 
 53 | template <typename TPolicies>
 54 | class NegativeLogLikelihoodLayer
 55 | {
 56 |     static_assert(IsPolicyContainer<TPolicies>, "TPolicies is not a policy container.");
 57 |     using CurLayerPolicy = PlainPolicy<TPolicies>;
 58 | 
 59 | public:
 60 |     static constexpr bool IsFeedbackOutput = PolicySelect<FeedbackPolicy, CurLayerPolicy>::IsFeedbackOutput;
 61 |     static constexpr bool IsUpdate = false;
 62 |     using InputType = CostLayerIn;
 63 |     using OutputType = LayerIO;
 64 | 
 65 | private:
 66 |     using ElementType = typename PolicySelect<OperandPolicy, CurLayerPolicy>::Element;
 67 |     using DeviceType = typename PolicySelect<OperandPolicy, CurLayerPolicy>::Device;
 68 | 
 69 |     using Feedback_ = NSNegativeLogLikelihoodLayer::Feedback_<IsFeedbackOutput>;
 70 | public:
 71 |     template <typename TIn>
 72 |     auto FeedForward(const TIn& p_in)
 73 |     {
 74 |         const auto& input = p_in.template Get<CostLayerIn>();
 75 |         const auto& label = p_in.template Get<CostLayerLabel>();
 76 | 
 77 |         using rawType1 = std::decay_t<decltype(input)>;
 78 |         using rawType2 = std::decay_t<decltype(label)>;
 79 |         static_assert(!std::is_same<rawType1, NullParameter>::value, "Input is invalid");
 80 |         static_assert(!std::is_same<rawType2, NullParameter>::value, "Label is invalid");
 81 | 
 82 |         Feedback_::Record(label, input, m_label, m_pred);
 83 |         return LayerIO::Create().template Set<LayerIO>(NegativeLogLikelihood(label, input));
 84 |     }
 85 | 
 86 |     template <typename TGrad>
 87 |     auto FeedBackward(TGrad&& p_grad)
 88 |     {
 89 |         return Feedback_::Feedback(std::forward<TGrad>(p_grad), m_label, m_pred);
 90 |     }
 91 | 
 92 |     void NeutralInvariant()
 93 |     {
 94 |         if constexpr(IsFeedbackOutput)
 95 |         {
 96 |             if ((!m_label.empty()) || (!m_pred.empty()))
 97 |             {
 98 |                 throw std::runtime_error("NeutralInvariant Fail!");
 99 |             }
100 |         }
101 |     }
102 | 
103 | private:
104 |     using DataType = LayerTraits::LayerInternalBuf<IsFeedbackOutput,
105 |                                                    PolicySelect<InputPolicy, CurLayerPolicy>::BatchMode,
106 |                                                    typename PolicySelect<OperandPolicy, CurLayerPolicy>::Element,
107 |                                                    typename PolicySelect<OperandPolicy, CurLayerPolicy>::Device,
108 |                                                    CategoryTags::Matrix, CategoryTags::BatchMatrix>;
109 |     DataType m_label;
110 |     DataType m_pred;
111 | };
112 | }
113 | 


--------------------------------------------------------------------------------
/MetaNN/model_rel/grad_col/grad_collector.h:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | #include <unordered_map>
  3 | 
  4 | namespace MetaNN
  5 | {
  6 | template <typename TElement, typename TDevice>
  7 | struct MatrixGradInfo
  8 | {
  9 |     using GradItemType = DynamicData<TElement, TDevice, CategoryTags::Matrix>;
 10 | 
 11 |     MatrixGradInfo(Matrix<TElement, TDevice> p_weight)
 12 |         : weight(std::move(p_weight))
 13 |         , grad(p_weight.RowNum(), p_weight.ColNum()) {}
 14 | 
 15 |     Matrix<TElement, TDevice> weight;
 16 |     Array<GradItemType> grad;
 17 | };
 18 | 
 19 | template <typename TElement, typename TDevice>
 20 | class GradCollectorIterator
 21 | {
 22 |     using IteratorType = typename std::unordered_map<const TElement*, MatrixGradInfo<TElement, TDevice>>::iterator;
 23 | 
 24 | public:
 25 |     GradCollectorIterator(IteratorType it)
 26 |         : m_it(std::move(it)) {}
 27 | 
 28 |     const auto& operator* () const
 29 |     {
 30 |         return m_it->second;
 31 |     }
 32 | 
 33 |     const auto operator-> () const
 34 |     {
 35 |         return &(m_it->second);
 36 |     }
 37 | 
 38 |     auto operator++()
 39 |     {
 40 |         ++m_it;
 41 |         return *this;
 42 |     }
 43 | 
 44 |     auto operator++(int)
 45 |     {
 46 |         auto tmp = *this;
 47 |         ++m_it;
 48 |         return tmp;
 49 |     }
 50 | 
 51 |     bool operator== (const GradCollectorIterator& git) const
 52 |     {
 53 |         return m_it == git.m_it;
 54 |     }
 55 | 
 56 |     bool operator!= (const GradCollectorIterator& git) const
 57 |     {
 58 |         return !(operator==(git));
 59 |     }
 60 | 
 61 | private:
 62 |     IteratorType m_it;
 63 | };
 64 | 
 65 | template <typename TElement, typename TDevice>
 66 | class GradCollector
 67 | {
 68 | public:
 69 |     GradCollector() = default;
 70 |     GradCollector(const GradCollector&) = delete;
 71 |     GradCollector(GradCollector&&) = default;
 72 |     GradCollector& operator = (const GradCollector&) = delete;
 73 |     GradCollector& operator = (GradCollector&&) = delete;
 74 | 
 75 |     template<typename TGrad>
 76 |     void Collect(const Matrix<TElement, TDevice>& weight,
 77 |                  const TGrad& grad)
 78 |     {
 79 |         auto mem = LowerAccess(weight);
 80 |         auto buf = mem.RawMemory();
 81 | 
 82 |         auto it = m_matricesInfo.find(buf);
 83 | 
 84 |         if (it != m_matricesInfo.end())
 85 |         {
 86 |             if constexpr(IsMatrix<TGrad>)
 87 |             {
 88 |                 it->second.grad.push_back(MakeDynamic(grad));
 89 |             }
 90 |             else if constexpr (IsBatchMatrix<TGrad>)
 91 |             {
 92 |                 it->second.grad.push_back(MakeDynamic(Collapse(grad)));
 93 |             }
 94 |             else
 95 |             {
 96 |                 static_assert(DependencyFalse<TGrad>);
 97 |             }
 98 |         }
 99 |         else
100 |         {
101 |             MatrixGradInfo<TElement, TDevice> mgi(weight);
102 |             
103 |             if constexpr(IsMatrix<TGrad>)
104 |             {
105 |                 mgi.grad.push_back(MakeDynamic(grad));
106 |             }
107 |             else if constexpr (IsBatchMatrix<TGrad>)
108 |             {
109 |                 mgi.grad.push_back(MakeDynamic(Collapse(grad)));
110 |             }
111 |             else
112 |             {
113 |                 static_assert(DependencyFalse<TGrad>);
114 |             }
115 | 
116 |             m_matricesInfo.insert({buf, std::move(mgi)});
117 |         }
118 |     }
119 | 
120 |     void clear()
121 |     {
122 |         m_matricesInfo.clear();
123 |     }
124 | 
125 |     size_t size() const
126 |     {
127 |         return m_matricesInfo.size();
128 |     }
129 | 
130 |     auto begin()
131 |     {
132 |         return GradCollectorIterator<TElement, TDevice>(m_matricesInfo.begin());
133 |     }
134 | 
135 |     auto begin() const
136 |     {
137 |         return GradCollectorIterator<TElement, TDevice>(m_matricesInfo.begin());
138 |     }
139 | 
140 |     auto end()
141 |     {
142 |         return GradCollectorIterator<TElement, TDevice>(m_matricesInfo.end());
143 |     }
144 | 
145 |     auto end() const
146 |     {
147 |         return GradCollectorIterator<TElement, TDevice>(m_matricesInfo.end());
148 |     }
149 | 
150 | private:
151 |     std::unordered_map<const TElement*, MatrixGradInfo<TElement, TDevice>> m_matricesInfo;
152 | };
153 | }
154 | 


--------------------------------------------------------------------------------
/MetaNN/layers/elementary/interpolate_layer.h:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | #include <MetaNN/facilities/var_type_dict.h>
  3 | #include <MetaNN/layers/facilities/common_io.h>
  4 | #include <MetaNN/layers/facilities/policies.h>
  5 | #include <MetaNN/policies/policy_operations.h>
  6 | 
  7 | namespace MetaNN
  8 | {
  9 | using InterpolateLayerInput = VarTypeDict<struct InterpolateLayerWeight1,
 10 |                                           struct InterpolateLayerWeight2,
 11 |                                           struct InterpolateLayerLambda>;
 12 | 
 13 | template <typename TPolicies>
 14 | class InterpolateLayer
 15 | {
 16 |     static_assert(IsPolicyContainer<TPolicies>, "TPolicies is not a policy container.");
 17 |     using CurLayerPolicy = PlainPolicy<TPolicies>;
 18 | 
 19 | public:
 20 |     static constexpr bool IsFeedbackOutput = PolicySelect<FeedbackPolicy, CurLayerPolicy>::IsFeedbackOutput;
 21 |     static constexpr bool IsUpdate = false;
 22 |     using InputType = InterpolateLayerInput;
 23 |     using OutputType = LayerIO;
 24 | 
 25 | public:
 26 |     template <typename TIn>
 27 |     auto FeedForward(const TIn& p_in)
 28 |     {
 29 |         const auto& val1 = p_in.template Get<InterpolateLayerWeight1>();
 30 |         const auto& val2 = p_in.template Get<InterpolateLayerWeight2>();
 31 |         const auto& val_lambda = p_in.template Get<InterpolateLayerLambda>();
 32 | 
 33 |         using rawType1 = std::decay_t<decltype(val1)>;
 34 |         using rawType2 = std::decay_t<decltype(val2)>;
 35 |         using rawType3 = std::decay_t<decltype(val_lambda)>;
 36 | 
 37 |         static_assert(!std::is_same<rawType1, NullParameter>::value, "parameter1 is invalid");
 38 |         static_assert(!std::is_same<rawType2, NullParameter>::value, "parameter2 is invalid");
 39 |         static_assert(!std::is_same<rawType3, NullParameter>::value, "parameter lambda is invalid");
 40 | 
 41 |         if constexpr(IsFeedbackOutput)
 42 |         {
 43 |             m_weight1.push(MakeDynamic(val1));
 44 |             m_weight2.push(MakeDynamic(val2));
 45 |             m_weight_lambda.push(MakeDynamic(val_lambda));
 46 |         }
 47 |         return LayerIO::Create().template Set<LayerIO>(Interpolate(val1, val2, val_lambda));
 48 |     }
 49 | 
 50 |     template <typename TGrad>
 51 |     auto FeedBackward(const TGrad& p_grad)
 52 |     {
 53 |         if constexpr (IsFeedbackOutput)
 54 |         {
 55 |             if ((m_weight1.empty()) || (m_weight2.empty()) || (m_weight_lambda.empty()))
 56 |             {
 57 |                 throw std::runtime_error("Cannot do FeedBackward for InterpolateLayer");
 58 |             }
 59 |             auto tmp = p_grad.template Get<LayerIO>();
 60 |             auto res1 = m_weight_lambda.top() * tmp;
 61 |         
 62 |             auto res2 = (Scalar<int>(1) - m_weight_lambda.top()) * tmp;
 63 |             auto res_lambda = (m_weight1.top() - m_weight2.top()) * tmp;
 64 |             auto res = InterpolateLayerInput::Create().template Set<InterpolateLayerWeight1>(std::move(res1))
 65 |                                                 .template Set<InterpolateLayerWeight2>(std::move(res2))
 66 |                                                 .template Set<InterpolateLayerLambda>(std::move(res_lambda));
 67 |             m_weight_lambda.pop();
 68 |             m_weight1.pop();
 69 |             m_weight2.pop();
 70 |             return res;
 71 |         }
 72 |         else
 73 |         {
 74 |             return InterpolateLayerInput::Create();
 75 |         }
 76 |     }
 77 | 
 78 |     void NeutralInvariant()
 79 |     {
 80 |         if constexpr(IsFeedbackOutput)
 81 |         {
 82 |             if ((!m_weight1.empty()) || (!m_weight2.empty()) || (!m_weight_lambda.empty()))
 83 |             {
 84 |                 throw std::runtime_error("NeutralInvariant Fail!");
 85 |             }
 86 |         }
 87 |     }
 88 | 
 89 | private:
 90 |     using DataType = LayerTraits::LayerInternalBuf<IsFeedbackOutput,
 91 |                                                    PolicySelect<InputPolicy, CurLayerPolicy>::BatchMode,
 92 |                                                    typename PolicySelect<OperandPolicy, CurLayerPolicy>::Element,
 93 |                                                    typename PolicySelect<OperandPolicy, CurLayerPolicy>::Device,
 94 |                                                    CategoryTags::Matrix, CategoryTags::BatchMatrix>;
 95 |     DataType m_weight1;
 96 |     DataType m_weight2;
 97 |     DataType m_weight_lambda;
 98 | };
 99 | }
100 | 


--------------------------------------------------------------------------------
/MetaNN/layers/compose/single_layer.h:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include <MetaNN/layers/compose/compose_kernel.h>
  4 | #include <MetaNN/layers/elementary/weight_layer.h>
  5 | #include <MetaNN/layers/elementary/bias_layer.h>
  6 | #include <MetaNN/layers/elementary/tanh_layer.h>
  7 | #include <MetaNN/layers/elementary/sigmoid_layer.h>
  8 | 
  9 | namespace MetaNN
 10 | {
 11 | template <typename TPolicies> class SingleLayer;
 12 | 
 13 | template <>
 14 | struct Sublayerof<SingleLayer>
 15 | {
 16 |     struct Weight;
 17 |     struct Bias;
 18 |     struct Action;
 19 | };
 20 | 
 21 | namespace NSSingleLayer
 22 | {
 23 | using WeightSublayer = Sublayerof<SingleLayer>::Weight;
 24 | using BiasSublayer   = Sublayerof<SingleLayer>::Bias;
 25 | using ActionSublayer = Sublayerof<SingleLayer>::Action;
 26 | 
 27 | template <typename TPlainPolicies>
 28 | constexpr static bool HasBias = PolicySelect<SingleLayerPolicy, TPlainPolicies>::HasBias;
 29 | 
 30 | template <typename TPlainPolicies> struct ActPick_;
 31 | 
 32 | template <>
 33 | struct ActPick_<SingleLayerPolicy::ActionTypeCate::Sigmoid>
 34 | {
 35 |     template <typename T>
 36 |     using type = SigmoidLayer<T>;
 37 | };
 38 | 
 39 | template <>
 40 | struct ActPick_<SingleLayerPolicy::ActionTypeCate::Tanh>
 41 | {
 42 |     template <typename T>
 43 |     using type = TanhLayer<T>;
 44 | };
 45 | 
 46 | template <bool hasBias, template <typename> class TActLayer>
 47 | struct TopologyHelper
 48 | {
 49 |     using type = ComposeTopology<SubLayer<WeightSublayer, WeightLayer>,
 50 |                                  SubLayer<BiasSublayer, BiasLayer>,
 51 |                                  SubLayer<ActionSublayer, TActLayer>,
 52 |                                  InConnect<LayerIO, WeightSublayer, LayerIO>,
 53 |                                  InternalConnect<WeightSublayer, LayerIO, BiasSublayer, LayerIO>,
 54 |                                  InternalConnect<BiasSublayer, LayerIO, ActionSublayer, LayerIO>,
 55 |                                  OutConnect<ActionSublayer, LayerIO, LayerIO>>;
 56 | };
 57 | 
 58 | template <template <typename> class TActLayer>
 59 | struct TopologyHelper<false, TActLayer>
 60 | {
 61 |     using type = ComposeTopology<SubLayer<WeightSublayer, WeightLayer>,
 62 |                                  SubLayer<ActionSublayer, TActLayer>,
 63 |                                  InConnect<LayerIO, WeightSublayer, LayerIO>,
 64 |                                  InternalConnect<WeightSublayer, LayerIO, ActionSublayer, LayerIO>,
 65 |                                  OutConnect<ActionSublayer, LayerIO, LayerIO>>;
 66 | };
 67 | 
 68 | template <typename TPolicies>
 69 | struct KernelHelper
 70 | {
 71 |     using PlainPolicies = PlainPolicy<TPolicies>;
 72 |     constexpr static bool hasBias = HasBias<PlainPolicies>;
 73 | 
 74 |     template <typename T>
 75 |     using ActType = typename ActPick_<typename PolicySelect<SingleLayerPolicy,
 76 |                                                             PlainPolicies>::Action>::template type<T>;
 77 | 
 78 |     using type = typename TopologyHelper<hasBias, ActType>::type;
 79 | };
 80 | 
 81 | template <typename TPolicies>
 82 | using Kernel = typename KernelHelper<TPolicies>::type;
 83 | 
 84 | template <typename TPolicies>
 85 | using Base = ComposeKernel<LayerIO, LayerIO, TPolicies, Kernel<TPolicies>>;
 86 | 
 87 | template <bool HasBias, typename TBase>
 88 | auto TupleCreator(const std::string& p_name, size_t p_inputLen, size_t p_outputLen)
 89 | {
 90 |     if constexpr (HasBias)
 91 |     {
 92 |         return TBase::CreateSubLayers()
 93 |                     .template Set<NSSingleLayer::WeightSublayer>(p_name + "-weight", p_inputLen, p_outputLen)
 94 |                     .template Set<NSSingleLayer::BiasSublayer>(p_name + "-bias", p_outputLen)
 95 |                     .template Set<NSSingleLayer::ActionSublayer>();
 96 |     }
 97 |     else
 98 |     {
 99 |         return TBase::CreateSubLayers()
100 |                     .template Set<NSSingleLayer::WeightSublayer>(p_name + "-weight", p_inputLen, p_outputLen)
101 |                     .template Set<NSSingleLayer::ActionSublayer>();
102 |     }
103 | }
104 | }
105 | 
106 | template <typename TPolicies>
107 | class SingleLayer : public NSSingleLayer::Base<TPolicies>
108 | {
109 |     using TBase = NSSingleLayer::Base<TPolicies>;
110 | public:
111 |     SingleLayer(const std::string& p_name, size_t p_inputLen, size_t p_outputLen)
112 |         : TBase(NSSingleLayer::TupleCreator<NSSingleLayer::KernelHelper<TPolicies>::hasBias, TBase>(p_name, p_inputLen, p_outputLen))
113 |     { }
114 | };
115 | }
116 | 


--------------------------------------------------------------------------------
/MetaNN/model_rel/param_initializer/facilities/traits.h:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include <MetaNN/model_rel/param_initializer/facilities/policies.h>
  4 | 
  5 | namespace MetaNN
  6 | {
  7 | namespace NSParamInit
  8 | {
  9 | template <typename TPolicyCont, typename TGroup>
 10 | struct Group2Initializer_;
 11 | 
 12 | template <typename TPolicyCont>
 13 | struct Group2Initializer_<TPolicyCont, InitPolicy::WeightTypeCate>
 14 | {
 15 |     using type = typename PolicySelect<InitPolicy, TPolicyCont>::Weight;
 16 | };
 17 | 
 18 | template <typename TPolicyCont>
 19 | struct Group2Initializer_<TPolicyCont, InitPolicy::BiasTypeCate>
 20 | {
 21 |     using type = typename PolicySelect<InitPolicy, TPolicyCont>::Bias;
 22 | };
 23 | 
 24 | template <typename TPolicyCont, typename TSpecInit>
 25 | struct PickInitializerBySpec_
 26 | {
 27 |     using type = TSpecInit;
 28 | };
 29 | 
 30 | template <typename TPolicyCont>
 31 | struct PickInitializerBySpec_<TPolicyCont, void>
 32 | {
 33 |     using type = typename PolicySelect<InitPolicy, TPolicyCont>::Overall;
 34 | };
 35 | 
 36 | template <typename TPolicyCont, typename TSpecInitializer>
 37 | struct PickInitializer_
 38 | {
 39 |     static_assert(IsPolicyContainer<TPolicyCont>);
 40 |     using CurInitPolicy = PlainPolicy<TPolicyCont>;
 41 |     
 42 |     static_assert(!std::is_same<TSpecInitializer, InitPolicy::OverallTypeCate>::value);
 43 |     
 44 |     using SpecInitializer = typename Group2Initializer_<CurInitPolicy, TSpecInitializer>::type;
 45 |     
 46 |     using type = typename PickInitializerBySpec_<CurInitPolicy, SpecInitializer>::type;
 47 | };
 48 | }
 49 | 
 50 | template <typename TPolicyCont, typename TSpecInit>
 51 | using PickInitializer = typename NSParamInit::PickInitializer_<TPolicyCont, TSpecInit>::type;
 52 | 
 53 | namespace NSParamInit
 54 | {
 55 | template <typename TRes, typename...TPolicies>
 56 | struct FillerTagFromPolicy_
 57 | {
 58 |     using type = TRes;
 59 | };
 60 | 
 61 | template <typename...TRes, typename TCur, typename...TRem>
 62 | struct FillerTagFromPolicy_<std::tuple<TRes...>, PInitializerIs<TCur>, TRem...>
 63 | {
 64 |     using type = typename FillerTagFromPolicy_<std::tuple<TRes..., TCur>, TRem...>::type;
 65 | };
 66 | 
 67 | template <typename...TRes, typename TCur, typename...TRem>
 68 | struct FillerTagFromPolicy_<std::tuple<TRes...>, PWeightInitializerIs<TCur>, TRem...>
 69 | {
 70 |     using type = typename FillerTagFromPolicy_<std::tuple<TRes..., TCur>, TRem...>::type;
 71 | };
 72 | 
 73 | template <typename...TRes, typename TCur, typename...TRem>
 74 | struct FillerTagFromPolicy_<std::tuple<TRes...>, PBiasInitializerIs<TCur>, TRem...>
 75 | {
 76 |     using type = typename FillerTagFromPolicy_<std::tuple<TRes..., TCur>, TRem...>::type;
 77 | };
 78 | 
 79 | template <typename...TRes, typename TLayer, typename...TSub, typename...TRem>
 80 | struct FillerTagFromPolicy_<std::tuple<TRes...>, SubPolicyContainer<TLayer, TSub...>, TRem...>
 81 | {
 82 |     using step1 = typename FillerTagFromPolicy_<std::tuple<TRes...>, TSub...>::type;
 83 |     using type = typename FillerTagFromPolicy_<step1, TRem...>::type;
 84 | };
 85 | 
 86 | template <typename TChecker, typename...TCheckes>
 87 | struct AlreadyExist_
 88 | {
 89 |     constexpr static bool value = false;
 90 | };
 91 | 
 92 | template <typename TChecker, typename TCur, typename...TRem>
 93 | struct AlreadyExist_<TChecker, TCur, TRem...>
 94 | {
 95 |     constexpr static bool value = OrValue<std::is_same<TChecker, TCur>::value,
 96 |                                           AlreadyExist_<TChecker, TRem...>>;
 97 | };
 98 | 
 99 | template <typename TRes, typename TPolicyTuple>
100 | struct TagDedupe_
101 | {
102 |     using type = TRes;
103 | };
104 | 
105 | template <typename...TRes, typename TCur, typename...TRem>
106 | struct TagDedupe_<VarTypeDict<TRes...>, std::tuple<TCur, TRem...>>
107 | {
108 |     using type = typename std::conditional_t<AlreadyExist_<TCur, TRes...>::value,
109 |                                              TagDedupe_<VarTypeDict<TRes...>, std::tuple<TRem...>>,
110 |                                              TagDedupe_<VarTypeDict<TRes..., TCur>, std::tuple<TRem...>>>::type;
111 | };
112 | 
113 | template <typename...TPolicies>
114 | struct FillerTags2NamedParams_
115 | {
116 |     using step1 = typename FillerTagFromPolicy_<std::tuple<>, TPolicies...>::type;
117 |     using type = typename TagDedupe_<VarTypeDict<>, step1>::type;
118 | };
119 | }
120 | 
121 | template <typename...TPolicies>
122 | using FillerTags2NamedParams = typename NSParamInit::FillerTags2NamedParams_<TPolicies...>::type;
123 | }
124 | 


--------------------------------------------------------------------------------