├── .clang-format ├── .github └── workflows │ └── build.yml ├── .gitignore ├── .gitmodules ├── CMakeLists.txt ├── Dependencies └── nlohmann │ └── json.hpp ├── LICENSE ├── NAM ├── activations.cpp ├── activations.h ├── convnet.cpp ├── convnet.h ├── dsp.cpp ├── dsp.h ├── get_dsp.cpp ├── get_dsp.h ├── lstm.cpp ├── lstm.h ├── util.cpp ├── util.h ├── version.h ├── wavenet.cpp └── wavenet.h ├── README.md ├── build └── .gitignore ├── example_models ├── lstm.nam └── wavenet.nam ├── format.sh └── tools ├── CMakeLists.txt ├── benchmodel.cpp ├── loadmodel.cpp ├── run_tests.cpp └── test ├── test_activations.cpp ├── test_dsp.cpp ├── test_get_dsp.cpp └── test_wavenet.cpp /.clang-format: -------------------------------------------------------------------------------- 1 | AccessModifierOffset: -2 2 | AlignAfterOpenBracket: Align 3 | AlignConsecutiveAssignments: false 4 | AlignConsecutiveDeclarations: false 5 | AlignEscapedNewlines: Right 6 | AlignOperands: true 7 | AlignTrailingComments: false 8 | AllowAllParametersOfDeclarationOnNextLine: false 9 | AllowShortBlocksOnASingleLine: true 10 | AllowShortCaseLabelsOnASingleLine: true 11 | AllowShortFunctionsOnASingleLine: Inline 12 | AllowShortIfStatementsOnASingleLine: false 13 | AllowShortLoopsOnASingleLine: false 14 | AlwaysBreakAfterDefinitionReturnType: None 15 | AlwaysBreakAfterReturnType: None 16 | AlwaysBreakBeforeMultilineStrings: true 17 | AlwaysBreakTemplateDeclarations: true 18 | BinPackArguments: true 19 | BinPackParameters: true 20 | BraceWrapping: 21 | AfterCaseLabel: true 22 | AfterClass: true 23 | AfterControlStatement: true 24 | AfterEnum: true 25 | AfterFunction: true 26 | AfterNamespace: true 27 | AfterStruct: true 28 | AfterUnion: true 29 | BeforeCatch: true 30 | BeforeElse: true 31 | IndentBraces: false 32 | SplitEmptyFunction: true 33 | SplitEmptyRecord: true 34 | SplitEmptyNamespace: true 35 | BreakBeforeBinaryOperators: NonAssignment 36 | BreakBeforeBraces: Custom 37 | BreakBeforeInheritanceComma: false 38 | BreakBeforeTernaryOperators: true 39 | BreakConstructorInitializers: BeforeComma 40 | BreakStringLiterals: true 41 | ColumnLimit: 120 42 | CompactNamespaces: false 43 | ConstructorInitializerAllOnOneLineOrOnePerLine: false 44 | ConstructorInitializerIndentWidth: 0 45 | ContinuationIndentWidth: 2 46 | Cpp11BracedListStyle: true 47 | DerivePointerAlignment: false 48 | FixNamespaceComments: true 49 | IndentCaseLabels: true 50 | IndentPPDirectives: BeforeHash 51 | IndentWidth: 2 52 | IndentWrappedFunctionNames: false 53 | KeepEmptyLinesAtTheStartOfBlocks: true 54 | MaxEmptyLinesToKeep: 2 55 | NamespaceIndentation: None 56 | ObjCBinPackProtocolList: Auto 57 | ObjCBlockIndentWidth: 2 58 | ObjCBreakBeforeNestedBlockParam: true 59 | ObjCSpaceAfterProperty: false 60 | ObjCSpaceBeforeProtocolList: true 61 | PenaltyBreakBeforeFirstCallParameter: 0 62 | PenaltyReturnTypeOnItsOwnLine: 1000 63 | PointerAlignment: Left 64 | ReflowComments: true 65 | SortIncludes: false 66 | SortUsingDeclarations: true 67 | SpaceAfterCStyleCast: false 68 | SpaceAfterTemplateKeyword: true 69 | SpaceBeforeAssignmentOperators: true 70 | SpaceBeforeParens: ControlStatements 71 | SpaceInEmptyParentheses: false 72 | SpacesBeforeTrailingComments: 1 73 | SpacesInAngles: false 74 | SpacesInContainerLiterals: true 75 | SpacesInCStyleCastParentheses: false 76 | SpacesInParentheses: false 77 | SpacesInSquareBrackets: false 78 | Standard: Cpp11 79 | UseTab: Never 80 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | 3 | on: 4 | [workflow_dispatch, pull_request] 5 | 6 | env: 7 | BUILD_TYPE: Release 8 | 9 | jobs: 10 | build-ubuntu: 11 | name: Build Ubuntu 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v3.3.0 15 | with: 16 | submodules: recursive 17 | 18 | - name: Build Tools 19 | working-directory: ${{github.workspace}}/build 20 | env: 21 | CXX: clang++ 22 | run: | 23 | cmake .. 24 | cmake --build . -j4 25 | 26 | - name: Run tests 27 | working-directory: ${{github.workspace}} 28 | run: | 29 | ./build/tools/run_tests 30 | ./build/tools/benchmodel ./example_models/wavenet.nam 31 | ./build/tools/benchmodel ./example_models/lstm.nam 32 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Prerequisites 2 | *.d 3 | 4 | # Compiled Object files 5 | *.slo 6 | *.lo 7 | *.o 8 | *.obj 9 | 10 | # Precompiled Headers 11 | *.gch 12 | *.pch 13 | 14 | # Compiled Dynamic libraries 15 | *.so 16 | *.dylib 17 | *.dll 18 | 19 | # Fortran module files 20 | *.mod 21 | *.smod 22 | 23 | # Compiled Static libraries 24 | *.lai 25 | *.la 26 | *.a 27 | *.lib 28 | 29 | # Executables 30 | *.exe 31 | *.out 32 | *.app 33 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "Dependencies/eigen"] 2 | path = Dependencies/eigen 3 | url = https://gitlab.com/libeigen/eigen 4 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.10) 2 | 3 | # Make sure this matches ./NAM/version.h! 4 | project(NAM VERSION 0.3.0) 5 | 6 | set(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake") 7 | 8 | set(CMAKE_CXX_STANDARD 20) 9 | set(CMAKE_CXX_STANDARD_REQUIRED OFF) 10 | set(CMAKE_CXX_EXTENSIONS OFF) 11 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++ -std=c++20") 12 | 13 | if (CMAKE_SYSTEM_NAME STREQUAL "Darwin") 14 | include_directories(SYSTEM /usr/local/include) 15 | elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux") 16 | link_libraries(stdc++fs) 17 | elseif (CMAKE_SYSTEM_NAME STREQUAL "Windows") 18 | add_compile_definitions(NOMINMAX WIN32_LEAN_AND_MEAN) 19 | else() 20 | message(FATAL_ERROR "Unrecognized Platform!") 21 | endif() 22 | 23 | set(NAM_DEPS_PATH "${CMAKE_CURRENT_SOURCE_DIR}/Dependencies") 24 | 25 | add_subdirectory(tools) 26 | 27 | #file(MAKE_DIRECTORY build/tools) 28 | 29 | #add_custom_target(copy_tools ALL 30 | # ${CMAKE_COMMAND} -E copy "$" tools/ 31 | # DEPENDS tools 32 | #) 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Steven Atkinson 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /NAM/activations.cpp: -------------------------------------------------------------------------------- 1 | #include "activations.h" 2 | 3 | nam::activations::ActivationTanh _TANH = nam::activations::ActivationTanh(); 4 | nam::activations::ActivationFastTanh _FAST_TANH = nam::activations::ActivationFastTanh(); 5 | nam::activations::ActivationHardTanh _HARD_TANH = nam::activations::ActivationHardTanh(); 6 | nam::activations::ActivationReLU _RELU = nam::activations::ActivationReLU(); 7 | nam::activations::ActivationLeakyReLU _LEAKY_RELU = nam::activations::ActivationLeakyReLU(); 8 | nam::activations::ActivationSigmoid _SIGMOID = nam::activations::ActivationSigmoid(); 9 | 10 | bool nam::activations::Activation::using_fast_tanh = false; 11 | 12 | std::unordered_map nam::activations::Activation::_activations = { 13 | {"Tanh", &_TANH}, {"Hardtanh", &_HARD_TANH}, {"Fasttanh", &_FAST_TANH}, 14 | {"ReLU", &_RELU}, {"LeakyReLU", &_LEAKY_RELU}, {"Sigmoid", &_SIGMOID}}; 15 | 16 | nam::activations::Activation* tanh_bak = nullptr; 17 | 18 | nam::activations::Activation* nam::activations::Activation::get_activation(const std::string name) 19 | { 20 | if (_activations.find(name) == _activations.end()) 21 | return nullptr; 22 | 23 | return _activations[name]; 24 | } 25 | 26 | void nam::activations::Activation::enable_fast_tanh() 27 | { 28 | nam::activations::Activation::using_fast_tanh = true; 29 | 30 | if (_activations["Tanh"] != _activations["Fasttanh"]) 31 | { 32 | tanh_bak = _activations["Tanh"]; 33 | _activations["Tanh"] = _activations["Fasttanh"]; 34 | } 35 | } 36 | 37 | void nam::activations::Activation::disable_fast_tanh() 38 | { 39 | nam::activations::Activation::using_fast_tanh = false; 40 | 41 | if (_activations["Tanh"] == _activations["Fasttanh"]) 42 | { 43 | _activations["Tanh"] = tanh_bak; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /NAM/activations.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include // expf 5 | #include 6 | #include 7 | 8 | namespace nam 9 | { 10 | namespace activations 11 | { 12 | inline float relu(float x) 13 | { 14 | return x > 0.0f ? x : 0.0f; 15 | }; 16 | 17 | inline float sigmoid(float x) 18 | { 19 | return 1.0f / (1.0f + expf(-x)); 20 | }; 21 | 22 | inline float hard_tanh(float x) 23 | { 24 | const float t = x < -1 ? -1 : x; 25 | return t > 1 ? 1 : t; 26 | } 27 | 28 | inline float fast_tanh(const float x) 29 | { 30 | const float ax = fabsf(x); 31 | const float x2 = x * x; 32 | 33 | return (x * (2.45550750702956f + 2.45550750702956f * ax + (0.893229853513558f + 0.821226666969744f * ax) * x2) 34 | / (2.44506634652299f + (2.44506634652299f + x2) * fabsf(x + 0.814642734961073f * x * ax))); 35 | } 36 | 37 | inline float fast_sigmoid(const float x) 38 | { 39 | return 0.5f * (fast_tanh(x * 0.5f) + 1.0f); 40 | } 41 | 42 | // Assumes PyTorch default of 0.01 for negative slope. This may change to be 43 | // configurable in the future. 44 | inline float leaky_relu(float x) 45 | { 46 | const float negative_slope = 0.01; 47 | return x > 0.0f ? x : negative_slope * x; 48 | } 49 | 50 | class Activation 51 | { 52 | public: 53 | Activation() = default; 54 | virtual ~Activation() = default; 55 | virtual void apply(Eigen::MatrixXf& matrix) { apply(matrix.data(), matrix.rows() * matrix.cols()); } 56 | virtual void apply(Eigen::Block block) { apply(block.data(), block.rows() * block.cols()); } 57 | virtual void apply(Eigen::Block block) 58 | { 59 | apply(block.data(), block.rows() * block.cols()); 60 | } 61 | virtual void apply(float* data, long size) {} 62 | 63 | static Activation* get_activation(const std::string name); 64 | static void enable_fast_tanh(); 65 | static void disable_fast_tanh(); 66 | static bool using_fast_tanh; 67 | 68 | protected: 69 | static std::unordered_map _activations; 70 | }; 71 | 72 | class ActivationTanh : public Activation 73 | { 74 | public: 75 | void apply(float* data, long size) override 76 | { 77 | for (long pos = 0; pos < size; pos++) 78 | { 79 | data[pos] = std::tanh(data[pos]); 80 | } 81 | } 82 | }; 83 | 84 | class ActivationHardTanh : public Activation 85 | { 86 | public: 87 | void apply(float* data, long size) override 88 | { 89 | for (long pos = 0; pos < size; pos++) 90 | { 91 | data[pos] = hard_tanh(data[pos]); 92 | } 93 | } 94 | }; 95 | 96 | class ActivationFastTanh : public Activation 97 | { 98 | public: 99 | void apply(float* data, long size) override 100 | { 101 | for (long pos = 0; pos < size; pos++) 102 | { 103 | data[pos] = fast_tanh(data[pos]); 104 | } 105 | } 106 | }; 107 | 108 | class ActivationReLU : public Activation 109 | { 110 | public: 111 | void apply(float* data, long size) override 112 | { 113 | for (long pos = 0; pos < size; pos++) 114 | { 115 | data[pos] = relu(data[pos]); 116 | } 117 | } 118 | }; 119 | 120 | class ActivationLeakyReLU : public Activation 121 | { 122 | public: 123 | void apply(float* data, long size) override 124 | { 125 | for (long pos = 0; pos < size; pos++) 126 | { 127 | data[pos] = leaky_relu(data[pos]); 128 | } 129 | } 130 | }; 131 | 132 | class ActivationSigmoid : public Activation 133 | { 134 | public: 135 | void apply(float* data, long size) override 136 | { 137 | for (long pos = 0; pos < size; pos++) 138 | { 139 | data[pos] = sigmoid(data[pos]); 140 | } 141 | } 142 | }; 143 | }; // namespace activations 144 | }; // namespace nam 145 | -------------------------------------------------------------------------------- /NAM/convnet.cpp: -------------------------------------------------------------------------------- 1 | #include // std::max_element 2 | #include 3 | #include // pow, tanh, expf 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include "dsp.h" 11 | #include "convnet.h" 12 | 13 | nam::convnet::BatchNorm::BatchNorm(const int dim, std::vector::iterator& weights) 14 | { 15 | // Extract from param buffer 16 | Eigen::VectorXf running_mean(dim); 17 | Eigen::VectorXf running_var(dim); 18 | Eigen::VectorXf _weight(dim); 19 | Eigen::VectorXf _bias(dim); 20 | for (int i = 0; i < dim; i++) 21 | running_mean(i) = *(weights++); 22 | for (int i = 0; i < dim; i++) 23 | running_var(i) = *(weights++); 24 | for (int i = 0; i < dim; i++) 25 | _weight(i) = *(weights++); 26 | for (int i = 0; i < dim; i++) 27 | _bias(i) = *(weights++); 28 | float eps = *(weights++); 29 | 30 | // Convert to scale & loc 31 | this->scale.resize(dim); 32 | this->loc.resize(dim); 33 | for (int i = 0; i < dim; i++) 34 | this->scale(i) = _weight(i) / sqrt(eps + running_var(i)); 35 | this->loc = _bias - this->scale.cwiseProduct(running_mean); 36 | } 37 | 38 | void nam::convnet::BatchNorm::process_(Eigen::MatrixXf& x, const long i_start, const long i_end) const 39 | { 40 | // todo using colwise? 41 | // #speed but conv probably dominates 42 | for (auto i = i_start; i < i_end; i++) 43 | { 44 | x.col(i) = x.col(i).cwiseProduct(this->scale); 45 | x.col(i) += this->loc; 46 | } 47 | } 48 | 49 | void nam::convnet::ConvNetBlock::set_weights_(const int in_channels, const int out_channels, const int _dilation, 50 | const bool batchnorm, const std::string activation, 51 | std::vector::iterator& weights) 52 | { 53 | this->_batchnorm = batchnorm; 54 | // HACK 2 kernel 55 | this->conv.set_size_and_weights_(in_channels, out_channels, 2, _dilation, !batchnorm, weights); 56 | if (this->_batchnorm) 57 | this->batchnorm = BatchNorm(out_channels, weights); 58 | this->activation = activations::Activation::get_activation(activation); 59 | } 60 | 61 | void nam::convnet::ConvNetBlock::process_(const Eigen::MatrixXf& input, Eigen::MatrixXf& output, const long i_start, 62 | const long i_end) const 63 | { 64 | const long ncols = i_end - i_start; 65 | this->conv.process_(input, output, i_start, ncols, i_start); 66 | if (this->_batchnorm) 67 | this->batchnorm.process_(output, i_start, i_end); 68 | 69 | this->activation->apply(output.middleCols(i_start, ncols)); 70 | } 71 | 72 | long nam::convnet::ConvNetBlock::get_out_channels() const 73 | { 74 | return this->conv.get_out_channels(); 75 | } 76 | 77 | nam::convnet::_Head::_Head(const int channels, std::vector::iterator& weights) 78 | { 79 | this->_weight.resize(channels); 80 | for (int i = 0; i < channels; i++) 81 | this->_weight[i] = *(weights++); 82 | this->_bias = *(weights++); 83 | } 84 | 85 | void nam::convnet::_Head::process_(const Eigen::MatrixXf& input, Eigen::VectorXf& output, const long i_start, 86 | const long i_end) const 87 | { 88 | const long length = i_end - i_start; 89 | output.resize(length); 90 | for (long i = 0, j = i_start; i < length; i++, j++) 91 | output(i) = this->_bias + input.col(j).dot(this->_weight); 92 | } 93 | 94 | nam::convnet::ConvNet::ConvNet(const int channels, const std::vector& dilations, const bool batchnorm, 95 | const std::string activation, std::vector& weights, 96 | const double expected_sample_rate) 97 | : Buffer(*std::max_element(dilations.begin(), dilations.end()), expected_sample_rate) 98 | { 99 | this->_verify_weights(channels, dilations, batchnorm, weights.size()); 100 | this->_blocks.resize(dilations.size()); 101 | std::vector::iterator it = weights.begin(); 102 | for (size_t i = 0; i < dilations.size(); i++) 103 | this->_blocks[i].set_weights_(i == 0 ? 1 : channels, channels, dilations[i], batchnorm, activation, it); 104 | this->_block_vals.resize(this->_blocks.size() + 1); 105 | for (auto& matrix : this->_block_vals) 106 | matrix.setZero(); 107 | std::fill(this->_input_buffer.begin(), this->_input_buffer.end(), 0.0f); 108 | this->_head = _Head(channels, it); 109 | if (it != weights.end()) 110 | throw std::runtime_error("Didn't touch all the weights when initializing ConvNet"); 111 | 112 | mPrewarmSamples = 1; 113 | for (size_t i = 0; i < dilations.size(); i++) 114 | mPrewarmSamples += dilations[i]; 115 | } 116 | 117 | 118 | void nam::convnet::ConvNet::process(NAM_SAMPLE* input, NAM_SAMPLE* output, const int num_frames) 119 | 120 | { 121 | this->_update_buffers_(input, num_frames); 122 | // Main computation! 123 | const long i_start = this->_input_buffer_offset; 124 | const long i_end = i_start + num_frames; 125 | // TODO one unnecessary copy :/ #speed 126 | for (auto i = i_start; i < i_end; i++) 127 | this->_block_vals[0](0, i) = this->_input_buffer[i]; 128 | for (size_t i = 0; i < this->_blocks.size(); i++) 129 | this->_blocks[i].process_(this->_block_vals[i], this->_block_vals[i + 1], i_start, i_end); 130 | // TODO clean up this allocation 131 | this->_head.process_(this->_block_vals[this->_blocks.size()], this->_head_output, i_start, i_end); 132 | // Copy to required output array (TODO tighten this up) 133 | for (int s = 0; s < num_frames; s++) 134 | output[s] = this->_head_output(s); 135 | 136 | // Prepare for next call: 137 | nam::Buffer::_advance_input_buffer_(num_frames); 138 | } 139 | 140 | void nam::convnet::ConvNet::_verify_weights(const int channels, const std::vector& dilations, const bool batchnorm, 141 | const size_t actual_weights) 142 | { 143 | // TODO 144 | } 145 | 146 | void nam::convnet::ConvNet::_update_buffers_(NAM_SAMPLE* input, const int num_frames) 147 | { 148 | this->Buffer::_update_buffers_(input, num_frames); 149 | 150 | const long buffer_size = (long)this->_input_buffer.size(); 151 | 152 | if (this->_block_vals[0].rows() != 1 || this->_block_vals[0].cols() != buffer_size) 153 | { 154 | this->_block_vals[0].resize(1, buffer_size); 155 | this->_block_vals[0].setZero(); 156 | } 157 | 158 | for (size_t i = 1; i < this->_block_vals.size(); i++) 159 | { 160 | if (this->_block_vals[i].rows() == this->_blocks[i - 1].get_out_channels() 161 | && this->_block_vals[i].cols() == buffer_size) 162 | continue; // Already has correct size 163 | this->_block_vals[i].resize(this->_blocks[i - 1].get_out_channels(), buffer_size); 164 | this->_block_vals[i].setZero(); 165 | } 166 | } 167 | 168 | void nam::convnet::ConvNet::_rewind_buffers_() 169 | { 170 | // Need to rewind the block vals first because Buffer::rewind_buffers() 171 | // resets the offset index 172 | // The last _block_vals is the output of the last block and doesn't need to be 173 | // rewound. 174 | for (size_t k = 0; k < this->_block_vals.size() - 1; k++) 175 | { 176 | // We actually don't need to pull back a lot...just as far as the first 177 | // input sample would grab from dilation 178 | const long _dilation = this->_blocks[k].conv.get_dilation(); 179 | for (long i = this->_receptive_field - _dilation, j = this->_input_buffer_offset - _dilation; 180 | j < this->_input_buffer_offset; i++, j++) 181 | for (long r = 0; r < this->_block_vals[k].rows(); r++) 182 | this->_block_vals[k](r, i) = this->_block_vals[k](r, j); 183 | } 184 | // Now we can do the rest of the rewind 185 | this->Buffer::_rewind_buffers_(); 186 | } 187 | -------------------------------------------------------------------------------- /NAM/convnet.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include 11 | 12 | namespace nam 13 | { 14 | namespace convnet 15 | { 16 | // Custom Conv that avoids re-computing on pieces of the input and trusts 17 | // that the corresponding outputs are where they need to be. 18 | // Beware: this is clever! 19 | 20 | // Batch normalization 21 | // In prod mode, so really just an elementwise affine layer. 22 | class BatchNorm 23 | { 24 | public: 25 | BatchNorm() {}; 26 | BatchNorm(const int dim, std::vector::iterator& weights); 27 | void process_(Eigen::MatrixXf& input, const long i_start, const long i_end) const; 28 | 29 | private: 30 | // TODO simplify to just ax+b 31 | // y = (x-m)/sqrt(v+eps) * w + bias 32 | // y = ax+b 33 | // a = w / sqrt(v+eps) 34 | // b = a * m + bias 35 | Eigen::VectorXf scale; 36 | Eigen::VectorXf loc; 37 | }; 38 | 39 | class ConvNetBlock 40 | { 41 | public: 42 | ConvNetBlock() {}; 43 | void set_weights_(const int in_channels, const int out_channels, const int _dilation, const bool batchnorm, 44 | const std::string activation, std::vector::iterator& weights); 45 | void process_(const Eigen::MatrixXf& input, Eigen::MatrixXf& output, const long i_start, const long i_end) const; 46 | long get_out_channels() const; 47 | Conv1D conv; 48 | 49 | private: 50 | BatchNorm batchnorm; 51 | bool _batchnorm = false; 52 | activations::Activation* activation = nullptr; 53 | }; 54 | 55 | class _Head 56 | { 57 | public: 58 | _Head() {}; 59 | _Head(const int channels, std::vector::iterator& weights); 60 | void process_(const Eigen::MatrixXf& input, Eigen::VectorXf& output, const long i_start, const long i_end) const; 61 | 62 | private: 63 | Eigen::VectorXf _weight; 64 | float _bias = 0.0f; 65 | }; 66 | 67 | class ConvNet : public Buffer 68 | { 69 | public: 70 | ConvNet(const int channels, const std::vector& dilations, const bool batchnorm, const std::string activation, 71 | std::vector& weights, const double expected_sample_rate = -1.0); 72 | ~ConvNet() = default; 73 | 74 | void process(NAM_SAMPLE* input, NAM_SAMPLE* output, const int num_frames) override; 75 | 76 | protected: 77 | std::vector _blocks; 78 | std::vector _block_vals; 79 | Eigen::VectorXf _head_output; 80 | _Head _head; 81 | void _verify_weights(const int channels, const std::vector& dilations, const bool batchnorm, 82 | const size_t actual_weights); 83 | void _update_buffers_(NAM_SAMPLE* input, const int num_frames) override; 84 | void _rewind_buffers_() override; 85 | 86 | int mPrewarmSamples = 0; // Pre-compute during initialization 87 | int PrewarmSamples() override { return mPrewarmSamples; }; 88 | }; 89 | }; // namespace convnet 90 | }; // namespace nam 91 | -------------------------------------------------------------------------------- /NAM/dsp.cpp: -------------------------------------------------------------------------------- 1 | #include // std::max_element 2 | #include 3 | #include // pow, tanh, expf 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include "dsp.h" 11 | 12 | #define tanh_impl_ std::tanh 13 | // #define tanh_impl_ fast_tanh_ 14 | 15 | constexpr const long _INPUT_BUFFER_SAFETY_FACTOR = 32; 16 | 17 | nam::DSP::DSP(const double expected_sample_rate) 18 | : mExpectedSampleRate(expected_sample_rate) 19 | { 20 | } 21 | 22 | void nam::DSP::prewarm() 23 | { 24 | if (mMaxBufferSize == 0) 25 | { 26 | SetMaxBufferSize(4096); 27 | } 28 | const int prewarmSamples = PrewarmSamples(); 29 | if (prewarmSamples == 0) 30 | return; 31 | 32 | const size_t bufferSize = std::max(mMaxBufferSize, 1); 33 | std::vector inputBuffer, outputBuffer; 34 | inputBuffer.resize(bufferSize); 35 | outputBuffer.resize(bufferSize); 36 | for (auto it = inputBuffer.begin(); it != inputBuffer.end(); ++it) 37 | { 38 | (*it) = (NAM_SAMPLE)0.0; 39 | } 40 | 41 | NAM_SAMPLE* inputPtr = inputBuffer.data(); 42 | NAM_SAMPLE* outputPtr = outputBuffer.data(); 43 | int samplesProcessed = 0; 44 | while (samplesProcessed < prewarmSamples) 45 | { 46 | this->process(inputPtr, outputPtr, bufferSize); 47 | samplesProcessed += bufferSize; 48 | } 49 | } 50 | 51 | void nam::DSP::process(NAM_SAMPLE* input, NAM_SAMPLE* output, const int num_frames) 52 | { 53 | // Default implementation is the null operation 54 | for (int i = 0; i < num_frames; i++) 55 | output[i] = input[i]; 56 | } 57 | 58 | double nam::DSP::GetLoudness() const 59 | { 60 | if (!HasLoudness()) 61 | { 62 | throw std::runtime_error("Asked for loudness of a model that doesn't know how loud it is!"); 63 | } 64 | return mLoudness; 65 | } 66 | 67 | void nam::DSP::Reset(const double sampleRate, const int maxBufferSize) 68 | { 69 | // Some subclasses might want to throw an exception if the sample rate is "wrong". 70 | // This could be under a debugging flag potentially. 71 | mExternalSampleRate = sampleRate; 72 | mHaveExternalSampleRate = true; 73 | SetMaxBufferSize(maxBufferSize); 74 | 75 | prewarm(); 76 | } 77 | 78 | void nam::DSP::SetLoudness(const double loudness) 79 | { 80 | mLoudness = loudness; 81 | mHasLoudness = true; 82 | } 83 | 84 | void nam::DSP::SetMaxBufferSize(const int maxBufferSize) 85 | { 86 | mMaxBufferSize = maxBufferSize; 87 | } 88 | 89 | // Buffer ===================================================================== 90 | 91 | nam::Buffer::Buffer(const int receptive_field, const double expected_sample_rate) 92 | : nam::DSP(expected_sample_rate) 93 | { 94 | this->_set_receptive_field(receptive_field); 95 | } 96 | 97 | void nam::Buffer::_set_receptive_field(const int new_receptive_field) 98 | { 99 | this->_set_receptive_field(new_receptive_field, _INPUT_BUFFER_SAFETY_FACTOR * new_receptive_field); 100 | }; 101 | 102 | void nam::Buffer::_set_receptive_field(const int new_receptive_field, const int input_buffer_size) 103 | { 104 | this->_receptive_field = new_receptive_field; 105 | this->_input_buffer.resize(input_buffer_size); 106 | std::fill(this->_input_buffer.begin(), this->_input_buffer.end(), 0.0f); 107 | this->_reset_input_buffer(); 108 | } 109 | 110 | void nam::Buffer::_update_buffers_(NAM_SAMPLE* input, const int num_frames) 111 | { 112 | // Make sure that the buffer is big enough for the receptive field and the 113 | // frames needed! 114 | { 115 | const long minimum_input_buffer_size = (long)this->_receptive_field + _INPUT_BUFFER_SAFETY_FACTOR * num_frames; 116 | if ((long)this->_input_buffer.size() < minimum_input_buffer_size) 117 | { 118 | long new_buffer_size = 2; 119 | while (new_buffer_size < minimum_input_buffer_size) 120 | new_buffer_size *= 2; 121 | this->_input_buffer.resize(new_buffer_size); 122 | std::fill(this->_input_buffer.begin(), this->_input_buffer.end(), 0.0f); 123 | } 124 | } 125 | 126 | // If we'd run off the end of the input buffer, then we need to move the data 127 | // back to the start of the buffer and start again. 128 | if (this->_input_buffer_offset + num_frames > (long)this->_input_buffer.size()) 129 | this->_rewind_buffers_(); 130 | // Put the new samples into the input buffer 131 | for (long i = this->_input_buffer_offset, j = 0; j < num_frames; i++, j++) 132 | this->_input_buffer[i] = input[j]; 133 | // And resize the output buffer: 134 | this->_output_buffer.resize(num_frames); 135 | std::fill(this->_output_buffer.begin(), this->_output_buffer.end(), 0.0f); 136 | } 137 | 138 | void nam::Buffer::_rewind_buffers_() 139 | { 140 | // Copy the input buffer back 141 | // RF-1 samples because we've got at least one new one inbound. 142 | for (long i = 0, j = this->_input_buffer_offset - this->_receptive_field; i < this->_receptive_field; i++, j++) 143 | this->_input_buffer[i] = this->_input_buffer[j]; 144 | // And reset the offset. 145 | // Even though we could be stingy about that one sample that we won't be using 146 | // (because a new set is incoming) it's probably not worth the 147 | // hyper-optimization and liable for bugs. And the code looks way tidier this 148 | // way. 149 | this->_input_buffer_offset = this->_receptive_field; 150 | } 151 | 152 | void nam::Buffer::_reset_input_buffer() 153 | { 154 | this->_input_buffer_offset = this->_receptive_field; 155 | } 156 | 157 | void nam::Buffer::_advance_input_buffer_(const int num_frames) 158 | { 159 | this->_input_buffer_offset += num_frames; 160 | } 161 | 162 | // Linear ===================================================================== 163 | 164 | nam::Linear::Linear(const int receptive_field, const bool _bias, const std::vector& weights, 165 | const double expected_sample_rate) 166 | : nam::Buffer(receptive_field, expected_sample_rate) 167 | { 168 | if ((int)weights.size() != (receptive_field + (_bias ? 1 : 0))) 169 | throw std::runtime_error( 170 | "Params vector does not match expected size based " 171 | "on architecture parameters"); 172 | 173 | this->_weight.resize(this->_receptive_field); 174 | // Pass in in reverse order so that dot products work out of the box. 175 | for (int i = 0; i < this->_receptive_field; i++) 176 | this->_weight(i) = weights[receptive_field - 1 - i]; 177 | this->_bias = _bias ? weights[receptive_field] : (float)0.0; 178 | } 179 | 180 | void nam::Linear::process(NAM_SAMPLE* input, NAM_SAMPLE* output, const int num_frames) 181 | { 182 | this->nam::Buffer::_update_buffers_(input, num_frames); 183 | 184 | // Main computation! 185 | for (int i = 0; i < num_frames; i++) 186 | { 187 | const long offset = this->_input_buffer_offset - this->_weight.size() + i + 1; 188 | auto input = Eigen::Map(&this->_input_buffer[offset], this->_receptive_field); 189 | output[i] = this->_bias + this->_weight.dot(input); 190 | } 191 | 192 | // Prepare for next call: 193 | nam::Buffer::_advance_input_buffer_(num_frames); 194 | } 195 | 196 | // NN modules ================================================================= 197 | 198 | void nam::Conv1D::set_weights_(std::vector::iterator& weights) 199 | { 200 | if (this->_weight.size() > 0) 201 | { 202 | const long out_channels = this->_weight[0].rows(); 203 | const long in_channels = this->_weight[0].cols(); 204 | // Crazy ordering because that's how it gets flattened. 205 | for (auto i = 0; i < out_channels; i++) 206 | for (auto j = 0; j < in_channels; j++) 207 | for (size_t k = 0; k < this->_weight.size(); k++) 208 | this->_weight[k](i, j) = *(weights++); 209 | } 210 | for (long i = 0; i < this->_bias.size(); i++) 211 | this->_bias(i) = *(weights++); 212 | } 213 | 214 | void nam::Conv1D::set_size_(const int in_channels, const int out_channels, const int kernel_size, const bool do_bias, 215 | const int _dilation) 216 | { 217 | this->_weight.resize(kernel_size); 218 | for (size_t i = 0; i < this->_weight.size(); i++) 219 | this->_weight[i].resize(out_channels, 220 | in_channels); // y = Ax, input array (C,L) 221 | if (do_bias) 222 | this->_bias.resize(out_channels); 223 | else 224 | this->_bias.resize(0); 225 | this->_dilation = _dilation; 226 | } 227 | 228 | void nam::Conv1D::set_size_and_weights_(const int in_channels, const int out_channels, const int kernel_size, 229 | const int _dilation, const bool do_bias, std::vector::iterator& weights) 230 | { 231 | this->set_size_(in_channels, out_channels, kernel_size, do_bias, _dilation); 232 | this->set_weights_(weights); 233 | } 234 | 235 | void nam::Conv1D::process_(const Eigen::MatrixXf& input, Eigen::MatrixXf& output, const long i_start, const long ncols, 236 | const long j_start) const 237 | { 238 | // This is the clever part ;) 239 | for (size_t k = 0; k < this->_weight.size(); k++) 240 | { 241 | const long offset = this->_dilation * (k + 1 - this->_weight.size()); 242 | if (k == 0) 243 | output.middleCols(j_start, ncols).noalias() = this->_weight[k] * input.middleCols(i_start + offset, ncols); 244 | else 245 | output.middleCols(j_start, ncols).noalias() += this->_weight[k] * input.middleCols(i_start + offset, ncols); 246 | } 247 | if (this->_bias.size() > 0) 248 | { 249 | output.middleCols(j_start, ncols).colwise() += this->_bias; 250 | } 251 | } 252 | 253 | long nam::Conv1D::get_num_weights() const 254 | { 255 | long num_weights = this->_bias.size(); 256 | for (size_t i = 0; i < this->_weight.size(); i++) 257 | num_weights += this->_weight[i].size(); 258 | return num_weights; 259 | } 260 | 261 | nam::Conv1x1::Conv1x1(const int in_channels, const int out_channels, const bool _bias) 262 | { 263 | this->_weight.resize(out_channels, in_channels); 264 | this->_do_bias = _bias; 265 | if (_bias) 266 | this->_bias.resize(out_channels); 267 | } 268 | 269 | Eigen::Block nam::Conv1x1::GetOutput(const int num_frames) 270 | { 271 | return _output.block(0, 0, _output.rows(), num_frames); 272 | } 273 | 274 | void nam::Conv1x1::SetMaxBufferSize(const int maxBufferSize) 275 | { 276 | _output.resize(get_out_channels(), maxBufferSize); 277 | } 278 | 279 | void nam::Conv1x1::set_weights_(std::vector::iterator& weights) 280 | { 281 | for (int i = 0; i < this->_weight.rows(); i++) 282 | for (int j = 0; j < this->_weight.cols(); j++) 283 | this->_weight(i, j) = *(weights++); 284 | if (this->_do_bias) 285 | for (int i = 0; i < this->_bias.size(); i++) 286 | this->_bias(i) = *(weights++); 287 | } 288 | 289 | Eigen::MatrixXf nam::Conv1x1::process(const Eigen::MatrixXf& input, const int num_frames) const 290 | { 291 | if (this->_do_bias) 292 | return (this->_weight * input.leftCols(num_frames)).colwise() + this->_bias; 293 | else 294 | return this->_weight * input.leftCols(num_frames); 295 | } 296 | 297 | void nam::Conv1x1::process_(const Eigen::MatrixXf& input, const int num_frames) 298 | { 299 | assert(num_frames <= _output.cols()); 300 | _output.leftCols(num_frames).noalias() = this->_weight * input.leftCols(num_frames); 301 | if (this->_do_bias) 302 | { 303 | _output.leftCols(num_frames).colwise() += this->_bias; 304 | } 305 | } 306 | -------------------------------------------------------------------------------- /NAM/dsp.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include 11 | 12 | #include "activations.h" 13 | #include "json.hpp" 14 | 15 | #ifdef NAM_SAMPLE_FLOAT 16 | #define NAM_SAMPLE float 17 | #else 18 | #define NAM_SAMPLE double 19 | #endif 20 | // Use a sample rate of -1 if we don't know what the model expects to be run at. 21 | // TODO clean this up and track a bool for whether it knows. 22 | #define NAM_UNKNOWN_EXPECTED_SAMPLE_RATE -1.0 23 | 24 | namespace nam 25 | { 26 | enum EArchitectures 27 | { 28 | kLinear = 0, 29 | kConvNet, 30 | kLSTM, 31 | kCatLSTM, 32 | kWaveNet, 33 | kCatWaveNet, 34 | kNumModels 35 | }; 36 | 37 | class DSP 38 | { 39 | public: 40 | // Older models won't know, but newer ones will come with a loudness from the training based on their response to a 41 | // standardized input. 42 | // We may choose to have the models figure out for themselves how loud they are in here in the future. 43 | DSP(const double expected_sample_rate); 44 | virtual ~DSP() = default; 45 | // prewarm() does any required intial work required to "settle" model initial conditions 46 | // it can be somewhat expensive, so should not be called during realtime audio processing 47 | // Important: don't expect the model to be outputting zeroes after this. Neural networks 48 | // Don't know that there's anything special about "zero", and forcing this gets rid of 49 | // some possibilities that I dont' want to rule out (e.g. models that "are noisy"). 50 | virtual void prewarm(); 51 | // process() does all of the processing requried to take `input` array and 52 | // fill in the required values on `output`. 53 | // To do this: 54 | // 1. The core DSP algorithm is run (This is what should probably be 55 | // overridden in subclasses). 56 | // 2. The output level is applied and the result stored to `output`. 57 | virtual void process(NAM_SAMPLE* input, NAM_SAMPLE* output, const int num_frames); 58 | // Expected sample rate, in Hz. 59 | // TODO throw if it doesn't know. 60 | double GetExpectedSampleRate() const { return mExpectedSampleRate; }; 61 | // Input Level, in dBu, corresponding to 0 dBFS for a sine wave 62 | // You should call HasInputLevel() first to be safe. 63 | double GetInputLevel() { return mInputLevel.level; }; 64 | // Get how loud this model is, in dB. 65 | // Throws a std::runtime_error if the model doesn't know how loud it is. 66 | double GetLoudness() const; 67 | // Output Level, in dBu, corresponding to 0 dBFS for a sine wave 68 | // You should call HasOutputLevel() first to be safe. 69 | double GetOutputLevel() { return mOutputLevel.level; }; 70 | // Does this model know its output level? 71 | bool HasInputLevel() { return mInputLevel.haveLevel; }; 72 | // Get whether the model knows how loud it is. 73 | bool HasLoudness() const { return mHasLoudness; }; 74 | // Does this model know its output level? 75 | bool HasOutputLevel() { return mOutputLevel.haveLevel; }; 76 | // General function for resetting the DSP unit. 77 | // This doesn't call prewarm(). If you want to do that, then you might want to use ResetAndPrewarm(). 78 | // See https://github.com/sdatkinson/NeuralAmpModelerCore/issues/96 for the reasoning. 79 | virtual void Reset(const double sampleRate, const int maxBufferSize); 80 | // Reset(), then prewarm() 81 | void ResetAndPrewarm(const double sampleRate, const int maxBufferSize) 82 | { 83 | Reset(sampleRate, maxBufferSize); 84 | prewarm(); 85 | } 86 | void SetInputLevel(const double inputLevel) 87 | { 88 | mInputLevel.haveLevel = true; 89 | mInputLevel.level = inputLevel; 90 | }; 91 | // Set the loudness, in dB. 92 | // This is usually defined to be the loudness to a standardized input. The trainer has its own, but you can always 93 | // use this to define it a different way if you like yours better. 94 | void SetLoudness(const double loudness); 95 | void SetOutputLevel(const double outputLevel) 96 | { 97 | mOutputLevel.haveLevel = true; 98 | mOutputLevel.level = outputLevel; 99 | }; 100 | 101 | protected: 102 | bool mHasLoudness = false; 103 | // How loud is the model? In dB 104 | double mLoudness = 0.0; 105 | // What sample rate does the model expect? 106 | double mExpectedSampleRate; 107 | // Have we been told what the external sample rate is? If so, what is it? 108 | bool mHaveExternalSampleRate = false; 109 | double mExternalSampleRate = -1.0; 110 | // The largest buffer I expect to be told to process: 111 | int mMaxBufferSize = 0; 112 | 113 | // How many samples should be processed for me to be considered "warmed up"? 114 | virtual int PrewarmSamples() { return 0; }; 115 | 116 | virtual void SetMaxBufferSize(const int maxBufferSize); 117 | 118 | private: 119 | struct Level 120 | { 121 | bool haveLevel = false; 122 | float level = 0.0; 123 | }; 124 | Level mInputLevel; 125 | Level mOutputLevel; 126 | }; 127 | 128 | // Class where an input buffer is kept so that long-time effects can be 129 | // captured. (e.g. conv nets or impulse responses, where we need history that's 130 | // longer than the sample buffer that's coming in.) 131 | class Buffer : public DSP 132 | { 133 | public: 134 | Buffer(const int receptive_field, const double expected_sample_rate = -1.0); 135 | 136 | protected: 137 | // Input buffer 138 | const int _input_buffer_channels = 1; // Mono 139 | int _receptive_field; 140 | // First location where we add new samples from the input 141 | long _input_buffer_offset; 142 | std::vector _input_buffer; 143 | std::vector _output_buffer; 144 | 145 | void _advance_input_buffer_(const int num_frames); 146 | void _set_receptive_field(const int new_receptive_field, const int input_buffer_size); 147 | void _set_receptive_field(const int new_receptive_field); 148 | void _reset_input_buffer(); 149 | // Use this->_input_post_gain 150 | virtual void _update_buffers_(NAM_SAMPLE* input, int num_frames); 151 | virtual void _rewind_buffers_(); 152 | }; 153 | 154 | // Basic linear model (an IR!) 155 | class Linear : public Buffer 156 | { 157 | public: 158 | Linear(const int receptive_field, const bool _bias, const std::vector& weights, 159 | const double expected_sample_rate = -1.0); 160 | void process(NAM_SAMPLE* input, NAM_SAMPLE* output, const int num_frames) override; 161 | 162 | protected: 163 | Eigen::VectorXf _weight; 164 | float _bias; 165 | }; 166 | 167 | // NN modules ================================================================= 168 | 169 | // TODO conv could take care of its own ring buffer. 170 | class Conv1D 171 | { 172 | public: 173 | Conv1D() { this->_dilation = 1; }; 174 | void set_weights_(std::vector::iterator& weights); 175 | void set_size_(const int in_channels, const int out_channels, const int kernel_size, const bool do_bias, 176 | const int _dilation); 177 | void set_size_and_weights_(const int in_channels, const int out_channels, const int kernel_size, const int _dilation, 178 | const bool do_bias, std::vector::iterator& weights); 179 | // Process from input to output 180 | // Rightmost indices of input go from i_start for ncols, 181 | // Indices on output for from j_start (to j_start + ncols - i_start) 182 | void process_(const Eigen::MatrixXf& input, Eigen::MatrixXf& output, const long i_start, const long ncols, 183 | const long j_start) const; 184 | long get_in_channels() const { return this->_weight.size() > 0 ? this->_weight[0].cols() : 0; }; 185 | long get_kernel_size() const { return this->_weight.size(); }; 186 | long get_num_weights() const; 187 | long get_out_channels() const { return this->_weight.size() > 0 ? this->_weight[0].rows() : 0; }; 188 | int get_dilation() const { return this->_dilation; }; 189 | 190 | private: 191 | // Gonna wing this... 192 | // conv[kernel](cout, cin) 193 | std::vector _weight; 194 | Eigen::VectorXf _bias; 195 | int _dilation; 196 | }; 197 | 198 | // Really just a linear layer 199 | class Conv1x1 200 | { 201 | public: 202 | Conv1x1(const int in_channels, const int out_channels, const bool _bias); 203 | Eigen::Block GetOutput(const int num_frames); 204 | void SetMaxBufferSize(const int maxBufferSize); 205 | void set_weights_(std::vector::iterator& weights); 206 | // :param input: (N,Cin) or (Cin,) 207 | // :return: (N,Cout) or (Cout,), respectively 208 | Eigen::MatrixXf process(const Eigen::MatrixXf& input) const { return process(input, (int)input.cols()); }; 209 | Eigen::MatrixXf process(const Eigen::MatrixXf& input, const int num_frames) const; 210 | // Store output to pre-allocated _output; access with GetOutput() 211 | void process_(const Eigen::MatrixXf& input, const int num_frames); 212 | 213 | long get_out_channels() const { return this->_weight.rows(); }; 214 | 215 | private: 216 | Eigen::MatrixXf _weight; 217 | Eigen::VectorXf _bias; 218 | Eigen::MatrixXf _output; 219 | bool _do_bias; 220 | }; 221 | 222 | // Utilities ================================================================== 223 | // Implemented in get_dsp.cpp 224 | 225 | // Data for a DSP object 226 | // :param version: Data version. Follows the conventions established in the trainer code. 227 | // :param architecture: Defines the high-level architecture. Supported are (as per `get-dsp()` in get_dsp.cpp): 228 | // * "CatLSTM" 229 | // * "CatWaveNet" 230 | // * "ConvNet" 231 | // * "LSTM" 232 | // * "Linear" 233 | // * "WaveNet" 234 | // :param config: 235 | // :param metadata: 236 | // :param weights: The model weights 237 | // :param expected_sample_rate: Most NAM models implicitly assume that data will be provided to them at some sample 238 | // rate. This captures it for other components interfacing with the model to understand its needs. Use -1.0 for "I 239 | // don't know". 240 | struct dspData 241 | { 242 | std::string version; 243 | std::string architecture; 244 | nlohmann::json config; 245 | nlohmann::json metadata; 246 | std::vector weights; 247 | double expected_sample_rate; 248 | }; 249 | 250 | // Verify that the config that we are building our model from is supported by 251 | // this plugin version. 252 | void verify_config_version(const std::string version); 253 | 254 | // Takes the model file and uses it to instantiate an instance of DSP. 255 | std::unique_ptr get_dsp(const std::filesystem::path model_file); 256 | // Creates an instance of DSP. Also returns a dspData struct that holds the data of the model. 257 | std::unique_ptr get_dsp(const std::filesystem::path model_file, dspData& returnedConfig); 258 | // Instantiates a DSP object from dsp_config struct. 259 | std::unique_ptr get_dsp(dspData& conf); 260 | // Legacy loader for directory-type DSPs 261 | std::unique_ptr get_dsp_legacy(const std::filesystem::path dirname); 262 | }; // namespace nam 263 | -------------------------------------------------------------------------------- /NAM/get_dsp.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include "dsp.h" 7 | #include "json.hpp" 8 | #include "lstm.h" 9 | #include "convnet.h" 10 | #include "wavenet.h" 11 | 12 | namespace nam 13 | { 14 | struct Version 15 | { 16 | int major; 17 | int minor; 18 | int patch; 19 | }; 20 | 21 | Version ParseVersion(const std::string& versionStr) 22 | { 23 | Version version; 24 | 25 | // Split the version string into major, minor, and patch components 26 | std::stringstream ss(versionStr); 27 | std::string majorStr, minorStr, patchStr; 28 | std::getline(ss, majorStr, '.'); 29 | std::getline(ss, minorStr, '.'); 30 | std::getline(ss, patchStr); 31 | 32 | // Parse the components as integers and assign them to the version struct 33 | try 34 | { 35 | version.major = std::stoi(majorStr); 36 | version.minor = std::stoi(minorStr); 37 | version.patch = std::stoi(patchStr); 38 | } 39 | catch (const std::invalid_argument&) 40 | { 41 | throw std::invalid_argument("Invalid version string: " + versionStr); 42 | } 43 | catch (const std::out_of_range&) 44 | { 45 | throw std::out_of_range("Version string out of range: " + versionStr); 46 | } 47 | 48 | // Validate the semver components 49 | if (version.major < 0 || version.minor < 0 || version.patch < 0) 50 | { 51 | throw std::invalid_argument("Negative version component: " + versionStr); 52 | } 53 | return version; 54 | } 55 | 56 | void verify_config_version(const std::string versionStr) 57 | { 58 | Version version = ParseVersion(versionStr); 59 | if (version.major != 0 || version.minor != 5) 60 | { 61 | std::stringstream ss; 62 | ss << "Model config is an unsupported version " << versionStr 63 | << ". Try either converting the model to a more recent version, or " 64 | "update your version of the NAM plugin."; 65 | throw std::runtime_error(ss.str()); 66 | } 67 | } 68 | 69 | std::vector GetWeights(nlohmann::json const& j) 70 | { 71 | auto it = j.find("weights"); 72 | if (it != j.end()) 73 | { 74 | return *it; 75 | } 76 | else 77 | throw std::runtime_error("Corrupted model file is missing weights."); 78 | } 79 | 80 | std::unique_ptr get_dsp(const std::filesystem::path config_filename) 81 | { 82 | dspData temp; 83 | return get_dsp(config_filename, temp); 84 | } 85 | 86 | std::unique_ptr get_dsp(const std::filesystem::path config_filename, dspData& returnedConfig) 87 | { 88 | if (!std::filesystem::exists(config_filename)) 89 | throw std::runtime_error("Config file doesn't exist!\n"); 90 | std::ifstream i(config_filename); 91 | nlohmann::json j; 92 | i >> j; 93 | verify_config_version(j["version"]); 94 | 95 | auto architecture = j["architecture"]; 96 | nlohmann::json config = j["config"]; 97 | std::vector weights = GetWeights(j); 98 | 99 | // Assign values to returnedConfig 100 | returnedConfig.version = j["version"]; 101 | returnedConfig.architecture = j["architecture"]; 102 | returnedConfig.config = j["config"]; 103 | returnedConfig.metadata = j["metadata"]; 104 | returnedConfig.weights = weights; 105 | if (j.find("sample_rate") != j.end()) 106 | returnedConfig.expected_sample_rate = j["sample_rate"]; 107 | else 108 | { 109 | returnedConfig.expected_sample_rate = -1.0; 110 | } 111 | 112 | /*Copy to a new dsp_config object for get_dsp below, 113 | since not sure if weights actually get modified as being non-const references on some 114 | model constructors inside get_dsp(dsp_config& conf). 115 | We need to return unmodified version of dsp_config via returnedConfig.*/ 116 | dspData conf = returnedConfig; 117 | 118 | return get_dsp(conf); 119 | } 120 | 121 | struct OptionalValue 122 | { 123 | bool have = false; 124 | double value = 0.0; 125 | }; 126 | 127 | std::unique_ptr get_dsp(dspData& conf) 128 | { 129 | verify_config_version(conf.version); 130 | 131 | auto& architecture = conf.architecture; 132 | nlohmann::json& config = conf.config; 133 | std::vector& weights = conf.weights; 134 | OptionalValue loudness, inputLevel, outputLevel; 135 | 136 | auto AssignOptional = [&conf](const std::string key, OptionalValue& v) { 137 | if (conf.metadata.find(key) != conf.metadata.end()) 138 | { 139 | if (!conf.metadata[key].is_null()) 140 | { 141 | v.value = conf.metadata[key]; 142 | v.have = true; 143 | } 144 | } 145 | }; 146 | 147 | if (!conf.metadata.is_null()) 148 | { 149 | AssignOptional("loudness", loudness); 150 | AssignOptional("input_level_dbu", inputLevel); 151 | AssignOptional("output_level_dbu", outputLevel); 152 | } 153 | const double expectedSampleRate = conf.expected_sample_rate; 154 | 155 | std::unique_ptr out = nullptr; 156 | if (architecture == "Linear") 157 | { 158 | const int receptive_field = config["receptive_field"]; 159 | const bool _bias = config["bias"]; 160 | out = std::make_unique(receptive_field, _bias, weights, expectedSampleRate); 161 | } 162 | else if (architecture == "ConvNet") 163 | { 164 | const int channels = config["channels"]; 165 | const bool batchnorm = config["batchnorm"]; 166 | std::vector dilations = config["dilations"]; 167 | const std::string activation = config["activation"]; 168 | out = std::make_unique(channels, dilations, batchnorm, activation, weights, expectedSampleRate); 169 | } 170 | else if (architecture == "LSTM") 171 | { 172 | const int num_layers = config["num_layers"]; 173 | const int input_size = config["input_size"]; 174 | const int hidden_size = config["hidden_size"]; 175 | out = std::make_unique(num_layers, input_size, hidden_size, weights, expectedSampleRate); 176 | } 177 | else if (architecture == "WaveNet") 178 | { 179 | std::vector layer_array_params; 180 | for (size_t i = 0; i < config["layers"].size(); i++) 181 | { 182 | nlohmann::json layer_config = config["layers"][i]; 183 | layer_array_params.push_back( 184 | wavenet::LayerArrayParams(layer_config["input_size"], layer_config["condition_size"], layer_config["head_size"], 185 | layer_config["channels"], layer_config["kernel_size"], layer_config["dilations"], 186 | layer_config["activation"], layer_config["gated"], layer_config["head_bias"])); 187 | } 188 | const bool with_head = !config["head"].is_null(); 189 | const float head_scale = config["head_scale"]; 190 | out = std::make_unique(layer_array_params, head_scale, with_head, weights, expectedSampleRate); 191 | } 192 | else 193 | { 194 | throw std::runtime_error("Unrecognized architecture"); 195 | } 196 | if (loudness.have) 197 | { 198 | out->SetLoudness(loudness.value); 199 | } 200 | if (inputLevel.have) 201 | { 202 | out->SetInputLevel(inputLevel.value); 203 | } 204 | if (outputLevel.have) 205 | { 206 | out->SetOutputLevel(outputLevel.value); 207 | } 208 | 209 | // "pre-warm" the model to settle initial conditions 210 | // Can this be removed now that it's part of Reset()? 211 | out->prewarm(); 212 | 213 | return out; 214 | } 215 | }; // namespace nam 216 | -------------------------------------------------------------------------------- /NAM/get_dsp.h: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "dsp.h" 4 | 5 | namespace nam 6 | { 7 | // Get NAM from a .nam file at the provided location 8 | std::unique_ptr get_dsp(const std::filesystem::path config_filename); 9 | 10 | // Get NAM from a provided configuration struct 11 | std::unique_ptr get_dsp(dspData& conf); 12 | 13 | // Get NAM from a provided .nam file path and store its configuration in the provided conf 14 | std::unique_ptr get_dsp(const std::filesystem::path config_filename, dspData& returnedConfig); 15 | }; // namespace nam 16 | -------------------------------------------------------------------------------- /NAM/lstm.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include "lstm.h" 6 | 7 | nam::lstm::LSTMCell::LSTMCell(const int input_size, const int hidden_size, std::vector::iterator& weights) 8 | { 9 | // Resize arrays 10 | this->_w.resize(4 * hidden_size, input_size + hidden_size); 11 | this->_b.resize(4 * hidden_size); 12 | this->_xh.resize(input_size + hidden_size); 13 | this->_ifgo.resize(4 * hidden_size); 14 | this->_c.resize(hidden_size); 15 | 16 | // Assign in row-major because that's how PyTorch goes. 17 | for (int i = 0; i < this->_w.rows(); i++) 18 | for (int j = 0; j < this->_w.cols(); j++) 19 | this->_w(i, j) = *(weights++); 20 | for (int i = 0; i < this->_b.size(); i++) 21 | this->_b[i] = *(weights++); 22 | const int h_offset = input_size; 23 | for (int i = 0; i < hidden_size; i++) 24 | this->_xh[i + h_offset] = *(weights++); 25 | for (int i = 0; i < hidden_size; i++) 26 | this->_c[i] = *(weights++); 27 | } 28 | 29 | void nam::lstm::LSTMCell::process_(const Eigen::VectorXf& x) 30 | { 31 | const long hidden_size = this->_get_hidden_size(); 32 | const long input_size = this->_get_input_size(); 33 | // Assign inputs 34 | this->_xh(Eigen::seq(0, input_size - 1)) = x; 35 | // The matmul 36 | this->_ifgo = this->_w * this->_xh + this->_b; 37 | // Elementwise updates (apply nonlinearities here) 38 | const long i_offset = 0; 39 | const long f_offset = hidden_size; 40 | const long g_offset = 2 * hidden_size; 41 | const long o_offset = 3 * hidden_size; 42 | const long h_offset = input_size; 43 | 44 | if (activations::Activation::using_fast_tanh) 45 | { 46 | for (auto i = 0; i < hidden_size; i++) 47 | this->_c[i] = 48 | activations::fast_sigmoid(this->_ifgo[i + f_offset]) * this->_c[i] 49 | + activations::fast_sigmoid(this->_ifgo[i + i_offset]) * activations::fast_tanh(this->_ifgo[i + g_offset]); 50 | 51 | for (int i = 0; i < hidden_size; i++) 52 | this->_xh[i + h_offset] = 53 | activations::fast_sigmoid(this->_ifgo[i + o_offset]) * activations::fast_tanh(this->_c[i]); 54 | } 55 | else 56 | { 57 | for (auto i = 0; i < hidden_size; i++) 58 | this->_c[i] = activations::sigmoid(this->_ifgo[i + f_offset]) * this->_c[i] 59 | + activations::sigmoid(this->_ifgo[i + i_offset]) * tanhf(this->_ifgo[i + g_offset]); 60 | 61 | for (int i = 0; i < hidden_size; i++) 62 | this->_xh[i + h_offset] = activations::sigmoid(this->_ifgo[i + o_offset]) * tanhf(this->_c[i]); 63 | } 64 | } 65 | 66 | nam::lstm::LSTM::LSTM(const int num_layers, const int input_size, const int hidden_size, std::vector& weights, 67 | const double expected_sample_rate) 68 | : DSP(expected_sample_rate) 69 | { 70 | this->_input.resize(1); 71 | std::vector::iterator it = weights.begin(); 72 | for (int i = 0; i < num_layers; i++) 73 | this->_layers.push_back(LSTMCell(i == 0 ? input_size : hidden_size, hidden_size, it)); 74 | this->_head_weight.resize(hidden_size); 75 | for (int i = 0; i < hidden_size; i++) 76 | this->_head_weight[i] = *(it++); 77 | this->_head_bias = *(it++); 78 | assert(it == weights.end()); 79 | } 80 | 81 | void nam::lstm::LSTM::process(NAM_SAMPLE* input, NAM_SAMPLE* output, const int num_frames) 82 | { 83 | for (int i = 0; i < num_frames; i++) 84 | output[i] = this->_process_sample(input[i]); 85 | } 86 | 87 | int nam::lstm::LSTM::PrewarmSamples() 88 | { 89 | int result = (int)(0.5 * mExpectedSampleRate); 90 | // If the expected sample rate wasn't provided, it'll be -1. 91 | // Make sure something still happens. 92 | return result <= 0 ? 1 : result; 93 | } 94 | 95 | float nam::lstm::LSTM::_process_sample(const float x) 96 | { 97 | if (this->_layers.size() == 0) 98 | return x; 99 | this->_input(0) = x; 100 | this->_layers[0].process_(this->_input); 101 | for (size_t i = 1; i < this->_layers.size(); i++) 102 | this->_layers[i].process_(this->_layers[i - 1].get_hidden_state()); 103 | return this->_head_weight.dot(this->_layers[this->_layers.size() - 1].get_hidden_state()) + this->_head_bias; 104 | } 105 | -------------------------------------------------------------------------------- /NAM/lstm.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | // LSTM implementation 3 | 4 | #include 5 | #include 6 | 7 | #include 8 | 9 | #include "dsp.h" 10 | 11 | namespace nam 12 | { 13 | namespace lstm 14 | { 15 | // A Single LSTM cell 16 | // i input 17 | // f forget 18 | // g cell 19 | // o output 20 | // c cell state 21 | // h hidden state 22 | class LSTMCell 23 | { 24 | public: 25 | LSTMCell(const int input_size, const int hidden_size, std::vector::iterator& weights); 26 | Eigen::VectorXf get_hidden_state() const { return this->_xh(Eigen::placeholders::lastN(this->_get_hidden_size())); }; 27 | void process_(const Eigen::VectorXf& x); 28 | 29 | private: 30 | // Parameters 31 | // xh -> ifgo 32 | // (dx+dh) -> (4*dh) 33 | Eigen::MatrixXf _w; 34 | Eigen::VectorXf _b; 35 | 36 | // State 37 | // Concatenated input and hidden state 38 | Eigen::VectorXf _xh; 39 | // Input, Forget, Cell, Output gates 40 | Eigen::VectorXf _ifgo; 41 | 42 | // Cell state 43 | Eigen::VectorXf _c; 44 | 45 | long _get_hidden_size() const { return this->_b.size() / 4; }; 46 | long _get_input_size() const { return this->_xh.size() - this->_get_hidden_size(); }; 47 | }; 48 | 49 | // The multi-layer LSTM model 50 | class LSTM : public DSP 51 | { 52 | public: 53 | LSTM(const int num_layers, const int input_size, const int hidden_size, std::vector& weights, 54 | const double expected_sample_rate = -1.0); 55 | ~LSTM() = default; 56 | 57 | protected: 58 | // Hacky, but a half-second seems to work for most models. 59 | int PrewarmSamples() override; 60 | 61 | Eigen::VectorXf _head_weight; 62 | float _head_bias; 63 | void process(NAM_SAMPLE* input, NAM_SAMPLE* output, const int num_frames) override; 64 | std::vector _layers; 65 | 66 | float _process_sample(const float x); 67 | 68 | // Input to the LSTM. 69 | // Since this is assumed to not be a parametric model, its shape should be (1,) 70 | Eigen::VectorXf _input; 71 | }; 72 | }; // namespace lstm 73 | }; // namespace nam 74 | -------------------------------------------------------------------------------- /NAM/util.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "util.h" 5 | 6 | std::string nam::util::lowercase(const std::string& s) 7 | { 8 | std::string out(s); 9 | std::transform(s.begin(), s.end(), out.begin(), [](unsigned char c) { return std::tolower(c); }); 10 | return out; 11 | } 12 | -------------------------------------------------------------------------------- /NAM/util.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | // Utilities 4 | 5 | #include 6 | #include // Eigen::MatrixXf 7 | 8 | namespace nam 9 | { 10 | namespace util 11 | { 12 | std::string lowercase(const std::string& s); 13 | }; // namespace util 14 | }; // namespace nam 15 | -------------------------------------------------------------------------------- /NAM/version.h: -------------------------------------------------------------------------------- 1 | #ifndef version_h 2 | #define version_h 3 | 4 | // Make sure this matches NAM version in ../CMakeLists.txt! 5 | #define NEURAL_AMP_MODELER_DSP_VERSION_MAJOR 0 6 | #define NEURAL_AMP_MODELER_DSP_VERSION_MINOR 3 7 | #define NEURAL_AMP_MODELER_DSP_VERSION_PATCH 0 8 | 9 | #endif 10 | -------------------------------------------------------------------------------- /NAM/wavenet.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include 6 | 7 | #include "wavenet.h" 8 | 9 | nam::wavenet::_DilatedConv::_DilatedConv(const int in_channels, const int out_channels, const int kernel_size, 10 | const int bias, const int dilation) 11 | { 12 | this->set_size_(in_channels, out_channels, kernel_size, bias, dilation); 13 | } 14 | 15 | // Layer ====================================================================== 16 | 17 | void nam::wavenet::_Layer::SetMaxBufferSize(const int maxBufferSize) 18 | { 19 | _input_mixin.SetMaxBufferSize(maxBufferSize); 20 | _z.resize(get_channels(), maxBufferSize); 21 | _1x1.SetMaxBufferSize(maxBufferSize); 22 | } 23 | 24 | void nam::wavenet::_Layer::set_weights_(std::vector::iterator& weights) 25 | { 26 | this->_conv.set_weights_(weights); 27 | this->_input_mixin.set_weights_(weights); 28 | this->_1x1.set_weights_(weights); 29 | } 30 | 31 | void nam::wavenet::_Layer::process_(const Eigen::MatrixXf& input, const Eigen::MatrixXf& condition, 32 | Eigen::MatrixXf& head_input, Eigen::MatrixXf& output, const long i_start, 33 | const long j_start, const int num_frames) 34 | { 35 | const long ncols = (long)num_frames; // TODO clean this up 36 | const long channels = this->get_channels(); 37 | // Input dilated conv 38 | this->_conv.process_(input, this->_z, i_start, ncols, 0); 39 | // Mix-in condition 40 | _input_mixin.process_(condition, num_frames); 41 | this->_z.leftCols(num_frames).noalias() += _input_mixin.GetOutput(num_frames); 42 | 43 | if (!this->_gated) 44 | { 45 | this->_activation->apply(this->_z.leftCols(num_frames)); 46 | } 47 | else 48 | { 49 | // CAREFUL: .topRows() and .bottomRows() won't be memory-contiguous for a column-major matrix (Issue 125). Need to 50 | // do this column-wise: 51 | for (int i = 0; i < num_frames; i++) 52 | { 53 | this->_activation->apply(this->_z.block(0, i, channels, 1)); 54 | activations::Activation::get_activation("Sigmoid")->apply(this->_z.block(channels, i, channels, 1)); 55 | } 56 | this->_z.block(0, 0, channels, num_frames).array() *= this->_z.block(channels, 0, channels, num_frames).array(); 57 | } 58 | 59 | head_input.leftCols(num_frames).noalias() += this->_z.block(0, 0, channels, num_frames); 60 | if (!_gated) 61 | { 62 | _1x1.process_(_z, num_frames); 63 | } 64 | else 65 | { 66 | // Probably not RT-safe yet 67 | _1x1.process_(_z.topRows(channels), num_frames); 68 | } 69 | output.middleCols(j_start, ncols).noalias() = input.middleCols(i_start, ncols) + _1x1.GetOutput(num_frames); 70 | } 71 | 72 | void nam::wavenet::_Layer::set_num_frames_(const long num_frames) 73 | { 74 | if (this->_z.rows() == this->_conv.get_out_channels() && this->_z.cols() == num_frames) 75 | return; // Already has correct size 76 | 77 | this->_z.resize(this->_conv.get_out_channels(), num_frames); 78 | this->_z.setZero(); 79 | } 80 | 81 | // LayerArray ================================================================= 82 | 83 | #define LAYER_ARRAY_BUFFER_SIZE 65536 84 | 85 | nam::wavenet::_LayerArray::_LayerArray(const int input_size, const int condition_size, const int head_size, 86 | const int channels, const int kernel_size, const std::vector& dilations, 87 | const std::string activation, const bool gated, const bool head_bias) 88 | : _rechannel(input_size, channels, false) 89 | , _head_rechannel(channels, head_size, head_bias) 90 | { 91 | for (size_t i = 0; i < dilations.size(); i++) 92 | this->_layers.push_back(_Layer(condition_size, channels, kernel_size, dilations[i], activation, gated)); 93 | const long receptive_field = this->_get_receptive_field(); 94 | for (size_t i = 0; i < dilations.size(); i++) 95 | { 96 | this->_layer_buffers.push_back(Eigen::MatrixXf(channels, LAYER_ARRAY_BUFFER_SIZE + receptive_field - 1)); 97 | this->_layer_buffers[i].setZero(); 98 | } 99 | this->_buffer_start = this->_get_receptive_field() - 1; 100 | } 101 | 102 | void nam::wavenet::_LayerArray::SetMaxBufferSize(const int maxBufferSize) 103 | { 104 | _rechannel.SetMaxBufferSize(maxBufferSize); 105 | _head_rechannel.SetMaxBufferSize(maxBufferSize); 106 | for (auto it = _layers.begin(); it != _layers.end(); ++it) 107 | { 108 | it->SetMaxBufferSize(maxBufferSize); 109 | } 110 | } 111 | 112 | void nam::wavenet::_LayerArray::advance_buffers_(const int num_frames) 113 | { 114 | this->_buffer_start += num_frames; 115 | } 116 | 117 | long nam::wavenet::_LayerArray::get_receptive_field() const 118 | { 119 | long result = 0; 120 | for (size_t i = 0; i < this->_layers.size(); i++) 121 | result += this->_layers[i].get_dilation() * (this->_layers[i].get_kernel_size() - 1); 122 | return result; 123 | } 124 | 125 | void nam::wavenet::_LayerArray::prepare_for_frames_(const long num_frames) 126 | { 127 | // Example: 128 | // _buffer_start = 0 129 | // num_frames = 64 130 | // buffer_size = 64 131 | // -> this will write on indices 0 through 63, inclusive. 132 | // -> No illegal writes. 133 | // -> no rewind needed. 134 | if (this->_buffer_start + num_frames > this->_get_buffer_size()) 135 | this->_rewind_buffers_(); 136 | } 137 | 138 | void nam::wavenet::_LayerArray::process_(const Eigen::MatrixXf& layer_inputs, const Eigen::MatrixXf& condition, 139 | Eigen::MatrixXf& head_inputs, Eigen::MatrixXf& layer_outputs, 140 | Eigen::MatrixXf& head_outputs, const int num_frames) 141 | { 142 | this->_rechannel.process_(layer_inputs, num_frames); 143 | this->_layer_buffers[0].middleCols(this->_buffer_start, num_frames) = _rechannel.GetOutput(num_frames); 144 | const size_t last_layer = this->_layers.size() - 1; 145 | for (size_t i = 0; i < this->_layers.size(); i++) 146 | { 147 | this->_layers[i].process_(this->_layer_buffers[i], condition, head_inputs, 148 | i == last_layer ? layer_outputs : this->_layer_buffers[i + 1], this->_buffer_start, 149 | i == last_layer ? 0 : this->_buffer_start, num_frames); 150 | } 151 | _head_rechannel.process_(head_inputs, num_frames); 152 | head_outputs.leftCols(num_frames) = _head_rechannel.GetOutput(num_frames); 153 | } 154 | 155 | void nam::wavenet::_LayerArray::set_num_frames_(const long num_frames) 156 | { 157 | // Wavenet checks for unchanged num_frames; if we made it here, there's 158 | // something to do. 159 | if (LAYER_ARRAY_BUFFER_SIZE - num_frames < this->_get_receptive_field()) 160 | { 161 | std::stringstream ss; 162 | ss << "Asked to accept a buffer of " << num_frames << " samples, but the buffer is too short (" 163 | << LAYER_ARRAY_BUFFER_SIZE << ") to get out of the recptive field (" << this->_get_receptive_field() 164 | << "); copy errors could occur!\n"; 165 | throw std::runtime_error(ss.str().c_str()); 166 | } 167 | for (size_t i = 0; i < this->_layers.size(); i++) 168 | this->_layers[i].set_num_frames_(num_frames); 169 | } 170 | 171 | void nam::wavenet::_LayerArray::set_weights_(std::vector::iterator& weights) 172 | { 173 | this->_rechannel.set_weights_(weights); 174 | for (size_t i = 0; i < this->_layers.size(); i++) 175 | this->_layers[i].set_weights_(weights); 176 | this->_head_rechannel.set_weights_(weights); 177 | } 178 | 179 | long nam::wavenet::_LayerArray::_get_channels() const 180 | { 181 | return this->_layers.size() > 0 ? this->_layers[0].get_channels() : 0; 182 | } 183 | 184 | long nam::wavenet::_LayerArray::_get_receptive_field() const 185 | { 186 | // TODO remove this and use get_receptive_field() instead! 187 | long res = 1; 188 | for (size_t i = 0; i < this->_layers.size(); i++) 189 | res += (this->_layers[i].get_kernel_size() - 1) * this->_layers[i].get_dilation(); 190 | return res; 191 | } 192 | 193 | void nam::wavenet::_LayerArray::_rewind_buffers_() 194 | // Consider wrapping instead... 195 | // Can make this smaller--largest dilation, not receptive field! 196 | { 197 | const long start = this->_get_receptive_field() - 1; 198 | for (size_t i = 0; i < this->_layer_buffers.size(); i++) 199 | { 200 | const long d = (this->_layers[i].get_kernel_size() - 1) * this->_layers[i].get_dilation(); 201 | this->_layer_buffers[i].middleCols(start - d, d) = this->_layer_buffers[i].middleCols(this->_buffer_start - d, d); 202 | } 203 | this->_buffer_start = start; 204 | } 205 | 206 | // Head ======================================================================= 207 | 208 | nam::wavenet::_Head::_Head(const int input_size, const int num_layers, const int channels, const std::string activation) 209 | : _channels(channels) 210 | , _head(num_layers > 0 ? channels : input_size, 1, true) 211 | , _activation(activations::Activation::get_activation(activation)) 212 | { 213 | assert(num_layers > 0); 214 | int dx = input_size; 215 | for (int i = 0; i < num_layers; i++) 216 | { 217 | this->_layers.push_back(Conv1x1(dx, i == num_layers - 1 ? 1 : channels, true)); 218 | dx = channels; 219 | if (i < num_layers - 1) 220 | this->_buffers.push_back(Eigen::MatrixXf()); 221 | } 222 | } 223 | 224 | void nam::wavenet::_Head::Reset(const double sampleRate, const int maxBufferSize) 225 | { 226 | set_num_frames_((long)maxBufferSize); 227 | } 228 | 229 | void nam::wavenet::_Head::set_weights_(std::vector::iterator& weights) 230 | { 231 | for (size_t i = 0; i < this->_layers.size(); i++) 232 | this->_layers[i].set_weights_(weights); 233 | } 234 | 235 | void nam::wavenet::_Head::process_(Eigen::MatrixXf& inputs, Eigen::MatrixXf& outputs) 236 | { 237 | const size_t num_layers = this->_layers.size(); 238 | this->_apply_activation_(inputs); 239 | if (num_layers == 1) 240 | outputs = this->_layers[0].process(inputs); 241 | else 242 | { 243 | this->_buffers[0] = this->_layers[0].process(inputs); 244 | for (size_t i = 1; i < num_layers; i++) 245 | { // Asserted > 0 layers 246 | this->_apply_activation_(this->_buffers[i - 1]); 247 | if (i < num_layers - 1) 248 | this->_buffers[i] = this->_layers[i].process(this->_buffers[i - 1]); 249 | else 250 | outputs = this->_layers[i].process(this->_buffers[i - 1]); 251 | } 252 | } 253 | } 254 | 255 | void nam::wavenet::_Head::set_num_frames_(const long num_frames) 256 | { 257 | for (size_t i = 0; i < this->_buffers.size(); i++) 258 | { 259 | if (this->_buffers[i].rows() == this->_channels && this->_buffers[i].cols() == num_frames) 260 | continue; // Already has correct size 261 | this->_buffers[i].resize(this->_channels, num_frames); 262 | this->_buffers[i].setZero(); // Shouldn't be needed--these are written to before they're used. 263 | } 264 | } 265 | 266 | void nam::wavenet::_Head::_apply_activation_(Eigen::MatrixXf& x) 267 | { 268 | this->_activation->apply(x); 269 | } 270 | 271 | // WaveNet ==================================================================== 272 | 273 | nam::wavenet::WaveNet::WaveNet(const std::vector& layer_array_params, 274 | const float head_scale, const bool with_head, std::vector weights, 275 | const double expected_sample_rate) 276 | : DSP(expected_sample_rate) 277 | , _head_scale(head_scale) 278 | { 279 | if (with_head) 280 | throw std::runtime_error("Head not implemented!"); 281 | for (size_t i = 0; i < layer_array_params.size(); i++) 282 | { 283 | this->_layer_arrays.push_back(nam::wavenet::_LayerArray( 284 | layer_array_params[i].input_size, layer_array_params[i].condition_size, layer_array_params[i].head_size, 285 | layer_array_params[i].channels, layer_array_params[i].kernel_size, layer_array_params[i].dilations, 286 | layer_array_params[i].activation, layer_array_params[i].gated, layer_array_params[i].head_bias)); 287 | this->_layer_array_outputs.push_back(Eigen::MatrixXf(layer_array_params[i].channels, 0)); 288 | if (i == 0) 289 | this->_head_arrays.push_back(Eigen::MatrixXf(layer_array_params[i].channels, 0)); 290 | if (i > 0) 291 | if (layer_array_params[i].channels != layer_array_params[i - 1].head_size) 292 | { 293 | std::stringstream ss; 294 | ss << "channels of layer " << i << " (" << layer_array_params[i].channels 295 | << ") doesn't match head_size of preceding layer (" << layer_array_params[i - 1].head_size << "!\n"; 296 | throw std::runtime_error(ss.str().c_str()); 297 | } 298 | this->_head_arrays.push_back(Eigen::MatrixXf(layer_array_params[i].head_size, 0)); 299 | } 300 | this->_head_output.resize(1, 0); // Mono output! 301 | this->set_weights_(weights); 302 | 303 | mPrewarmSamples = 1; 304 | for (size_t i = 0; i < this->_layer_arrays.size(); i++) 305 | mPrewarmSamples += this->_layer_arrays[i].get_receptive_field(); 306 | } 307 | 308 | void nam::wavenet::WaveNet::set_weights_(std::vector& weights) 309 | { 310 | std::vector::iterator it = weights.begin(); 311 | for (size_t i = 0; i < this->_layer_arrays.size(); i++) 312 | this->_layer_arrays[i].set_weights_(it); 313 | // this->_head.set_params_(it); 314 | this->_head_scale = *(it++); 315 | if (it != weights.end()) 316 | { 317 | std::stringstream ss; 318 | for (size_t i = 0; i < weights.size(); i++) 319 | if (weights[i] == *it) 320 | { 321 | ss << "Weight mismatch: assigned " << i + 1 << " weights, but " << weights.size() << " were provided."; 322 | throw std::runtime_error(ss.str().c_str()); 323 | } 324 | ss << "Weight mismatch: provided " << weights.size() << " weights, but the model expects more."; 325 | throw std::runtime_error(ss.str().c_str()); 326 | } 327 | } 328 | 329 | void nam::wavenet::WaveNet::SetMaxBufferSize(const int maxBufferSize) 330 | { 331 | DSP::SetMaxBufferSize(maxBufferSize); 332 | 333 | this->_condition.resize(this->_get_condition_dim(), maxBufferSize); 334 | for (size_t i = 0; i < this->_head_arrays.size(); i++) 335 | this->_head_arrays[i].resize(this->_head_arrays[i].rows(), maxBufferSize); 336 | for (size_t i = 0; i < this->_layer_array_outputs.size(); i++) 337 | this->_layer_array_outputs[i].resize(this->_layer_array_outputs[i].rows(), maxBufferSize); 338 | this->_head_output.resize(this->_head_output.rows(), maxBufferSize); 339 | this->_head_output.setZero(); 340 | 341 | for (size_t i = 0; i < this->_layer_arrays.size(); i++) 342 | this->_layer_arrays[i].SetMaxBufferSize(maxBufferSize); 343 | // this->_head.SetMaxBufferSize(maxBufferSize); 344 | } 345 | 346 | void nam::wavenet::WaveNet::_advance_buffers_(const int num_frames) 347 | { 348 | for (size_t i = 0; i < this->_layer_arrays.size(); i++) 349 | this->_layer_arrays[i].advance_buffers_(num_frames); 350 | } 351 | 352 | void nam::wavenet::WaveNet::_prepare_for_frames_(const long num_frames) 353 | { 354 | for (size_t i = 0; i < this->_layer_arrays.size(); i++) 355 | this->_layer_arrays[i].prepare_for_frames_(num_frames); 356 | } 357 | 358 | void nam::wavenet::WaveNet::_set_condition_array(NAM_SAMPLE* input, const int num_frames) 359 | { 360 | for (int j = 0; j < num_frames; j++) 361 | { 362 | this->_condition(0, j) = input[j]; 363 | } 364 | } 365 | 366 | void nam::wavenet::WaveNet::process(NAM_SAMPLE* input, NAM_SAMPLE* output, const int num_frames) 367 | { 368 | assert(num_frames <= mMaxBufferSize); 369 | this->_prepare_for_frames_(num_frames); 370 | this->_set_condition_array(input, num_frames); 371 | 372 | // Main layer arrays: 373 | // Layer-to-layer 374 | // Sum on head output 375 | this->_head_arrays[0].setZero(); 376 | for (size_t i = 0; i < this->_layer_arrays.size(); i++) 377 | this->_layer_arrays[i].process_(i == 0 ? this->_condition : this->_layer_array_outputs[i - 1], this->_condition, 378 | this->_head_arrays[i], this->_layer_array_outputs[i], this->_head_arrays[i + 1], 379 | num_frames); 380 | // this->_head.process_( 381 | // this->_head_input, 382 | // this->_head_output 383 | //); 384 | // Copy to required output array 385 | // Hack: apply head scale here; revisit when/if I activate the head. 386 | // assert(this->_head_output.rows() == 1); 387 | 388 | const long final_head_array = this->_head_arrays.size() - 1; 389 | assert(this->_head_arrays[final_head_array].rows() == 1); 390 | for (int s = 0; s < num_frames; s++) 391 | { 392 | const float out = this->_head_scale * this->_head_arrays[final_head_array](0, s); 393 | output[s] = out; 394 | } 395 | 396 | // Finalize to prepare for the next call: 397 | this->_advance_buffers_(num_frames); 398 | } 399 | -------------------------------------------------------------------------------- /NAM/wavenet.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | #include "json.hpp" 7 | #include 8 | 9 | #include "dsp.h" 10 | 11 | namespace nam 12 | { 13 | namespace wavenet 14 | { 15 | // Rework the initialization API slightly. Merge w/ dsp.h later. 16 | class _DilatedConv : public Conv1D 17 | { 18 | public: 19 | _DilatedConv(const int in_channels, const int out_channels, const int kernel_size, const int bias, 20 | const int dilation); 21 | }; 22 | 23 | class _Layer 24 | { 25 | public: 26 | _Layer(const int condition_size, const int channels, const int kernel_size, const int dilation, 27 | const std::string activation, const bool gated) 28 | : _conv(channels, gated ? 2 * channels : channels, kernel_size, true, dilation) 29 | , _input_mixin(condition_size, gated ? 2 * channels : channels, false) 30 | , _1x1(channels, channels, true) 31 | , _activation(activations::Activation::get_activation(activation)) 32 | , _gated(gated) {}; 33 | void SetMaxBufferSize(const int maxBufferSize); 34 | void set_weights_(std::vector::iterator& weights); 35 | // :param `input`: from previous layer 36 | // :param `output`: to next layer 37 | void process_(const Eigen::MatrixXf& input, const Eigen::MatrixXf& condition, Eigen::MatrixXf& head_input, 38 | Eigen::MatrixXf& output, const long i_start, const long j_start, const int num_frames); 39 | void set_num_frames_(const long num_frames); 40 | long get_channels() const { return this->_conv.get_in_channels(); }; 41 | int get_dilation() const { return this->_conv.get_dilation(); }; 42 | long get_kernel_size() const { return this->_conv.get_kernel_size(); }; 43 | 44 | private: 45 | // The dilated convolution at the front of the block 46 | _DilatedConv _conv; 47 | // Input mixin 48 | Conv1x1 _input_mixin; 49 | // The post-activation 1x1 convolution 50 | Conv1x1 _1x1; 51 | // The internal state 52 | Eigen::MatrixXf _z; 53 | 54 | activations::Activation* _activation; 55 | const bool _gated; 56 | }; 57 | 58 | class LayerArrayParams 59 | { 60 | public: 61 | LayerArrayParams(const int input_size_, const int condition_size_, const int head_size_, const int channels_, 62 | const int kernel_size_, const std::vector&& dilations_, const std::string activation_, 63 | const bool gated_, const bool head_bias_) 64 | : input_size(input_size_) 65 | , condition_size(condition_size_) 66 | , head_size(head_size_) 67 | , channels(channels_) 68 | , kernel_size(kernel_size_) 69 | , dilations(std::move(dilations_)) 70 | , activation(activation_) 71 | , gated(gated_) 72 | , head_bias(head_bias_) 73 | { 74 | } 75 | 76 | const int input_size; 77 | const int condition_size; 78 | const int head_size; 79 | const int channels; 80 | const int kernel_size; 81 | std::vector dilations; 82 | const std::string activation; 83 | const bool gated; 84 | const bool head_bias; 85 | }; 86 | 87 | // An array of layers with the same channels, kernel sizes, activations. 88 | class _LayerArray 89 | { 90 | public: 91 | _LayerArray(const int input_size, const int condition_size, const int head_size, const int channels, 92 | const int kernel_size, const std::vector& dilations, const std::string activation, const bool gated, 93 | const bool head_bias); 94 | 95 | void SetMaxBufferSize(const int maxBufferSize); 96 | 97 | void advance_buffers_(const int num_frames); 98 | 99 | // Preparing for frames: 100 | // Rewind buffers if needed 101 | // Shift index to prepare 102 | // 103 | void prepare_for_frames_(const long num_frames); 104 | 105 | // All arrays are "short". 106 | void process_(const Eigen::MatrixXf& layer_inputs, // Short 107 | const Eigen::MatrixXf& condition, // Short 108 | Eigen::MatrixXf& layer_outputs, // Short 109 | Eigen::MatrixXf& head_inputs, // Sum up on this. 110 | Eigen::MatrixXf& head_outputs, // post head-rechannel 111 | const int num_frames); 112 | void set_num_frames_(const long num_frames); 113 | void set_weights_(std::vector::iterator& it); 114 | 115 | // "Zero-indexed" receptive field. 116 | // E.g. a 1x1 convolution has a z.i.r.f. of zero. 117 | long get_receptive_field() const; 118 | 119 | private: 120 | long _buffer_start; 121 | // The rechannel before the layers 122 | Conv1x1 _rechannel; 123 | 124 | // Buffers in between layers. 125 | // buffer [i] is the input to layer [i]. 126 | // the last layer outputs to a short array provided by outside. 127 | std::vector _layer_buffers; 128 | // The layer objects 129 | std::vector<_Layer> _layers; 130 | 131 | // Rechannel for the head 132 | Conv1x1 _head_rechannel; 133 | 134 | long _get_buffer_size() const { return this->_layer_buffers.size() > 0 ? this->_layer_buffers[0].cols() : 0; }; 135 | long _get_channels() const; 136 | // "One-indexed" receptive field 137 | // TODO remove! 138 | // E.g. a 1x1 convolution has a o.i.r.f. of one. 139 | long _get_receptive_field() const; 140 | void _rewind_buffers_(); 141 | }; 142 | 143 | // The head module 144 | // [Act->Conv] x L 145 | class _Head 146 | { 147 | public: 148 | _Head(const int input_size, const int num_layers, const int channels, const std::string activation); 149 | void Reset(const double sampleRate, const int maxBufferSize); 150 | void set_weights_(std::vector::iterator& weights); 151 | // NOTE: the head transforms the provided input by applying a nonlinearity 152 | // to it in-place! 153 | void process_(Eigen::MatrixXf& inputs, Eigen::MatrixXf& outputs); 154 | void set_num_frames_(const long num_frames); 155 | 156 | private: 157 | int _channels; 158 | std::vector _layers; 159 | Conv1x1 _head; 160 | activations::Activation* _activation; 161 | 162 | // Stores the outputs of the convs *except* the last one, which goes in 163 | // The array `outputs` provided to .process_() 164 | std::vector _buffers; 165 | 166 | // Apply the activation to the provided array, in-place 167 | void _apply_activation_(Eigen::MatrixXf& x); 168 | }; 169 | 170 | // The main WaveNet model 171 | class WaveNet : public DSP 172 | { 173 | public: 174 | WaveNet(const std::vector& layer_array_params, const float head_scale, const bool with_head, 175 | std::vector weights, const double expected_sample_rate = -1.0); 176 | ~WaveNet() = default; 177 | void process(NAM_SAMPLE* input, NAM_SAMPLE* output, const int num_frames) override; 178 | void set_weights_(std::vector& weights); 179 | 180 | protected: 181 | // Element-wise arrays: 182 | Eigen::MatrixXf _condition; 183 | 184 | void SetMaxBufferSize(const int maxBufferSize) override; 185 | // Fill in the "condition" array that's fed into the various parts of the net. 186 | virtual void _set_condition_array(NAM_SAMPLE* input, const int num_frames); 187 | // How many conditioning inputs are there. 188 | // Just one--the audio. 189 | virtual int _get_condition_dim() const { return 1; }; 190 | 191 | private: 192 | std::vector<_LayerArray> _layer_arrays; 193 | // Their outputs 194 | std::vector _layer_array_outputs; 195 | // _Head _head; 196 | 197 | // One more than total layer arrays 198 | std::vector _head_arrays; 199 | float _head_scale; 200 | Eigen::MatrixXf _head_output; 201 | 202 | void _advance_buffers_(const int num_frames); 203 | void _prepare_for_frames_(const long num_frames); 204 | 205 | // Ensure that all buffer arrays are the right size for this num_frames 206 | void _set_num_frames_(const long num_frames); 207 | 208 | int mPrewarmSamples = 0; // Pre-compute during initialization 209 | int PrewarmSamples() override { return mPrewarmSamples; }; 210 | }; 211 | }; // namespace wavenet 212 | }; // namespace nam 213 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # NeuralAmpModelerCore 2 | Core C++ DSP library for NAM plugins. 3 | 4 | For an example how to use, see [NeuralAmpModelerPlugin](https://github.com/sdatkinson/NeuralAmpModelerPlugin). 5 | 6 | ## Testing 7 | A workflow for testing the library is provided in `.github/workflows/build.yml`. 8 | You should be able to run it locally to test if you'd like. 9 | 10 | ## Sharp edges 11 | This library uses [Eigen](http://eigen.tuxfamily.org) to do the linear algebra routines that its neural networks require. Since these models hold their parameters as eigen object members, there is a risk with certain compilers and compiler optimizations that their memory is not aligned properly. This can be worked around by providing two preprocessor macros: `EIGEN_MAX_ALIGN_BYTES 0` and `EIGEN_DONT_VECTORIZE`, though this will probably harm performance. See [Structs Having Eigen Members](http://eigen.tuxfamily.org/dox-3.2/group__TopicStructHavingEigenMembers.html) for more information. This is being tracked as [Issue 67](https://github.com/sdatkinson/NeuralAmpModelerCore/issues/67). 12 | -------------------------------------------------------------------------------- /build/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore everything in this directory 2 | * 3 | # Except this file 4 | !.gitignore -------------------------------------------------------------------------------- /example_models/lstm.nam: -------------------------------------------------------------------------------- 1 | {"version": "0.5.4", "metadata": {"date": {"year": 2024, "month": 10, "day": 9, "hour": 18, "minute": 44, "second": 41}, "loudness": -37.8406867980957, "gain": 0.13508800804658277, "name": "Test LSTM", "modeled_by": "Steve", "gear_type": "amp", "gear_make": "Darkglass Electronics", "gear_model": "Microtubes 900 v2", "tone_type": "clean", "input_level_dbu": 18.3, "output_level_dbu": 12.3, "training": {"settings": {"ignore_checks": false}, "data": {"latency": {"manual": null, "calibration": {"algorithm_version": 1, "delays": [-16], "safety_factor": 1, "recommended": -17, "warnings": {"matches_lookahead": false, "disagreement_too_high": false}}}, "checks": {"version": 3, "passed": true}}, "validation_esr": null}}, "architecture": "LSTM", "config": {"input_size": 1, "hidden_size": 3, "num_layers": 1}, "weights": [-0.21677088737487793, -0.6683622002601624, -0.2560940980911255, -0.3588429093360901, 0.17952610552310944, 0.19445613026618958, -0.01662646047770977, 0.5353694558143616, -0.2536540627479553, -0.5132213234901428, -0.020476307719945908, 0.08592455089092255, -0.6891753673553467, 0.3627359867095947, 0.008421811275184155, 0.3113192617893219, 0.14251480996608734, 0.07989779114723206, -0.18211324512958527, 0.7118963003158569, 0.41084015369415283, -0.6571938395500183, -0.13214066624641418, -0.2698603868484497, 0.49387243390083313, -0.3491725027561188, 0.6353667974472046, -0.5005152225494385, 0.2052856683731079, -0.4301638901233673, -0.15770092606544495, -0.7181791067123413, 0.056290093809366226, -0.49049463868141174, 0.6623441576957703, 0.09029324352741241, 0.34005245566368103, 0.16416560113430023, 0.15520110726356506, -0.4155678153038025, -0.36928507685661316, 0.3211132884025574, -0.6769840121269226, -0.1575538069009781, 0.05268515646457672, -0.4191459119319916, 0.599330484867096, 0.21518059074878693, -4.246325492858887, -3.315647840499878, -4.328850746154785, 4.496089458465576, 5.015639305114746, 3.6492037773132324, 0.14431169629096985, -0.6633821725845337, 0.11673200130462646, -0.1418764889240265, -0.4897872805595398, -0.8689419031143188, -0.06714004278182983, -0.4450395107269287, -0.02142983116209507, -0.15136894583702087, -2.775207996368408, -0.08681213855743408, 0.05702732503414154, 0.670292317867279, 0.31442636251449585, 0.30793967843055725], "sample_rate": 48000} -------------------------------------------------------------------------------- /example_models/wavenet.nam: -------------------------------------------------------------------------------- 1 | {"version": "0.5.4", "metadata": {"date": {"year": 2024, "month": 10, "day": 9, "hour": 18, "minute": 32, "second": 27}, "loudness": -20.020729064941406, "gain": 0.19575619747898518, "name": "Test Model", "modeled_by": "Steve", "gear_type": "amp", "gear_make": "Darkglass Electronics", "gear_model": "Microtubes 900 v2", "tone_type": "clean", "input_level_dbu": 18.3, "output_level_dbu": 12.3, "training": {"settings": {"ignore_checks": false}, "data": {"latency": {"manual": null, "calibration": {"algorithm_version": 1, "delays": [-16], "safety_factor": 1, "recommended": -17, "warnings": {"matches_lookahead": false, "disagreement_too_high": false}}}, "checks": {"version": 3, "passed": true}}, "validation_esr": 0.13345033695550146}}, "architecture": "WaveNet", "config": {"layers": [{"input_size": 1, "condition_size": 1, "head_size": 2, "channels": 3, "kernel_size": 3, "dilations": [1, 2], "activation": "Tanh", "gated": false, "head_bias": false}, {"input_size": 3, "condition_size": 1, "head_size": 1, "channels": 2, "kernel_size": 3, "dilations": [8], "activation": "Tanh", "gated": false, "head_bias": true}], "head": null, "head_scale": 0.02}, "weights": [-0.6180188059806824, 1.0314024686813354, -1.0111560821533203, -0.38462021946907043, 0.35968291759490967, 0.5255971550941467, 0.19149275124073029, -0.18075695633888245, -0.33711034059524536, -0.21037575602531433, 0.2007753700017929, 0.21644853055477142, -1.3216396570205688, -0.35082393884658813, 0.43541353940963745, 0.9693092107772827, 0.2394428700208664, -0.41078877449035645, -1.193748116493225, -0.14876757562160492, 0.8413559198379517, 0.24491633474826813, 0.8857091665267944, 0.5647665858268738, 0.08301573246717453, -0.801490843296051, 0.168976828455925, -0.5413634181022644, 0.484220415353775, 0.021656272932887077, -0.15155009925365448, 0.07081033289432526, 0.00019397131109144539, -0.7408013939857483, -1.3308452367782593, -1.0403972864151, 0.016809873282909393, 0.6778652667999268, 0.28265541791915894, -0.28287461400032043, 1.0525944232940674, -0.6385797262191772, -0.2195468544960022, -0.3150196671485901, -0.8814508318901062, -0.2746180295944214, 0.15367186069488525, 0.22431065142154694, -0.056788790971040726, -0.38902369141578674, 0.5406259894371033, 0.3566059470176697, 0.14383991062641144, -0.25409433245658875, 0.16139137744903564, -0.05857989564538002, -0.18448838591575623, -0.253485769033432, -0.42405444383621216, -0.030114537104964256, 0.47283637523651123, 0.14930365979671478, -0.4410354793071747, -0.21976807713508606, -0.12736600637435913, -0.5674286484718323, -0.347588449716568, -0.3687525689601898, 0.4130803942680359, 0.8551775217056274, -0.05746064335107803, -0.38243237137794495, 0.20036561787128448, 0.25542038679122925, -0.0819990262389183, 0.19469600915908813, 0.10215214639902115, -0.26087674498558044, -0.1773151010274887, -0.09658292680978775, -0.7381710410118103, 0.7003506422042847, 0.7253592014312744, -0.07488955557346344, -0.23439547419548035, -0.5138604044914246, -0.7976311445236206, -0.8090851902961731, -0.37562188506126404, -1.163352131843567, 0.30907657742500305, 0.0564480796456337, 0.1190297082066536, 0.2310808300971985, -0.45360898971557617, -0.11524498462677002, 0.2552330493927002, 0.2913571298122406, -0.23171702027320862, -0.35578709840774536, 0.40732908248901367, 0.7458747029304504, 0.27514976263046265, -0.7503036856651306, -0.6707972884178162, -0.4248569905757904, -0.1671624332666397, -0.14162226021289825, 0.37550851702690125, -0.038120146840810776, 0.16232982277870178, -0.05173371359705925, -0.2842361629009247, 0.38820165395736694, 0.521754801273346, -0.3581869900226593, 0.21531476080417633, 0.11342520266771317, 0.01764630153775215, 0.07780527323484421, 0.9356631636619568, 0.04235581308603287, -0.3450177311897278, 0.3345951437950134, -0.678291380405426, -0.4191069006919861, -0.1770099401473999, -5.386871337890625, -5.130850791931152, -0.4049331247806549, 0.019999999552965164], "sample_rate": 48000} -------------------------------------------------------------------------------- /format.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Apply project formatting (i.e. clang-format with LLVM style) 3 | # 4 | # Usage: 5 | # $ ./format.sh 6 | 7 | echo "Formatting..." 8 | 9 | git ls-files "*.h" "*.cpp" | xargs clang-format -i . 10 | 11 | echo "Formatting complete!" 12 | echo "You can stage all of the files using:" 13 | echo "" 14 | echo ' git ls-files "*.h" "*.cpp" | xargs git add' 15 | echo "" 16 | -------------------------------------------------------------------------------- /tools/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB_RECURSE NAM_SOURCES ../NAM/*.cpp ../NAM/*.c ../NAM*.h) 2 | 3 | set(TOOLS benchmodel) 4 | 5 | add_custom_target(tools ALL 6 | DEPENDS ${TOOLS}) 7 | 8 | include_directories(tools ..) 9 | include_directories(tools ${NAM_DEPS_PATH}/eigen) 10 | include_directories(tools ${NAM_DEPS_PATH}/nlohmann) 11 | 12 | add_executable(loadmodel loadmodel.cpp ${NAM_SOURCES}) 13 | add_executable(benchmodel benchmodel.cpp ${NAM_SOURCES}) 14 | add_executable(run_tests run_tests.cpp ${NAM_SOURCES}) 15 | 16 | source_group(NAM ${CMAKE_CURRENT_SOURCE_DIR} FILES ${NAM_SOURCES}) 17 | 18 | target_compile_features(${TOOLS} PUBLIC cxx_std_17) 19 | 20 | set_target_properties(${TOOLS} 21 | PROPERTIES 22 | CXX_VISIBILITY_PRESET hidden 23 | INTERPROCEDURAL_OPTIMIZATION TRUE 24 | PREFIX "" 25 | ) 26 | 27 | if (CMAKE_SYSTEM_NAME STREQUAL "Windows") 28 | target_compile_definitions(${TOOLS} PRIVATE NOMINMAX WIN32_LEAN_AND_MEAN) 29 | endif() 30 | 31 | if (MSVC) 32 | target_compile_options(${TOOLS} PRIVATE 33 | "$<$:/W4>" 34 | "$<$:/O2>" 35 | ) 36 | else() 37 | target_compile_options(${TOOLS} PRIVATE 38 | -Wall -Wextra -Wpedantic -Wstrict-aliasing -Wunreachable-code -Weffc++ -Wno-unused-parameter 39 | "$<$:-Og;-ggdb;-Werror>" 40 | "$<$:-Ofast>" 41 | ) 42 | endif() 43 | 44 | # There's an error in eigen's 45 | # /Users/steve/src/NeuralAmpModelerCore/Dependencies/eigen/Eigen/src/Core/products/GeneralBlockPanelKernel.h 46 | # Don't let this break my build on debug: 47 | set_source_files_properties(../NAM/dsp.cpp PROPERTIES COMPILE_FLAGS "-Wno-error") 48 | -------------------------------------------------------------------------------- /tools/benchmodel.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "NAM/dsp.h" 5 | 6 | using std::chrono::duration; 7 | using std::chrono::duration_cast; 8 | using std::chrono::high_resolution_clock; 9 | using std::chrono::milliseconds; 10 | 11 | #define AUDIO_BUFFER_SIZE 64 12 | 13 | double inputBuffer[AUDIO_BUFFER_SIZE]; 14 | double outputBuffer[AUDIO_BUFFER_SIZE]; 15 | 16 | int main(int argc, char* argv[]) 17 | { 18 | if (argc > 1) 19 | { 20 | const char* modelPath = argv[1]; 21 | 22 | std::cout << "Loading model " << modelPath << "\n"; 23 | 24 | // Turn on fast tanh approximation 25 | nam::activations::Activation::enable_fast_tanh(); 26 | 27 | std::unique_ptr model; 28 | 29 | model.reset(); 30 | model = nam::get_dsp(modelPath); 31 | 32 | if (model == nullptr) 33 | { 34 | std::cerr << "Failed to load model\n"; 35 | 36 | exit(1); 37 | } 38 | 39 | size_t bufferSize = AUDIO_BUFFER_SIZE; 40 | model->Reset(model->GetExpectedSampleRate(), bufferSize); 41 | size_t numBuffers = (48000 / bufferSize) * 2; 42 | 43 | // Fill input buffer with zeroes. 44 | // Output buffer doesn't matter. 45 | for (int i = 0; i < AUDIO_BUFFER_SIZE; i++) 46 | { 47 | inputBuffer[i] = 0.0; 48 | } 49 | 50 | std::cout << "Running benchmark\n"; 51 | auto t1 = high_resolution_clock::now(); 52 | for (size_t i = 0; i < numBuffers; i++) 53 | { 54 | model->process(inputBuffer, outputBuffer, AUDIO_BUFFER_SIZE); 55 | } 56 | auto t2 = high_resolution_clock::now(); 57 | std::cout << "Finished\n"; 58 | 59 | 60 | /* Getting number of milliseconds as an integer. */ 61 | auto ms_int = duration_cast(t2 - t1); 62 | 63 | /* Getting number of milliseconds as a double. */ 64 | duration ms_double = t2 - t1; 65 | 66 | std::cout << ms_int.count() << "ms\n"; 67 | std::cout << ms_double.count() << "ms\n"; 68 | } 69 | else 70 | { 71 | std::cerr << "Usage: benchmodel \n"; 72 | } 73 | 74 | exit(0); 75 | } 76 | -------------------------------------------------------------------------------- /tools/loadmodel.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include "NAM/dsp.h" 3 | 4 | int main(int argc, char* argv[]) 5 | { 6 | if (argc > 1) 7 | { 8 | char* modelPath = argv[1]; 9 | 10 | fprintf(stderr, "Loading model [%s]\n", modelPath); 11 | 12 | auto model = nam::get_dsp(modelPath); 13 | 14 | if (model != nullptr) 15 | { 16 | fprintf(stderr, "Model loaded successfully\n"); 17 | } 18 | else 19 | { 20 | fprintf(stderr, "Failed to load model\n"); 21 | 22 | exit(1); 23 | } 24 | } 25 | else 26 | { 27 | fprintf(stderr, "Usage: loadmodel \n"); 28 | } 29 | 30 | exit(0); 31 | } 32 | -------------------------------------------------------------------------------- /tools/run_tests.cpp: -------------------------------------------------------------------------------- 1 | // Entry point for tests 2 | // See the GitHub Action for a demo how to build and run tests. 3 | 4 | #include 5 | #include "test/test_activations.cpp" 6 | #include "test/test_dsp.cpp" 7 | #include "test/test_get_dsp.cpp" 8 | #include "test/test_wavenet.cpp" 9 | 10 | int main() 11 | { 12 | std::cout << "Running tests..." << std::endl; 13 | // TODO Automatically loop, catch exceptions, log results 14 | 15 | test_activations::TestFastTanh::test_core_function(); 16 | test_activations::TestFastTanh::test_get_by_init(); 17 | test_activations::TestFastTanh::test_get_by_str(); 18 | 19 | test_activations::TestLeakyReLU::test_core_function(); 20 | test_activations::TestLeakyReLU::test_get_by_init(); 21 | test_activations::TestLeakyReLU::test_get_by_str(); 22 | 23 | test_dsp::test_construct(); 24 | test_dsp::test_get_input_level(); 25 | test_dsp::test_get_output_level(); 26 | test_dsp::test_has_input_level(); 27 | test_dsp::test_has_output_level(); 28 | test_dsp::test_set_input_level(); 29 | test_dsp::test_set_output_level(); 30 | 31 | test_get_dsp::test_gets_input_level(); 32 | test_get_dsp::test_gets_output_level(); 33 | test_get_dsp::test_null_input_level(); 34 | test_get_dsp::test_null_output_level(); 35 | 36 | test_wavenet::test_gated(); 37 | 38 | std::cout << "Success!" << std::endl; 39 | return 0; 40 | } -------------------------------------------------------------------------------- /tools/test/test_activations.cpp: -------------------------------------------------------------------------------- 1 | // Tests for activation functions 2 | // 3 | // Things you want ot test for: 4 | // 1. That the core elementwise funciton is snapshot-correct. 5 | // 2. The class that wraps the core function for an array of data 6 | // 3. .cpp: that you have the singleton defined, and that it's in the unordered map to get by string 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | #include "NAM/activations.h" 13 | 14 | namespace test_activations 15 | { 16 | // TODO get nonzero cases 17 | class TestFastTanh 18 | { 19 | public: 20 | static void test_core_function() 21 | { 22 | auto TestCase = [](float input, float expectedOutput) { 23 | float actualOutput = nam::activations::fast_tanh(input); 24 | assert(actualOutput == expectedOutput); 25 | }; 26 | // A few snapshot tests 27 | TestCase(0.0f, 0.0f); 28 | // TestCase(1.0f, 1.0f); 29 | // TestCase(-1.0f, -0.01f); 30 | }; 31 | 32 | static void test_get_by_init() 33 | { 34 | auto a = nam::activations::ActivationLeakyReLU(); 35 | _test_class(&a); 36 | } 37 | 38 | // Get the singleton and test it 39 | static void test_get_by_str() 40 | { 41 | const std::string name = "Fasttanh"; 42 | auto a = nam::activations::Activation::get_activation(name); 43 | _test_class(a); 44 | } 45 | 46 | private: 47 | // Put the class through its paces 48 | static void _test_class(nam::activations::Activation* a) 49 | { 50 | std::vector inputs, expectedOutputs; 51 | 52 | inputs.push_back(0.0f); 53 | expectedOutputs.push_back(0.0f); 54 | 55 | // inputs.push_back(1.0f); 56 | // expectedOutputs.push_back(1.0f); 57 | 58 | // inputs.push_back(-1.0f); 59 | // expectedOutputs.push_back(-0.01f); 60 | 61 | a->apply(inputs.data(), (long)inputs.size()); 62 | for (auto itActual = inputs.begin(), itExpected = expectedOutputs.begin(); itActual != inputs.end(); 63 | ++itActual, ++itExpected) 64 | { 65 | assert(*itActual == *itExpected); 66 | } 67 | }; 68 | }; 69 | 70 | class TestLeakyReLU 71 | { 72 | public: 73 | static void test_core_function() 74 | { 75 | auto TestCase = [](float input, float expectedOutput) { 76 | float actualOutput = nam::activations::leaky_relu(input); 77 | assert(actualOutput == expectedOutput); 78 | }; 79 | // A few snapshot tests 80 | TestCase(0.0f, 0.0f); 81 | TestCase(1.0f, 1.0f); 82 | TestCase(-1.0f, -0.01f); 83 | }; 84 | 85 | static void test_get_by_init() 86 | { 87 | auto a = nam::activations::ActivationLeakyReLU(); 88 | _test_class(&a); 89 | } 90 | 91 | // Get the singleton and test it 92 | static void test_get_by_str() 93 | { 94 | const std::string name = "LeakyReLU"; 95 | auto a = nam::activations::Activation::get_activation(name); 96 | _test_class(a); 97 | } 98 | 99 | private: 100 | // Put the class through its paces 101 | static void _test_class(nam::activations::Activation* a) 102 | { 103 | std::vector inputs, expectedOutputs; 104 | 105 | inputs.push_back(0.0f); 106 | expectedOutputs.push_back(0.0f); 107 | 108 | inputs.push_back(1.0f); 109 | expectedOutputs.push_back(1.0f); 110 | 111 | inputs.push_back(-1.0f); 112 | expectedOutputs.push_back(-0.01f); 113 | 114 | a->apply(inputs.data(), (long)inputs.size()); 115 | for (auto itActual = inputs.begin(), itExpected = expectedOutputs.begin(); itActual != inputs.end(); 116 | ++itActual, ++itExpected) 117 | { 118 | assert(*itActual == *itExpected); 119 | } 120 | }; 121 | }; 122 | }; // namespace test_activations 123 | -------------------------------------------------------------------------------- /tools/test/test_dsp.cpp: -------------------------------------------------------------------------------- 1 | // Tests for dsp 2 | 3 | #include "NAM/dsp.h" 4 | 5 | namespace test_dsp 6 | { 7 | // Simplest test: can I construct something! 8 | void test_construct() 9 | { 10 | nam::DSP myDsp(48000.0); 11 | } 12 | 13 | void test_get_input_level() 14 | { 15 | nam::DSP myDsp(48000.0); 16 | const double expected = 19.0; 17 | myDsp.SetInputLevel(expected); 18 | assert(myDsp.HasInputLevel()); 19 | const double actual = myDsp.GetInputLevel(); 20 | 21 | assert(actual == expected); 22 | } 23 | 24 | void test_get_output_level() 25 | { 26 | nam::DSP myDsp(48000.0); 27 | const double expected = 12.0; 28 | myDsp.SetOutputLevel(expected); 29 | assert(myDsp.HasOutputLevel()); 30 | const double actual = myDsp.GetOutputLevel(); 31 | 32 | assert(actual == expected); 33 | } 34 | 35 | // Test correct function of DSP::HasInputLevel() 36 | void test_has_input_level() 37 | { 38 | nam::DSP myDsp(48000.0); 39 | assert(!myDsp.HasInputLevel()); 40 | 41 | myDsp.SetInputLevel(19.0); 42 | assert(myDsp.HasInputLevel()); 43 | } 44 | 45 | void test_has_output_level() 46 | { 47 | nam::DSP myDsp(48000.0); 48 | assert(!myDsp.HasOutputLevel()); 49 | 50 | myDsp.SetOutputLevel(12.0); 51 | assert(myDsp.HasOutputLevel()); 52 | } 53 | 54 | // Test correct function of DSP::HasInputLevel() 55 | void test_set_input_level() 56 | { 57 | nam::DSP myDsp(48000.0); 58 | myDsp.SetInputLevel(19.0); 59 | } 60 | 61 | void test_set_output_level() 62 | { 63 | nam::DSP myDsp(48000.0); 64 | myDsp.SetOutputLevel(19.0); 65 | } 66 | }; // namespace test_dsp 67 | -------------------------------------------------------------------------------- /tools/test/test_get_dsp.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include "json.hpp" 7 | 8 | #include "NAM/get_dsp.h" 9 | 10 | namespace test_get_dsp 11 | { 12 | // Config 13 | const std::string basicConfigStr = 14 | R"({"version": "0.5.4", "metadata": {"date": {"year": 2024, "month": 10, "day": 9, "hour": 18, "minute": 44, "second": 41}, "loudness": -37.8406867980957, "gain": 0.13508800804658277, "name": "Test LSTM", "modeled_by": "Steve", "gear_type": "amp", "gear_make": "Darkglass Electronics", "gear_model": "Microtubes 900 v2", "tone_type": "clean", "input_level_dbu": 18.3, "output_level_dbu": 12.3, "training": {"settings": {"ignore_checks": false}, "data": {"latency": {"manual": null, "calibration": {"algorithm_version": 1, "delays": [-16], "safety_factor": 1, "recommended": -17, "warnings": {"matches_lookahead": false, "disagreement_too_high": false}}}, "checks": {"version": 3, "passed": true}}, "validation_esr": null}}, "architecture": "LSTM", "config": {"input_size": 1, "hidden_size": 3, "num_layers": 1}, "weights": [-0.21677088737487793, -0.6683622002601624, -0.2560940980911255, -0.3588429093360901, 0.17952610552310944, 0.19445613026618958, -0.01662646047770977, 0.5353694558143616, -0.2536540627479553, -0.5132213234901428, -0.020476307719945908, 0.08592455089092255, -0.6891753673553467, 0.3627359867095947, 0.008421811275184155, 0.3113192617893219, 0.14251480996608734, 0.07989779114723206, -0.18211324512958527, 0.7118963003158569, 0.41084015369415283, -0.6571938395500183, -0.13214066624641418, -0.2698603868484497, 0.49387243390083313, -0.3491725027561188, 0.6353667974472046, -0.5005152225494385, 0.2052856683731079, -0.4301638901233673, -0.15770092606544495, -0.7181791067123413, 0.056290093809366226, -0.49049463868141174, 0.6623441576957703, 0.09029324352741241, 0.34005245566368103, 0.16416560113430023, 0.15520110726356506, -0.4155678153038025, -0.36928507685661316, 0.3211132884025574, -0.6769840121269226, -0.1575538069009781, 0.05268515646457672, -0.4191459119319916, 0.599330484867096, 0.21518059074878693, -4.246325492858887, -3.315647840499878, -4.328850746154785, 4.496089458465576, 5.015639305114746, 3.6492037773132324, 0.14431169629096985, -0.6633821725845337, 0.11673200130462646, -0.1418764889240265, -0.4897872805595398, -0.8689419031143188, -0.06714004278182983, -0.4450395107269287, -0.02142983116209507, -0.15136894583702087, -2.775207996368408, -0.08681213855743408, 0.05702732503414154, 0.670292317867279, 0.31442636251449585, 0.30793967843055725], "sample_rate": 48000})"; 15 | 16 | // Copied over but shouldn't be publicly-exposed. 17 | std::vector GetWeights(nlohmann::json const& j) 18 | { 19 | auto it = j.find("weights"); 20 | if (it != j.end()) 21 | { 22 | return *it; 23 | } 24 | else 25 | { 26 | throw std::runtime_error("Corrupted model file is missing weights."); 27 | } 28 | } 29 | 30 | nam::dspData _GetConfig(const std::string& configStr = basicConfigStr) 31 | { 32 | nlohmann::json j = nlohmann::json::parse(configStr); 33 | 34 | std::vector weights = GetWeights(j); 35 | nam::dspData returnedConfig; 36 | returnedConfig.version = j["version"]; 37 | returnedConfig.architecture = j["architecture"]; 38 | returnedConfig.config = j["config"]; 39 | returnedConfig.metadata = j["metadata"]; 40 | returnedConfig.weights = weights; 41 | 42 | return returnedConfig; 43 | } 44 | 45 | void test_gets_input_level() 46 | { 47 | nam::dspData config = _GetConfig(); 48 | std::unique_ptr dsp = get_dsp(config); 49 | assert(dsp->HasInputLevel()); 50 | } 51 | void test_gets_output_level() 52 | { 53 | nam::dspData config = _GetConfig(); 54 | std::unique_ptr dsp = get_dsp(config); 55 | assert(dsp->HasOutputLevel()); 56 | } 57 | 58 | void test_null_input_level() 59 | { 60 | // Issue 129 61 | const std::string configStr = 62 | R"({"version": "0.5.4", "metadata": {"date": {"year": 2024, "month": 10, "day": 9, "hour": 18, "minute": 44, "second": 41}, "loudness": -37.8406867980957, "gain": 0.13508800804658277, "name": "Test LSTM", "modeled_by": "Steve", "gear_type": "amp", "gear_make": "Darkglass Electronics", "gear_model": "Microtubes 900 v2", "tone_type": "clean", "input_level_dbu": null, "output_level_dbu": 12.3, "training": {"settings": {"ignore_checks": false}, "data": {"latency": {"manual": null, "calibration": {"algorithm_version": 1, "delays": [-16], "safety_factor": 1, "recommended": -17, "warnings": {"matches_lookahead": false, "disagreement_too_high": false}}}, "checks": {"version": 3, "passed": true}}, "validation_esr": null}}, "architecture": "LSTM", "config": {"input_size": 1, "hidden_size": 3, "num_layers": 1}, "weights": [-0.21677088737487793, -0.6683622002601624, -0.2560940980911255, -0.3588429093360901, 0.17952610552310944, 0.19445613026618958, -0.01662646047770977, 0.5353694558143616, -0.2536540627479553, -0.5132213234901428, -0.020476307719945908, 0.08592455089092255, -0.6891753673553467, 0.3627359867095947, 0.008421811275184155, 0.3113192617893219, 0.14251480996608734, 0.07989779114723206, -0.18211324512958527, 0.7118963003158569, 0.41084015369415283, -0.6571938395500183, -0.13214066624641418, -0.2698603868484497, 0.49387243390083313, -0.3491725027561188, 0.6353667974472046, -0.5005152225494385, 0.2052856683731079, -0.4301638901233673, -0.15770092606544495, -0.7181791067123413, 0.056290093809366226, -0.49049463868141174, 0.6623441576957703, 0.09029324352741241, 0.34005245566368103, 0.16416560113430023, 0.15520110726356506, -0.4155678153038025, -0.36928507685661316, 0.3211132884025574, -0.6769840121269226, -0.1575538069009781, 0.05268515646457672, -0.4191459119319916, 0.599330484867096, 0.21518059074878693, -4.246325492858887, -3.315647840499878, -4.328850746154785, 4.496089458465576, 5.015639305114746, 3.6492037773132324, 0.14431169629096985, -0.6633821725845337, 0.11673200130462646, -0.1418764889240265, -0.4897872805595398, -0.8689419031143188, -0.06714004278182983, -0.4450395107269287, -0.02142983116209507, -0.15136894583702087, -2.775207996368408, -0.08681213855743408, 0.05702732503414154, 0.670292317867279, 0.31442636251449585, 0.30793967843055725], "sample_rate": 48000})"; 63 | nam::dspData config = _GetConfig(configStr); 64 | // The first part of this is that the following line doesn't fail: 65 | std::unique_ptr dsp = get_dsp(config); 66 | 67 | assert(!dsp->HasInputLevel()); 68 | assert(dsp->HasOutputLevel()); 69 | } 70 | 71 | void test_null_output_level() 72 | { 73 | // Issue 129 74 | const std::string configStr = 75 | R"({"version": "0.5.4", "metadata": {"date": {"year": 2024, "month": 10, "day": 9, "hour": 18, "minute": 44, "second": 41}, "loudness": -37.8406867980957, "gain": 0.13508800804658277, "name": "Test LSTM", "modeled_by": "Steve", "gear_type": "amp", "gear_make": "Darkglass Electronics", "gear_model": "Microtubes 900 v2", "tone_type": "clean", "input_level_dbu": 19.0, "output_level_dbu": null, "training": {"settings": {"ignore_checks": false}, "data": {"latency": {"manual": null, "calibration": {"algorithm_version": 1, "delays": [-16], "safety_factor": 1, "recommended": -17, "warnings": {"matches_lookahead": false, "disagreement_too_high": false}}}, "checks": {"version": 3, "passed": true}}, "validation_esr": null}}, "architecture": "LSTM", "config": {"input_size": 1, "hidden_size": 3, "num_layers": 1}, "weights": [-0.21677088737487793, -0.6683622002601624, -0.2560940980911255, -0.3588429093360901, 0.17952610552310944, 0.19445613026618958, -0.01662646047770977, 0.5353694558143616, -0.2536540627479553, -0.5132213234901428, -0.020476307719945908, 0.08592455089092255, -0.6891753673553467, 0.3627359867095947, 0.008421811275184155, 0.3113192617893219, 0.14251480996608734, 0.07989779114723206, -0.18211324512958527, 0.7118963003158569, 0.41084015369415283, -0.6571938395500183, -0.13214066624641418, -0.2698603868484497, 0.49387243390083313, -0.3491725027561188, 0.6353667974472046, -0.5005152225494385, 0.2052856683731079, -0.4301638901233673, -0.15770092606544495, -0.7181791067123413, 0.056290093809366226, -0.49049463868141174, 0.6623441576957703, 0.09029324352741241, 0.34005245566368103, 0.16416560113430023, 0.15520110726356506, -0.4155678153038025, -0.36928507685661316, 0.3211132884025574, -0.6769840121269226, -0.1575538069009781, 0.05268515646457672, -0.4191459119319916, 0.599330484867096, 0.21518059074878693, -4.246325492858887, -3.315647840499878, -4.328850746154785, 4.496089458465576, 5.015639305114746, 3.6492037773132324, 0.14431169629096985, -0.6633821725845337, 0.11673200130462646, -0.1418764889240265, -0.4897872805595398, -0.8689419031143188, -0.06714004278182983, -0.4450395107269287, -0.02142983116209507, -0.15136894583702087, -2.775207996368408, -0.08681213855743408, 0.05702732503414154, 0.670292317867279, 0.31442636251449585, 0.30793967843055725], "sample_rate": 48000})"; 76 | nam::dspData config = _GetConfig(configStr); 77 | // The first part of this is that the following line doesn't fail: 78 | std::unique_ptr dsp = get_dsp(config); 79 | assert(dsp->HasInputLevel()); 80 | assert(!dsp->HasOutputLevel()); 81 | } 82 | }; // namespace test_get_dsp -------------------------------------------------------------------------------- /tools/test/test_wavenet.cpp: -------------------------------------------------------------------------------- 1 | // Tests for the WaveNet 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | #include "NAM/wavenet.h" 8 | 9 | namespace test_wavenet 10 | { 11 | void test_gated() 12 | { 13 | // Assert correct nuemrics of the gating activation. 14 | // Issue 101 15 | const int conditionSize = 1; 16 | const int channels = 1; 17 | const int kernelSize = 1; 18 | const int dilation = 1; 19 | const std::string activation = "ReLU"; 20 | const bool gated = true; 21 | auto layer = nam::wavenet::_Layer(conditionSize, channels, kernelSize, dilation, activation, gated); 22 | 23 | // Conv, input mixin, 1x1 24 | std::vector weights{ 25 | // Conv (weight, bias) NOTE: 2 channels out bc gated, so shapes are (2,1,1), (2,) 26 | 1.0f, 1.0f, 0.0f, 0.0f, 27 | // Input mixin (weight only: (2,1,1)) 28 | 1.0f, -1.0f, 29 | // 1x1 (weight (1,1,1), bias (1,)) 30 | // NOTE: Weights are (1,1) on conv, (1,-1), so the inputs sum on the upper channel and cancel on the lower. 31 | // This should give us a nice zero if the input & condition are the same, so that'll sigmoid to 0.5 for the 32 | // gate. 33 | 1.0f, 0.0f}; 34 | auto it = weights.begin(); 35 | layer.set_weights_(it); 36 | assert(it == weights.end()); 37 | 38 | const long numFrames = 4; 39 | layer.set_num_frames_(numFrames); 40 | 41 | Eigen::MatrixXf input, condition, headInput, output; 42 | input.resize(channels, numFrames); 43 | condition.resize(channels, numFrames); 44 | headInput.resize(channels, numFrames); 45 | output.resize(channels, numFrames); 46 | 47 | const float signalValue = 0.25f; 48 | input.fill(signalValue); 49 | condition.fill(signalValue); 50 | // So input & condition will sum to 0.5 on the top channel (-> ReLU), cancel to 0 on bottom (-> sigmoid) 51 | 52 | headInput.setZero(); 53 | output.setZero(); 54 | 55 | layer.process_(input, condition, headInput, output, 0, 0, (int)numFrames); 56 | 57 | // 0.25 + 0.25 -> 0.5 for conv & input mixin top channel 58 | // (0 on bottom channel) 59 | // Top ReLU -> preseves 0.5 60 | // Bottom sigmoid 0->0.5 61 | // Product is 0.25 62 | // 1x1 is unity 63 | // Skip-connect -> 0.25 (input) + 0.25 (output) -> 0.5 output 64 | // head output gets 0+0.25 = 0.25 65 | const float expectedOutput = 0.5; 66 | const float expectedHeadInput = 0.25; 67 | for (int i = 0; i < numFrames; i++) 68 | { 69 | const float actualOutput = output(0, i); 70 | const float actualHeadInput = headInput(0, i); 71 | // std::cout << actualOutput << std::endl; 72 | assert(actualOutput == expectedOutput); 73 | assert(actualHeadInput == expectedHeadInput); 74 | } 75 | } 76 | }; // namespace test_wavenet --------------------------------------------------------------------------------