Merge pull request #36 from mikeoliphant/nam_a2

NAM a2 support

Author: mikeoliphant

NeuralAudio/CMakeLists.txt

@@ -11,14 +11,20 @@ else()
     message(STATUS "NOT Building static RTNeural models")
 endif()
 
-option(BUILD_NAMCORE "Build NAM Core" OFF)
+option(BUILD_NAMCORE "Build NAM Core" ON)
 if(BUILD_NAMCORE)
     message(STATUS "Building NAM Core implementation")
     add_definitions(-DBUILD_NAMCORE)
 else()
     message(STATUS "NOT Building NAM Core implementation")
 endif()
 
+option(NAM_USE_INLINE_GEMM "Use inline matrix multiplications in NAM Core" OFF)
+if(NAM_USE_INLINE_GEMM)
+    message(STATUS "Using NAM Core inline matrix multiplications")
+    add_definitions(-DNAM_USE_INLINE_GEMM)
+else()
+    message(STATUS "NOT Using NAM Core inline matrix multiplications")
 option(BUILD_INTERNAL_STATIC_WAVENET "Build Internal static WaveNet models" ON)
 if(BUILD_INTERNAL_STATIC_WAVENET)
     message(STATUS "Building Internal static WaveNet models")
@@ -77,7 +83,16 @@ set(SOURCES
 if(BUILD_NAMCORE)
 	set(NAM_SOURCES	../deps/NeuralAmpModelerCore/NAM/activations.h
 		../deps/NeuralAmpModelerCore/NAM/activations.cpp
+		../deps/NeuralAmpModelerCore/NAM/conv1d.cpp
+		../deps/NeuralAmpModelerCore/NAM/conv1d.h
+		../deps/NeuralAmpModelerCore/NAM/film.h 
+		../deps/NeuralAmpModelerCore/NAM/gating_activations.h
+		../deps/NeuralAmpModelerCore/NAM/get_dsp.h
+		../deps/NeuralAmpModelerCore/NAM/get_dsp.cpp
 		../deps/NeuralAmpModelerCore/NAM/lstm.h
+		../deps/NeuralAmpModelerCore/NAM/registry.h
+		../deps/NeuralAmpModelerCore/NAM/ring_buffer.h
+		../deps/NeuralAmpModelerCore/NAM/ring_buffer.cpp
 		../deps/NeuralAmpModelerCore/NAM/lstm.cpp
 		../deps/NeuralAmpModelerCore/NAM/dsp.h
 		../deps/NeuralAmpModelerCore/NAM/dsp.cpp
@@ -92,7 +107,7 @@ set(RTNEURAL_WN_SOURCES ../deps/RTNeural-NAM/wavenet/wavenet_layer.hpp
 	../deps/RTNeural-NAM/wavenet/arena.hpp)
 endif()
 
-add_library(NeuralAudio STATIC ${SOURCES} ${NAM_SOURCES} ${RTNEURAL_WN_SOURCES})
+add_library(NeuralAudio OBJECT ${SOURCES} ${NAM_SOURCES} ${RTNEURAL_WN_SOURCES})
 
 target_include_directories(NeuralAudio PUBLIC ..)
 target_include_directories(NeuralAudio SYSTEM PRIVATE ../deps/NeuralAmpModelerCore)

NeuralAudio/NAMModel.h

@@ -2,9 +2,9 @@
 
 #include "NeuralModel.h"
 #include <NAM/activations.h>
+#include <NAM/get_dsp.h>
 #include <NAM/dsp.h>
-#include <NAM/lstm.h>
-#include <NAM/wavenet.h>
+#include <NAM/registry.h>
 
 namespace NeuralAudio
 {
@@ -34,46 +34,14 @@ namespace NeuralAudio
 
 			ReadNAMConfig(modelJson);
 
-			std::string arch = modelJson.at("architecture");
-
-			nlohmann::json config = modelJson.at("config");
-
-			std::vector<float> weights = modelJson.at("weights");
-
-			if (arch == "WaveNet")
-			{
-				std::vector<nam::wavenet::LayerArrayParams> layer_array_params;
-
-				for (size_t i = 0; i < config.at("layers").size(); i++)
-				{
-					nlohmann::json layerConfig = config.at("layers").at(i);
-
-					layer_array_params.push_back(
-						nam::wavenet::LayerArrayParams(layerConfig.at("input_size"), layerConfig.at("condition_size"), layerConfig.at("head_size"),
-							layerConfig.at("channels"), layerConfig.at("kernel_size"), layerConfig.at("dilations"),
-							layerConfig.at("activation"), layerConfig.at("gated"), layerConfig.at("head_bias")));
-				}
-
-				const bool with_head = !config.at("head").is_null();
-				const float head_scale = config.at("head_scale");
-
-				namModel = std::make_unique<nam::wavenet::WaveNet>(layer_array_params, head_scale, with_head, weights, sampleRate);
-			}
-			else if (arch == "LSTM")
-			{
-				const int num_layers = config.at("num_layers");
-				const int input_size = config.at("input_size");
-				const int hidden_size = config.at("hidden_size");
-
-				namModel = std::make_unique<nam::lstm::LSTM>(num_layers, input_size, hidden_size, weights, sampleRate);
-			}
+			namModel = nam::get_dsp(modelJson);
 
 			return true;
 		}
 
 		void Process(float* input, float* output, size_t numSamples)
 		{
-			namModel->process(input, output, (int)numSamples);
+			namModel->process(&input, &output, (int)numSamples);
 		}
 
 		void Prewarm()

NeuralAudio/NeuralModel.cpp

@@ -126,6 +126,17 @@ namespace NeuralAudio
 		return CreateFromStream(jsonStream, modelPath.extension());
 	}
 
+	bool NAMIsA2(std::string version)
+	{
+		int major = 0, minor = 0, patch = 0;
+		char dot;
+		std::stringstream ss(version);
+
+		ss >> major >> dot >> minor >> dot >> patch;
+
+		return (major > 0) || (minor > 5) || ((minor == 5) && (patch > 4));
+	}
+
 	NeuralModel* NeuralModel::CreateFromStream(std::basic_istream<char>& jsonStream, std::filesystem::path extension)
 	{
 		EnsureModelDefsAreLoaded();
@@ -137,10 +148,10 @@ namespace NeuralAudio
 
 		if (extension == ".nam")
 		{
-			std::string arch = modelJson.at("architecture");
-
 #ifdef BUILD_NAMCORE
-			if (wavenetLoadMode == EModelLoadMode::NAMCore)
+			std::string version = modelJson.at("version");
+
+			if ((wavenetLoadMode == EModelLoadMode::NAMCore) || NAMIsA2(version))
 			{
 				NAMModel* model = new NAMModel;
 
@@ -150,12 +161,13 @@ namespace NeuralAudio
 			}
 #endif
 
+			std::string arch = modelJson.at("architecture");
+			nlohmann::json config = modelJson.at("config");
+
 			if (newModel == nullptr)
 			{
 				if (arch == "WaveNet")
 				{
-					nlohmann::json config = modelJson.at("config");
-
 					if (config.at("layers").size() == 2)
 					{
 						nlohmann::json firstLayerConfig = config.at("layers").at(0);
@@ -219,8 +231,6 @@ namespace NeuralAudio
 				}
 				else if (arch == "LSTM")
 				{
-					nlohmann::json config = modelJson.at("config");
-
 #ifdef BUILD_STATIC_RTNEURAL
 					if (lstmLoadMode == EModelLoadMode::RTNeural)
 					{

README.md

@@ -130,6 +130,8 @@ Note - you'll have to change the Visual Studio version if you are using a differ
 
 ```-DBUILD_NAMCORE=ON|OFF```: Support loading models using the NAM Core implemenations.
 
+```-DNAM_USE_INLINE_GEMM=ON```: Enable use of inline matrix multiplication in NAM Core.
+
 ```-DBUILD_STATIC_RTNEURAL=ON|OFF```: Build static RTNeural model architectures (slower compile, larger size - only use if you plan on forcing RTNeural model loading).
 
 ```-DBUILD_INTERNAL_STATIC_WAVENET=ON|OFF```: Build internal static WaveNet model architectures (faster internal WaveNet, but slower compile, larger size).

Utils/ModelTest/ModelTest.cpp

@@ -12,6 +12,13 @@ NeuralModel* LoadModel(std::filesystem::path modelPath, EModelLoadMode loadMode)
 	NeuralModel::SetWaveNetLoadMode(loadMode);
 	NeuralModel::SetLSTMLoadMode(loadMode);
 
+	if (!std::filesystem::exists(modelPath))
+	{
+		std::cout << "Model file does not exist: " << modelPath << std::endl;
+
+		return nullptr;
+	}
+
 	try
 	{
 		auto model = NeuralAudio::NeuralModel::CreateFromFile(modelPath);
@@ -134,29 +141,36 @@ void RunNAMTests(std::filesystem::path modelPath, int blockSize)
 
 	double rms;
 
-	if (namCoreModel != nullptr)
-	{
-	}
-
 	std::tuple<double, double> internal;
 	std::tuple<double, double> rtNeural;
 	std::tuple<double, double> namCore;
 
-	internal = BenchModel(internalModel, blockSize, numBlocks);
+	if (internalModel != nullptr)
+	{
+		internal = BenchModel(internalModel, blockSize, numBlocks);
 
-	std::cout << "Internal: " << std::get<0>(internal) << " (" << std::get<1>(internal) << ")" << std::endl;
+		std::cout << "Internal: " << std::get<0>(internal) << " (" << std::get<1>(internal) << ")" << std::endl;
+	}
+	else
+	{
+		std::cout << "Model can't be loaded as internal model" << std::endl;
+	}
 
 	if (namCoreModel != nullptr)
 	{
 		std::cout << std::endl;
 
 		namCore = BenchModel(namCoreModel, blockSize, numBlocks);
 
-		rms = ComputeError(namCoreModel, internalModel, blockSize, numBlocks);
-
 		std::cout << "NAM Core: " << std::get<0>(namCore) << " (" << std::get<1>(namCore) << ")" << std::endl;
-		std::cout << "NAM vs Internal RMS err: " << rms << std::endl;
-		std::cout << "Internal is: " << (std::get<0>(namCore) / std::get<0>(internal)) << "x NAM" << std::endl;
+
+		if (internalModel != nullptr)
+		{
+			rms = ComputeError(namCoreModel, internalModel, blockSize, numBlocks);
+
+			std::cout << "NAM vs Internal RMS err: " << rms << std::endl;
+			std::cout << "Internal is: " << (std::get<0>(namCore) / std::get<0>(internal)) << "x NAM" << std::endl;
+		}
 	}
 
 	if (rtNeuralModel != nullptr)
@@ -166,12 +180,16 @@ void RunNAMTests(std::filesystem::path modelPath, int blockSize)
 		rtNeural = BenchModel(rtNeuralModel, blockSize, numBlocks);
 
 		std::cout << "RTNeural: " << std::get<0>(rtNeural) << " (" << std::get<1>(rtNeural) << ")" << std::endl;
-		rms = ComputeError(namCoreModel, rtNeuralModel, blockSize, numBlocks);
-		std::cout << "NAM vs RTNeural RMS err: " << rms << std::endl;
 
 		if (namCoreModel != nullptr)
 		{
-			std::cout << "RTNeural is: " << (std::get<0>(namCore) / std::get<0>(rtNeural)) << "x NAM" << std::endl;
+			rms = ComputeError(namCoreModel, rtNeuralModel, blockSize, numBlocks);
+			std::cout << "NAM vs RTNeural RMS err: " << rms << std::endl;
+
+			if (namCoreModel != nullptr)
+			{
+				std::cout << "RTNeural is: " << (std::get<0>(namCore) / std::get<0>(rtNeural)) << "x NAM" << std::endl;
+			}
 		}
 	}