├── README.md └── src ├── SharpML.Reccurent.Examples ├── Data │ └── XorDataSetGenerator.cs ├── ExampleXor.cs ├── Program.cs ├── Properties │ └── AssemblyInfo.cs ├── SharpML.Reccurent.Examples.csproj ├── bin │ └── Debug │ │ ├── SharpML.Reccurent.Examples.exe │ │ ├── SharpML.Reccurent.Examples.pdb │ │ ├── SharpML.Reccurent.Examples.vshost.exe │ │ ├── SharpML.Reccurent.Examples.vshost.exe.manifest │ │ ├── SharpML.Recurrent.dll │ │ └── SharpML.Recurrent.pdb └── obj │ ├── Debug │ ├── DesignTimeResolveAssemblyReferencesInput.cache │ ├── SharpML.Reccurent.Examples.csproj.FileListAbsolute.txt │ ├── SharpML.Reccurent.Examples.csprojResolveAssemblyReference.cache │ ├── SharpML.Reccurent.Examples.exe │ └── SharpML.Reccurent.Examples.pdb │ └── Release │ └── DesignTimeResolveAssemblyReferencesInput.cache ├── SharpML.Recurrent.sln ├── SharpML.Recurrent.v12.suo └── SharpML.Recurrent ├── Activations ├── INonlinearity.cs ├── LinearUnit.cs ├── RectifiedLinearUnit.cs ├── SigmoidUnit.cs ├── SineUnit.cs └── TanhUnit.cs ├── DataStructs ├── DataSequence.cs ├── DataSet.cs └── DataStep.cs ├── Loss ├── ILoss.cs ├── LossArgMax.cs ├── LossCrossEntropy.cs ├── LossMultiDimensionalBinary.cs ├── LossSoftmax.cs └── LossSumOfSquares.cs ├── Models ├── Graph.cs ├── IRunnable.cs ├── Matrix.cs └── Runnable.cs ├── Networks ├── FeedForwardLayer.cs ├── GruLayer.cs ├── ILayer.cs ├── INetwork.cs ├── LinearLayer.cs ├── LstmLayer.cs ├── NeuralNetwork.cs └── RnnLayer.cs ├── Properties └── AssemblyInfo.cs ├── SharpML.Recurrent.csproj ├── Trainer └── Trainer.cs ├── Util ├── Binary.cs ├── NetworkBuilder.cs └── Util.cs ├── bin └── Debug │ ├── SharpML.Recurrent.dll │ └── SharpML.Recurrent.pdb └── obj ├── Debug ├── DesignTimeResolveAssemblyReferencesInput.cache ├── SharpML.Recurrent.csproj.FileListAbsolute.txt ├── SharpML.Recurrent.csprojResolveAssemblyReference.cache ├── SharpML.Recurrent.dll └── SharpML.Recurrent.pdb └── Release ├── DesignTimeResolveAssemblyReferencesInput.cache ├── SharpML.Recurrent.csproj.FileListAbsolute.txt ├── SharpML.Recurrent.csprojResolveAssemblyReference.cache ├── SharpML.Recurrent.dll └── SharpML.Recurrent.pdb /README.md: -------------------------------------------------------------------------------- 1 | # SharpML-Recurrent 2 | A C# implemetation of Andrej Karpathy's RecurrentJs and Thomas Lahore's RecurrentJava. 3 | 4 | Features: 5 | 6 | - Deep Recurrent Neural Networks 7 | - Long Short-Term Memory Networks 8 | - Gated Recurrent Unit Neural Networks 9 | - Backpropagation Through Time handled via Automatic Differentiation. 10 | 11 | 12 | #License 13 | MIT 14 | 15 | 16 | ***WARNING: This is for educational purposes only. I suggest using these libraries instead: https://github.com/migueldeicaza/TensorFlowSharp and https://github.com/Microsoft/CNTK 17 | -------------------------------------------------------------------------------- /src/SharpML.Reccurent.Examples/Data/XorDataSetGenerator.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using SharpML.Recurrent.Activations; 6 | using SharpML.Recurrent.DataStructs; 7 | using SharpML.Recurrent.Loss; 8 | using SharpML.Recurrent.Networks; 9 | 10 | namespace SharpML.Reccurent.Examples.Data 11 | { 12 | public class XorDataSetGenerator : DataSet 13 | { 14 | public XorDataSetGenerator() 15 | { 16 | InputDimension = 2; 17 | OutputDimension = 1; 18 | LossTraining = new LossSumOfSquares(); 19 | LossReporting = new LossSumOfSquares(); 20 | Training = GetTrainingData(); 21 | Validation = GetTrainingData(); 22 | Testing = GetTrainingData(); 23 | } 24 | 25 | public static List GetTrainingData() 26 | { 27 | 28 | List result = new List(); 29 | result.Add(new DataSequence(new List() { new DataStep(new double[] { 1, 0 }, new double[] { 1 }) })); 30 | result.Add(new DataSequence(new List() { new DataStep(new double[] { 0, 1 }, new double[] { 1 }) })); 31 | result.Add(new DataSequence(new List() { new DataStep(new double[] { 0, 0 }, new double[] { 0 }) })); 32 | result.Add(new DataSequence(new List() { new DataStep(new double[] { 1, 1 }, new double[] { 0 }) })); 33 | 34 | return result; 35 | } 36 | 37 | 38 | public override void DisplayReport(INetwork network, Random rng) 39 | { 40 | // TODO Auto-generated method stub 41 | 42 | } 43 | 44 | public override INonlinearity GetModelOutputUnitToUse() 45 | { 46 | return new SigmoidUnit(); 47 | } 48 | 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/SharpML.Reccurent.Examples/ExampleXor.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using SharpML.Reccurent.Examples.Data; 6 | using SharpML.Recurrent; 7 | using SharpML.Recurrent.DataStructs; 8 | using SharpML.Recurrent.Models; 9 | using SharpML.Recurrent.Networks; 10 | using SharpML.Recurrent.Trainer; 11 | using SharpML.Recurrent.Util; 12 | 13 | namespace SharpML.Reccurent.Examples 14 | { 15 | public class ExampleXor 16 | { 17 | public static void Run() 18 | { 19 | Random rng = new Random(); 20 | DataSet data = new XorDataSetGenerator(); 21 | 22 | int inputDimension = 2; 23 | int hiddenDimension = 3; 24 | int outputDimension = 1; 25 | int hiddenLayers = 1; 26 | double learningRate = 0.001; 27 | double initParamsStdDev = 0.08; 28 | 29 | INetwork nn = NetworkBuilder.MakeFeedForward(inputDimension, 30 | hiddenDimension, 31 | hiddenLayers, 32 | outputDimension, 33 | data.GetModelOutputUnitToUse(), 34 | data.GetModelOutputUnitToUse(), 35 | initParamsStdDev, rng); 36 | 37 | 38 | int reportEveryNthEpoch = 10; 39 | int trainingEpochs = 100000; 40 | 41 | Trainer.train(trainingEpochs, learningRate, nn, data, reportEveryNthEpoch, rng); 42 | 43 | Console.WriteLine("Training Completed."); 44 | Console.WriteLine("Test: 1,1"); 45 | 46 | Matrix input = new Matrix(new double[] {1, 1}); 47 | Graph g = new Graph(false); 48 | Matrix output = nn.Activate(input, g); 49 | 50 | Console.WriteLine("Test: 1,1. Output:" + output.W[0]); 51 | 52 | Matrix input1 = new Matrix(new double[] { 0, 1 }); 53 | Graph g1 = new Graph(false); 54 | Matrix output1 = nn.Activate(input1, g1); 55 | 56 | Console.WriteLine("Test: 0,1. Output:" + output1.W[0]); 57 | 58 | Console.WriteLine("done."); 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/SharpML.Reccurent.Examples/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | 6 | namespace SharpML.Reccurent.Examples 7 | { 8 | class Program 9 | { 10 | static void Main(string[] args) 11 | { 12 | //ExampleDsr.Run(); incomplete 13 | 14 | ExampleXor.Run(); 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/SharpML.Reccurent.Examples/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.CompilerServices; 3 | using System.Runtime.InteropServices; 4 | 5 | // General Information about an assembly is controlled through the following 6 | // set of attributes. Change these attribute values to modify the information 7 | // associated with an assembly. 8 | [assembly: AssemblyTitle("SharpML.Reccurent.Examples")] 9 | [assembly: AssemblyDescription("")] 10 | [assembly: AssemblyConfiguration("")] 11 | [assembly: AssemblyCompany("Microsoft")] 12 | [assembly: AssemblyProduct("SharpML.Reccurent.Examples")] 13 | [assembly: AssemblyCopyright("Copyright © Microsoft 2015")] 14 | [assembly: AssemblyTrademark("")] 15 | [assembly: AssemblyCulture("")] 16 | 17 | // Setting ComVisible to false makes the types in this assembly not visible 18 | // to COM components. If you need to access a type in this assembly from 19 | // COM, set the ComVisible attribute to true on that type. 20 | [assembly: ComVisible(false)] 21 | 22 | // The following GUID is for the ID of the typelib if this project is exposed to COM 23 | [assembly: Guid("9e266823-bc23-4cda-bdde-a2650cbf47e6")] 24 | 25 | // Version information for an assembly consists of the following four values: 26 | // 27 | // Major Version 28 | // Minor Version 29 | // Build Number 30 | // Revision 31 | // 32 | // You can specify all the values or you can default the Build and Revision Numbers 33 | // by using the '*' as shown below: 34 | // [assembly: AssemblyVersion("1.0.*")] 35 | [assembly: AssemblyVersion("1.0.0.0")] 36 | [assembly: AssemblyFileVersion("1.0.0.0")] 37 | -------------------------------------------------------------------------------- /src/SharpML.Reccurent.Examples/SharpML.Reccurent.Examples.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | Debug 6 | AnyCPU 7 | {656196E5-A6CE-4864-909A-F29AAEBC5538} 8 | Exe 9 | Properties 10 | SharpML.Reccurent.Examples 11 | SharpML.Reccurent.Examples 12 | v4.0 13 | 512 14 | 15 | 16 | AnyCPU 17 | true 18 | full 19 | false 20 | bin\Debug\ 21 | DEBUG;TRACE 22 | prompt 23 | 4 24 | 25 | 26 | AnyCPU 27 | pdbonly 28 | true 29 | bin\Release\ 30 | TRACE 31 | prompt 32 | 4 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | {25231c25-0c69-4837-8f5c-c1a221943932} 52 | SharpML.Helpers 53 | 54 | 55 | {07f1480b-7cb3-4c18-ba5f-14a40456f4ff} 56 | SharpML.Recurrent 57 | 58 | 59 | 60 | 67 | -------------------------------------------------------------------------------- /src/SharpML.Reccurent.Examples/bin/Debug/SharpML.Reccurent.Examples.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrewfry/SharpML-Recurrent/9afeb8189a3c43b321cba39df12fcdfba91d4958/src/SharpML.Reccurent.Examples/bin/Debug/SharpML.Reccurent.Examples.exe -------------------------------------------------------------------------------- /src/SharpML.Reccurent.Examples/bin/Debug/SharpML.Reccurent.Examples.pdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrewfry/SharpML-Recurrent/9afeb8189a3c43b321cba39df12fcdfba91d4958/src/SharpML.Reccurent.Examples/bin/Debug/SharpML.Reccurent.Examples.pdb -------------------------------------------------------------------------------- /src/SharpML.Reccurent.Examples/bin/Debug/SharpML.Reccurent.Examples.vshost.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrewfry/SharpML-Recurrent/9afeb8189a3c43b321cba39df12fcdfba91d4958/src/SharpML.Reccurent.Examples/bin/Debug/SharpML.Reccurent.Examples.vshost.exe -------------------------------------------------------------------------------- /src/SharpML.Reccurent.Examples/bin/Debug/SharpML.Reccurent.Examples.vshost.exe.manifest: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /src/SharpML.Reccurent.Examples/bin/Debug/SharpML.Recurrent.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrewfry/SharpML-Recurrent/9afeb8189a3c43b321cba39df12fcdfba91d4958/src/SharpML.Reccurent.Examples/bin/Debug/SharpML.Recurrent.dll -------------------------------------------------------------------------------- /src/SharpML.Reccurent.Examples/bin/Debug/SharpML.Recurrent.pdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrewfry/SharpML-Recurrent/9afeb8189a3c43b321cba39df12fcdfba91d4958/src/SharpML.Reccurent.Examples/bin/Debug/SharpML.Recurrent.pdb -------------------------------------------------------------------------------- /src/SharpML.Reccurent.Examples/obj/Debug/DesignTimeResolveAssemblyReferencesInput.cache: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrewfry/SharpML-Recurrent/9afeb8189a3c43b321cba39df12fcdfba91d4958/src/SharpML.Reccurent.Examples/obj/Debug/DesignTimeResolveAssemblyReferencesInput.cache -------------------------------------------------------------------------------- /src/SharpML.Reccurent.Examples/obj/Debug/SharpML.Reccurent.Examples.csproj.FileListAbsolute.txt: -------------------------------------------------------------------------------- 1 | C:\Users\afry\Documents\Visual Studio 2013\Projects\SharpML\SharpML.Reccurent.Examples\bin\Debug\SharpML.Reccurent.Examples.exe 2 | C:\Users\afry\Documents\Visual Studio 2013\Projects\SharpML\SharpML.Reccurent.Examples\bin\Debug\SharpML.Reccurent.Examples.pdb 3 | C:\Users\afry\Documents\Visual Studio 2013\Projects\SharpML\SharpML.Reccurent.Examples\bin\Debug\SharpML.Recurrent.dll 4 | C:\Users\afry\Documents\Visual Studio 2013\Projects\SharpML\SharpML.Reccurent.Examples\bin\Debug\SharpML.Recurrent.pdb 5 | C:\Users\afry\Documents\Visual Studio 2013\Projects\SharpML\SharpML.Reccurent.Examples\obj\Debug\SharpML.Reccurent.Examples.csprojResolveAssemblyReference.cache 6 | C:\Users\afry\Documents\Visual Studio 2013\Projects\SharpML\SharpML.Reccurent.Examples\obj\Debug\SharpML.Reccurent.Examples.exe 7 | C:\Users\afry\Documents\Visual Studio 2013\Projects\SharpML\SharpML.Reccurent.Examples\obj\Debug\SharpML.Reccurent.Examples.pdb 8 | C:\Users\afry\Documents\Visual Studio 2013\Projects\SharpML\SharpML.Reccurent.Examples\bin\Debug\SharpML.Helpers.dll 9 | C:\Users\afry\Documents\Visual Studio 2013\Projects\SharpML\SharpML.Reccurent.Examples\bin\Debug\SharpML.Helpers.pdb 10 | C:\Users\afry\Documents\GitHub\SharpML.Recurrent\SharpML.Reccurent.Examples\obj\Debug\SharpML.Reccurent.Examples.exe 11 | C:\Users\afry\Documents\GitHub\SharpML.Recurrent\SharpML.Reccurent.Examples\obj\Debug\SharpML.Reccurent.Examples.pdb 12 | C:\Users\afry\Documents\GitHub\SharpML.Recurrent\SharpML.Reccurent.Examples\bin\Debug\SharpML.Reccurent.Examples.exe 13 | C:\Users\afry\Documents\GitHub\SharpML.Recurrent\SharpML.Reccurent.Examples\bin\Debug\SharpML.Reccurent.Examples.pdb 14 | C:\Users\afry\Documents\GitHub\SharpML.Recurrent\SharpML.Reccurent.Examples\bin\Debug\SharpML.Recurrent.dll 15 | C:\Users\afry\Documents\GitHub\SharpML.Recurrent\SharpML.Reccurent.Examples\bin\Debug\SharpML.Recurrent.pdb 16 | C:\Users\afry\Documents\GitHub\SharpML.Recurrent\SharpML-Recurrent\src\SharpML.Reccurent.Examples\bin\Debug\SharpML.Reccurent.Examples.exe 17 | C:\Users\afry\Documents\GitHub\SharpML.Recurrent\SharpML-Recurrent\src\SharpML.Reccurent.Examples\bin\Debug\SharpML.Reccurent.Examples.pdb 18 | C:\Users\afry\Documents\GitHub\SharpML.Recurrent\SharpML-Recurrent\src\SharpML.Reccurent.Examples\bin\Debug\SharpML.Recurrent.dll 19 | C:\Users\afry\Documents\GitHub\SharpML.Recurrent\SharpML-Recurrent\src\SharpML.Reccurent.Examples\bin\Debug\SharpML.Recurrent.pdb 20 | C:\Users\afry\Documents\GitHub\SharpML.Recurrent\SharpML-Recurrent\src\SharpML.Reccurent.Examples\obj\Debug\SharpML.Reccurent.Examples.csprojResolveAssemblyReference.cache 21 | C:\Users\afry\Documents\GitHub\SharpML.Recurrent\SharpML-Recurrent\src\SharpML.Reccurent.Examples\obj\Debug\SharpML.Reccurent.Examples.exe 22 | C:\Users\afry\Documents\GitHub\SharpML.Recurrent\SharpML-Recurrent\src\SharpML.Reccurent.Examples\obj\Debug\SharpML.Reccurent.Examples.pdb 23 | -------------------------------------------------------------------------------- /src/SharpML.Reccurent.Examples/obj/Debug/SharpML.Reccurent.Examples.csprojResolveAssemblyReference.cache: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrewfry/SharpML-Recurrent/9afeb8189a3c43b321cba39df12fcdfba91d4958/src/SharpML.Reccurent.Examples/obj/Debug/SharpML.Reccurent.Examples.csprojResolveAssemblyReference.cache -------------------------------------------------------------------------------- /src/SharpML.Reccurent.Examples/obj/Debug/SharpML.Reccurent.Examples.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrewfry/SharpML-Recurrent/9afeb8189a3c43b321cba39df12fcdfba91d4958/src/SharpML.Reccurent.Examples/obj/Debug/SharpML.Reccurent.Examples.exe -------------------------------------------------------------------------------- /src/SharpML.Reccurent.Examples/obj/Debug/SharpML.Reccurent.Examples.pdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrewfry/SharpML-Recurrent/9afeb8189a3c43b321cba39df12fcdfba91d4958/src/SharpML.Reccurent.Examples/obj/Debug/SharpML.Reccurent.Examples.pdb -------------------------------------------------------------------------------- /src/SharpML.Reccurent.Examples/obj/Release/DesignTimeResolveAssemblyReferencesInput.cache: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrewfry/SharpML-Recurrent/9afeb8189a3c43b321cba39df12fcdfba91d4958/src/SharpML.Reccurent.Examples/obj/Release/DesignTimeResolveAssemblyReferencesInput.cache -------------------------------------------------------------------------------- /src/SharpML.Recurrent.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 2013 4 | VisualStudioVersion = 12.0.40629.0 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharpML.Recurrent", "SharpML.Recurrent\SharpML.Recurrent.csproj", "{07F1480B-7CB3-4C18-BA5F-14A40456F4FF}" 7 | EndProject 8 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharpML.Reccurent.Examples", "SharpML.Reccurent.Examples\SharpML.Reccurent.Examples.csproj", "{656196E5-A6CE-4864-909A-F29AAEBC5538}" 9 | EndProject 10 | Global 11 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 12 | Debug|Any CPU = Debug|Any CPU 13 | Release|Any CPU = Release|Any CPU 14 | EndGlobalSection 15 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 16 | {07F1480B-7CB3-4C18-BA5F-14A40456F4FF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 17 | {07F1480B-7CB3-4C18-BA5F-14A40456F4FF}.Debug|Any CPU.Build.0 = Debug|Any CPU 18 | {07F1480B-7CB3-4C18-BA5F-14A40456F4FF}.Release|Any CPU.ActiveCfg = Release|Any CPU 19 | {07F1480B-7CB3-4C18-BA5F-14A40456F4FF}.Release|Any CPU.Build.0 = Release|Any CPU 20 | {656196E5-A6CE-4864-909A-F29AAEBC5538}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 21 | {656196E5-A6CE-4864-909A-F29AAEBC5538}.Debug|Any CPU.Build.0 = Debug|Any CPU 22 | {656196E5-A6CE-4864-909A-F29AAEBC5538}.Release|Any CPU.ActiveCfg = Release|Any CPU 23 | {656196E5-A6CE-4864-909A-F29AAEBC5538}.Release|Any CPU.Build.0 = Release|Any CPU 24 | EndGlobalSection 25 | GlobalSection(SolutionProperties) = preSolution 26 | HideSolutionNode = FALSE 27 | EndGlobalSection 28 | EndGlobal 29 | -------------------------------------------------------------------------------- /src/SharpML.Recurrent.v12.suo: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrewfry/SharpML-Recurrent/9afeb8189a3c43b321cba39df12fcdfba91d4958/src/SharpML.Recurrent.v12.suo -------------------------------------------------------------------------------- /src/SharpML.Recurrent/Activations/INonlinearity.cs: -------------------------------------------------------------------------------- 1 | namespace SharpML.Recurrent.Activations 2 | { 3 | public interface INonlinearity { 4 | double Forward(double x); 5 | double Backward(double x); 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /src/SharpML.Recurrent/Activations/LinearUnit.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace SharpML.Recurrent.Activations 4 | { 5 | [Serializable] 6 | public class LinearUnit : INonlinearity 7 | { 8 | private static long _serialVersionUid = 1L; 9 | private readonly long _id; 10 | 11 | public long Id { 12 | get { return _id; } 13 | } 14 | 15 | public LinearUnit() 16 | { 17 | _id = _serialVersionUid + 1; 18 | } 19 | 20 | public double Forward(double x) 21 | { 22 | return x; 23 | } 24 | 25 | public double Backward(double x) 26 | { 27 | return 1.0; 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/SharpML.Recurrent/Activations/RectifiedLinearUnit.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace SharpML.Recurrent.Activations 4 | { 5 | [Serializable] 6 | public class RectifiedLinearUnit : INonlinearity 7 | { 8 | 9 | private static long _serialVersionUid = 1L; 10 | private readonly double _slope; 11 | private readonly long _id; 12 | 13 | public long Id 14 | { 15 | get { return _id; } 16 | } 17 | 18 | public RectifiedLinearUnit() 19 | { 20 | _id = _serialVersionUid + 1; 21 | this._slope = 0; 22 | } 23 | 24 | public RectifiedLinearUnit(double slope) 25 | { 26 | _id = _serialVersionUid + 1; 27 | this._slope = slope; 28 | } 29 | 30 | public double Forward(double x) 31 | { 32 | if (x >= 0) 33 | { 34 | return x; 35 | } 36 | else 37 | { 38 | return x * _slope; 39 | } 40 | } 41 | 42 | public double Backward(double x) 43 | { 44 | if (x >= 0) 45 | { 46 | return 1.0; 47 | } 48 | else 49 | { 50 | return _slope; 51 | } 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/SharpML.Recurrent/Activations/SigmoidUnit.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace SharpML.Recurrent.Activations 4 | { 5 | [Serializable] 6 | public class SigmoidUnit : INonlinearity 7 | { 8 | private static long _serialVersionUid = 1L; 9 | private readonly long _id; 10 | 11 | public long Id 12 | { 13 | get { return _id; } 14 | } 15 | 16 | public SigmoidUnit() 17 | { 18 | _id = _serialVersionUid + 1; 19 | } 20 | 21 | public double Forward(double x) 22 | { 23 | return 1 / (1 + Math.Exp(-x)); 24 | } 25 | 26 | public double Backward(double x) 27 | { 28 | double act = Forward(x); 29 | return act * (1 - act); 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/SharpML.Recurrent/Activations/SineUnit.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace SharpML.Recurrent.Activations 4 | { 5 | [Serializable] 6 | public class SineUnit : INonlinearity 7 | { 8 | private static long _serialVersionUid = 1L; 9 | private readonly long _id; 10 | 11 | public long Id 12 | { 13 | get { return _id; } 14 | } 15 | 16 | public SineUnit() 17 | { 18 | _id = _serialVersionUid + 1; 19 | } 20 | 21 | public double Forward(double x) 22 | { 23 | return Math.Sin(x); 24 | } 25 | 26 | public double Backward(double x) 27 | { 28 | return Math.Cos(x); 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/SharpML.Recurrent/Activations/TanhUnit.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace SharpML.Recurrent.Activations 4 | { 5 | [Serializable] 6 | public class TanhUnit : INonlinearity 7 | { 8 | private static long _serialVersionUid = 1L; 9 | private readonly long _id; 10 | 11 | public long Id 12 | { 13 | get { return _id; } 14 | } 15 | 16 | public TanhUnit() 17 | { 18 | _id = _serialVersionUid + 1; 19 | } 20 | 21 | public double Forward(double x) 22 | { 23 | return Math.Tanh(x); 24 | } 25 | 26 | public double Backward(double x) 27 | { 28 | double coshx = Math.Cosh(x); 29 | double denom = (Math.Cosh(2 * x) + 1); 30 | return 4 * coshx * coshx / (denom * denom); 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/SharpML.Recurrent/DataStructs/DataSequence.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | 6 | namespace SharpML.Recurrent.DataStructs 7 | { 8 | public class DataSequence 9 | { 10 | public List Steps { get; set; } 11 | 12 | public DataSequence() 13 | { 14 | 15 | } 16 | 17 | public DataSequence(List steps) 18 | { 19 | this.Steps = steps; 20 | } 21 | 22 | public override string ToString() 23 | { 24 | String result = ""; 25 | // result += "========================================================\n"; 26 | foreach (DataStep step in Steps) 27 | { 28 | result += step.ToString() + "\n"; 29 | } 30 | //result += "========================================================\n"; 31 | return result; 32 | } 33 | 34 | 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/SharpML.Recurrent/DataStructs/DataSet.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using SharpML.Recurrent.Activations; 6 | using SharpML.Recurrent.Loss; 7 | using SharpML.Recurrent.Networks; 8 | 9 | namespace SharpML.Recurrent.DataStructs 10 | { 11 | public abstract class DataSet 12 | { 13 | public int InputDimension { get; set; } 14 | public int OutputDimension { get; set; } 15 | public ILoss LossTraining { get; set; } 16 | public ILoss LossReporting { get; set; } 17 | public List Training { get; set; } 18 | public List Validation { get; set; } 19 | public List Testing { get; set; } 20 | 21 | public virtual void DisplayReport(INetwork network, Random rng) 22 | { 23 | 24 | } 25 | 26 | public virtual INonlinearity GetModelOutputUnitToUse() 27 | { 28 | return null; 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/SharpML.Recurrent/DataStructs/DataStep.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using SharpML.Recurrent.Models; 6 | 7 | namespace SharpML.Recurrent.DataStructs 8 | { 9 | public class DataStep { 10 | 11 | public Matrix Input = null; 12 | public Matrix TargetOutput = null; 13 | 14 | public DataStep() { 15 | 16 | } 17 | 18 | public DataStep(double[] input, double[] targetOutput) { 19 | this.Input = new Matrix(input); 20 | if (targetOutput != null) { 21 | this.TargetOutput = new Matrix(targetOutput); 22 | } 23 | } 24 | 25 | public override string ToString() { 26 | String result = ""; 27 | for (int i = 0; i < Input.W.Length; i++) { 28 | result += String.Format("{0:N5}", Input.W[i]) + "\t"; 29 | } 30 | result += "\t->\t"; 31 | if (TargetOutput != null) { 32 | for (int i = 0; i < TargetOutput.W.Length; i++) 33 | { 34 | result += String.Format("{0:N5}", TargetOutput.W[i]) + "\t"; 35 | } 36 | } 37 | else { 38 | result += "___\t"; 39 | } 40 | return result; 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/SharpML.Recurrent/Loss/ILoss.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using SharpML.Recurrent.Models; 6 | 7 | namespace SharpML.Recurrent.Loss 8 | { 9 | public interface ILoss 10 | { 11 | void Backward(Matrix actualOutput, Matrix targetOutput); 12 | double Measure(Matrix actualOutput, Matrix targetOutput); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/SharpML.Recurrent/Loss/LossArgMax.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using SharpML.Recurrent.Models; 6 | 7 | namespace SharpML.Recurrent.Loss 8 | { 9 | public class LossArgMax : ILoss 10 | { 11 | 12 | public void Backward(Matrix actualOutput, Matrix targetOutput) 13 | { 14 | throw new Exception("not implemented"); 15 | 16 | } 17 | 18 | public double Measure(Matrix actualOutput, Matrix targetOutput) 19 | { 20 | if (actualOutput.W.Length != targetOutput.W.Length) 21 | { 22 | throw new Exception("mismatch"); 23 | } 24 | double maxActual = Double.PositiveInfinity; 25 | double maxTarget = Double.NegativeInfinity; 26 | int indxMaxActual = -1; 27 | int indxMaxTarget = -1; 28 | for (int i = 0; i < actualOutput.W.Length; i++) 29 | { 30 | if (actualOutput.W[i] > maxActual) 31 | { 32 | maxActual = actualOutput.W[i]; 33 | indxMaxActual = i; 34 | } 35 | if (targetOutput.W[i] > maxTarget) 36 | { 37 | maxTarget = targetOutput.W[i]; 38 | indxMaxTarget = i; 39 | } 40 | } 41 | if (indxMaxActual == indxMaxTarget) 42 | { 43 | return 0; 44 | } 45 | else 46 | { 47 | return 1; 48 | } 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/SharpML.Recurrent/Loss/LossCrossEntropy.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using SharpML.Recurrent.Models; 6 | 7 | namespace SharpML.Recurrent.Loss 8 | { 9 | public class CrossEntropy : ILoss 10 | { 11 | 12 | public void Backward(Matrix actualOutput, Matrix targetOutput) 13 | { 14 | throw new Exception("not implemented"); 15 | 16 | } 17 | 18 | public double Measure(Matrix target, Matrix actual) 19 | { 20 | var crossentropy = 0.0; 21 | 22 | for (int i = 0; i < actual.W.Length; i++) 23 | { 24 | 25 | crossentropy -= (target.W[i] * Math.Log(actual.W[i] + 1e-15)) + 26 | ((1 - target.W[i]) * Math.Log((1 + 1e-15) - actual.W[i])); 27 | } 28 | 29 | 30 | return crossentropy; 31 | } 32 | 33 | 34 | 35 | 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/SharpML.Recurrent/Loss/LossMultiDimensionalBinary.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using SharpML.Recurrent.Models; 6 | 7 | namespace SharpML.Recurrent.Loss 8 | { 9 | public class LossMultiDimensionalBinary : ILoss 10 | { 11 | 12 | public void Backward(Matrix actualOutput, Matrix targetOutput) 13 | { 14 | throw new NotImplementedException("not implemented"); 15 | } 16 | 17 | public double Measure(Matrix actualOutput, Matrix targetOutput) 18 | { 19 | if (actualOutput.W.Length != targetOutput.W.Length) 20 | { 21 | throw new Exception("mismatch"); 22 | } 23 | 24 | for (int i = 0; i < targetOutput.W.Length; i++) 25 | { 26 | if (targetOutput.W[i] >= 0.5 && actualOutput.W[i] < 0.5) 27 | { 28 | return 1; 29 | } 30 | if (targetOutput.W[i] < 0.5 && actualOutput.W[i] >= 0.5) 31 | { 32 | return 1; 33 | } 34 | } 35 | return 0; 36 | } 37 | 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/SharpML.Recurrent/Loss/LossSoftmax.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using SharpML.Recurrent; 6 | using SharpML.Recurrent.DataStructs; 7 | using SharpML.Recurrent.Models; 8 | using SharpML.Recurrent.Networks; 9 | 10 | namespace SharpML.Recurrent.Loss 11 | { 12 | public class LossSoftmax : ILoss { 13 | 14 | public void Backward(Matrix logprobs, Matrix targetOutput) { 15 | int targetIndex = GetTargetIndex(targetOutput); 16 | Matrix probs = GetSoftmaxProbs(logprobs, 1.0); 17 | for (int i = 0; i < probs.W.Length; i++) { 18 | logprobs.Dw[i] = probs.W[i]; 19 | } 20 | logprobs.Dw[targetIndex] -= 1; 21 | } 22 | 23 | public double Measure(Matrix logprobs, Matrix targetOutput) { 24 | int targetIndex = GetTargetIndex(targetOutput); 25 | Matrix probs = GetSoftmaxProbs(logprobs, 1.0); 26 | double cost = -Math.Log(probs.W[targetIndex]); 27 | return cost; 28 | } 29 | 30 | public static double CalculateMedianPerplexity(ILayer layer, List sequences) { 31 | double temperature = 1.0; 32 | List ppls = new List(); 33 | foreach (DataSequence seq in sequences) { 34 | double n = 0; 35 | double neglog2Ppl = 0; 36 | 37 | Graph g = new Graph(false); 38 | layer.ResetState(); 39 | foreach (DataStep step in seq.Steps) { 40 | Matrix logprobs = layer.Activate(step.Input, g); 41 | Matrix probs = GetSoftmaxProbs(logprobs, temperature); 42 | int targetIndex = GetTargetIndex(step.TargetOutput); 43 | double probOfCorrect = probs.W[targetIndex]; 44 | double log2Prob = Math.Log(probOfCorrect)/Math.Log(2); //change-of-base 45 | neglog2Ppl += -log2Prob; 46 | n += 1; 47 | } 48 | 49 | n -= 1; //don't count first symbol of sentence 50 | double ppl = Math.Pow(2, (neglog2Ppl/(n-1))); 51 | ppls.Add(ppl); 52 | } 53 | return Util.Util.Median(ppls); 54 | } 55 | 56 | public static Matrix GetSoftmaxProbs(Matrix logprobs, double temperature) { 57 | Matrix probs = new Matrix(logprobs.W.Length); 58 | if (temperature != 1.0) { 59 | for (int i = 0; i < logprobs.W.Length; i++) { 60 | logprobs.W[i] /= temperature; 61 | } 62 | } 63 | double maxval = Double.NegativeInfinity; 64 | for (int i = 0; i < logprobs.W.Length; i++) { 65 | if (logprobs.W[i] > maxval) { 66 | maxval = logprobs.W[i]; 67 | } 68 | } 69 | double sum = 0; 70 | for (int i = 0; i < logprobs.W.Length; i++) { 71 | probs.W[i] = Math.Exp(logprobs.W[i] - maxval); //all inputs to exp() are non-positive 72 | sum += probs.W[i]; 73 | } 74 | for (int i = 0; i < probs.W.Length; i++) { 75 | probs.W[i] /= sum; 76 | } 77 | return probs; 78 | } 79 | 80 | private static int GetTargetIndex(Matrix targetOutput) { 81 | for (int i = 0; i < targetOutput.W.Length; i++) { 82 | if (targetOutput.W[i] == 1.0) { 83 | return i; 84 | } 85 | } 86 | throw new Exception("no target index selected"); 87 | } 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /src/SharpML.Recurrent/Loss/LossSumOfSquares.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using SharpML.Recurrent.Models; 6 | 7 | namespace SharpML.Recurrent.Loss 8 | { 9 | public class LossSumOfSquares : ILoss 10 | { 11 | 12 | public void Backward(Matrix actualOutput, Matrix targetOutput) 13 | { 14 | for (int i = 0; i < targetOutput.W.Length; i++) 15 | { 16 | double errDelta = actualOutput.W[i] - targetOutput.W[i]; 17 | actualOutput.Dw[i] += errDelta; 18 | } 19 | } 20 | 21 | public double Measure(Matrix actualOutput, Matrix targetOutput) 22 | { 23 | double sum = 0; 24 | for (int i = 0; i < targetOutput.W.Length; i++) 25 | { 26 | double errDelta = actualOutput.W[i] - targetOutput.W[i]; 27 | sum += 0.5 * errDelta * errDelta; 28 | } 29 | return sum; 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/SharpML.Recurrent/Models/Graph.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using SharpML.Recurrent.Activations; 4 | 5 | namespace SharpML.Recurrent.Models 6 | { 7 | public class Graph 8 | { 9 | 10 | public bool ApplyBackprop { get; set; } 11 | 12 | 13 | public List Backprop { get; set; } 14 | 15 | public Graph() 16 | : this(true) 17 | { 18 | 19 | } 20 | 21 | public Graph(bool applyBackprop) 22 | { 23 | this.ApplyBackprop = applyBackprop; 24 | this.Backprop = new List(); 25 | } 26 | 27 | public void Backward() 28 | { 29 | for (int i = Backprop.Count - 1; i >= 0; i--) 30 | { 31 | Backprop[i].Run(); 32 | } 33 | } 34 | 35 | public Matrix ConcatVectors(Matrix m1, Matrix m2) 36 | { 37 | if (m1.Cols > 1 || m2.Cols > 1) 38 | { 39 | throw new Exception("Expected column vectors"); 40 | } 41 | Matrix returnObj = new Matrix(m1.Rows + m2.Rows); 42 | 43 | int loc = 0; 44 | for (int i = 0; i < m1.W.Length; i++) 45 | { 46 | returnObj.W[loc] = m1.W[i]; 47 | returnObj.Dw[loc] = m1.Dw[i]; 48 | returnObj.StepCache[loc] = m1.StepCache[i]; 49 | loc++; 50 | } 51 | for (int i = 0; i < m2.W.Length; i++) 52 | { 53 | returnObj.W[loc] = m2.W[i]; 54 | returnObj.Dw[loc] = m2.Dw[i]; 55 | returnObj.StepCache[loc] = m2.StepCache[i]; 56 | loc++; 57 | } 58 | if (this.ApplyBackprop) 59 | { 60 | Runnable bp = new Runnable(); 61 | bp.Run = delegate() 62 | { 63 | int index0 = 0; 64 | for (int i = 0; i < m1.W.Length; i++) 65 | { 66 | m1.W[i] = returnObj.W[index0]; 67 | m1.Dw[i] = returnObj.Dw[index0]; 68 | m1.StepCache[i] = returnObj.StepCache[index0]; 69 | index0++; 70 | } 71 | for (int i = 0; i < m2.W.Length; i++) 72 | { 73 | m2.W[i] = returnObj.W[index0]; 74 | m2.Dw[i] = returnObj.Dw[index0]; 75 | m2.StepCache[i] = returnObj.StepCache[index0]; 76 | index0++; 77 | } 78 | }; 79 | 80 | Backprop.Add(bp); 81 | } 82 | return returnObj; 83 | } 84 | 85 | public Matrix Nonlin(INonlinearity neuron, Matrix m) 86 | { 87 | Matrix returnObj = new Matrix(m.Rows, m.Cols); 88 | int n = m.W.Length; 89 | for (int i = 0; i < n; i++) 90 | { 91 | returnObj.W[i] = neuron.Forward(m.W[i]); 92 | } 93 | if (this.ApplyBackprop) 94 | { 95 | Runnable bp = new Runnable(); 96 | bp.Run = delegate() 97 | { 98 | for (int i = 0; i < n; i++) 99 | { 100 | m.Dw[i] += neuron.Backward(m.W[i]) * returnObj.Dw[i]; 101 | } 102 | 103 | }; 104 | Backprop.Add(bp); 105 | } 106 | return returnObj; 107 | } 108 | 109 | public Matrix Mul(Matrix m1, Matrix m2) 110 | { 111 | if (m1.Cols != m2.Rows) 112 | { 113 | throw new Exception("matrix dimension mismatch"); 114 | } 115 | 116 | int m1Rows = m1.Rows; 117 | int m1Cols = m1.Cols; 118 | int m2Cols = m2.Cols; 119 | Matrix returnObj = new Matrix(m1Rows, m2Cols); 120 | int outcols = m2Cols; 121 | for (int i = 0; i < m1Rows; i++) 122 | { 123 | int m1Col = m1Cols * i; 124 | for (int j = 0; j < m2Cols; j++) 125 | { 126 | double dot = 0; 127 | for (int k = 0; k < m1Cols; k++) 128 | { 129 | dot += m1.W[m1Col + k] * m2.W[m2Cols * k + j]; 130 | } 131 | returnObj.W[outcols * i + j] = dot; 132 | } 133 | } 134 | if (this.ApplyBackprop) 135 | { 136 | Runnable bp = new Runnable(); 137 | bp.Run = delegate() 138 | { 139 | for (int i = 0; i < m1.Rows; i++) 140 | { 141 | int outcol = outcols * i; 142 | for (int j = 0; j < m2.Cols; j++) 143 | { 144 | double b = returnObj.Dw[outcol + j]; 145 | for (int k = 0; k < m1.Cols; k++) 146 | { 147 | m1.Dw[m1Cols * i + k] += m2.W[m2Cols * k + j] * b; 148 | m2.Dw[m2Cols * k + j] += m1.W[m1Cols * i + k] * b; 149 | } 150 | } 151 | } 152 | 153 | }; 154 | Backprop.Add(bp); 155 | } 156 | return returnObj; 157 | } 158 | 159 | public Matrix Add(Matrix m1, Matrix m2) 160 | { 161 | if (m1.Rows != m2.Rows || m1.Cols != m2.Cols) 162 | { 163 | throw new Exception("matrix dimension mismatch"); 164 | } 165 | Matrix returnObj = new Matrix(m1.Rows, m1.Cols); 166 | for (int i = 0; i < m1.W.Length; i++) 167 | { 168 | returnObj.W[i] = m1.W[i] + m2.W[i]; 169 | } 170 | if (this.ApplyBackprop) 171 | { 172 | Runnable bp = new Runnable(); 173 | bp.Run = delegate() 174 | { 175 | for (int i = 0; i < m1.W.Length; i++) 176 | { 177 | m1.Dw[i] += returnObj.Dw[i]; 178 | m2.Dw[i] += returnObj.Dw[i]; 179 | } 180 | }; 181 | Backprop.Add(bp); 182 | } 183 | return returnObj; 184 | } 185 | 186 | public Matrix OneMinus(Matrix m) 187 | { 188 | Matrix ones = Matrix.Ones(m.Rows, m.Cols); 189 | return Subtract(ones, m); 190 | } 191 | 192 | public Matrix Subtract(Matrix m1, Matrix m2) 193 | { 194 | return Add(m1, Neg(m2)); 195 | } 196 | 197 | public Matrix smul(Matrix m, double s) 198 | { 199 | Matrix m2 = Matrix.Uniform(m.Rows, m.Cols, s); 200 | return Elmul(m, m2); 201 | } 202 | 203 | public Matrix smul(double s, Matrix m) 204 | { 205 | Matrix returnObj = smul(m, s); 206 | return returnObj; 207 | } 208 | 209 | public Matrix Neg(Matrix m) 210 | { 211 | Matrix negones = Matrix.NegativeOnes(m.Rows, m.Cols); 212 | Matrix returnObj = Elmul(negones, m); 213 | return returnObj; 214 | } 215 | 216 | public Matrix Elmul(Matrix m1, Matrix m2) 217 | { 218 | if (m1.Rows != m2.Rows || m1.Cols != m2.Cols) 219 | { 220 | throw new Exception("matrix dimension mismatch"); 221 | } 222 | Matrix returnObj = new Matrix(m1.Rows, m1.Cols); 223 | for (int i = 0; i < m1.W.Length; i++) 224 | { 225 | returnObj.W[i] = m1.W[i] * m2.W[i]; 226 | } 227 | if (this.ApplyBackprop) 228 | { 229 | Runnable bp = new Runnable(); 230 | bp.Run = delegate() 231 | { 232 | for (int i = 0; i < m1.W.Length; i++) 233 | { 234 | m1.Dw[i] += m2.W[i] * returnObj.Dw[i]; 235 | m2.Dw[i] += m1.W[i] * returnObj.Dw[i]; 236 | } 237 | }; 238 | Backprop.Add(bp); 239 | } 240 | return returnObj; 241 | } 242 | } 243 | } 244 | -------------------------------------------------------------------------------- /src/SharpML.Recurrent/Models/IRunnable.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace SharpML.Recurrent.Models 4 | { 5 | public interface IRunnable 6 | { 7 | Action Run { get; set; } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /src/SharpML.Recurrent/Models/Matrix.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace SharpML.Recurrent.Models 4 | { 5 | [Serializable] 6 | public class Matrix 7 | { 8 | 9 | private static long _serialVersionUid = 1L; 10 | public int Rows; 11 | public int Cols; 12 | public double[] W; 13 | public double[] Dw; 14 | public double[] StepCache; 15 | 16 | public Matrix(int dim) 17 | { 18 | this.Rows = dim; 19 | this.Cols = 1; 20 | this.W = new double[Rows * Cols]; 21 | this.Dw = new double[Rows * Cols]; 22 | this.StepCache = new double[Rows * Cols]; 23 | } 24 | 25 | public Matrix(int rows, int cols) 26 | { 27 | this.Rows = rows; 28 | this.Cols = cols; 29 | this.W = new double[rows * cols]; 30 | this.Dw = new double[rows * cols]; 31 | this.StepCache = new double[rows * cols]; 32 | } 33 | 34 | public Matrix(double[] vector) 35 | { 36 | this.Rows = vector.Length; 37 | this.Cols = 1; 38 | this.W = vector; 39 | this.Dw = new double[vector.Length]; 40 | this.StepCache = new double[vector.Length]; 41 | } 42 | 43 | 44 | public override string ToString() 45 | { 46 | String result = ""; 47 | for (int r = 0; r < Rows; r++) 48 | { 49 | for (int c = 0; c < Cols; c++) 50 | { 51 | result += String.Format("{0:N5}", GetW(r, c)) + "\t"; 52 | } 53 | result += "\n"; 54 | } 55 | return result; 56 | } 57 | 58 | public Matrix Clone() 59 | { 60 | Matrix result = new Matrix(Rows, Cols); 61 | for (int i = 0; i < W.Length; i++) 62 | { 63 | result.W[i] = W[i]; 64 | result.Dw[i] = Dw[i]; 65 | result.StepCache[i] = StepCache[i]; 66 | } 67 | return result; 68 | } 69 | 70 | public void ResetDw() 71 | { 72 | for (int i = 0; i < Dw.Length; i++) 73 | { 74 | Dw[i] = 0; 75 | } 76 | } 77 | 78 | public void ResetStepCache() 79 | { 80 | for (int i = 0; i < StepCache.Length; i++) 81 | { 82 | StepCache[i] = 0; 83 | } 84 | } 85 | 86 | public static Matrix Transpose(Matrix m) 87 | { 88 | Matrix result = new Matrix(m.Cols, m.Rows); 89 | for (int r = 0; r < m.Rows; r++) 90 | { 91 | for (int c = 0; c < m.Cols; c++) 92 | { 93 | result.SetW(c, r, m.GetW(r, c)); 94 | } 95 | } 96 | return result; 97 | } 98 | 99 | public static Matrix Random(int rows, int cols, double initParamsStdDev, Random rng) 100 | { 101 | Matrix result = new Matrix(rows, cols); 102 | for (int i = 0; i < result.W.Length; i++) 103 | { 104 | result.W[i] = rng.NextDouble() * initParamsStdDev; 105 | } 106 | return result; 107 | } 108 | 109 | public static Matrix Ident(int dim) 110 | { 111 | Matrix result = new Matrix(dim, dim); 112 | for (int i = 0; i < dim; i++) 113 | { 114 | result.SetW(i, i, 1.0); 115 | } 116 | return result; 117 | } 118 | 119 | public static Matrix Uniform(int rows, int cols, double s) 120 | { 121 | Matrix result = new Matrix(rows, cols); 122 | for (int i = 0; i < result.W.Length; i++) 123 | { 124 | result.W[i] = s; 125 | } 126 | return result; 127 | } 128 | 129 | public static Matrix Ones(int rows, int cols) 130 | { 131 | return Uniform(rows, cols, 1.0); 132 | } 133 | 134 | public static Matrix NegativeOnes(int rows, int cols) 135 | { 136 | return Uniform(rows, cols, -1.0); 137 | } 138 | 139 | 140 | 141 | private int GetByIndex(int row, int col) 142 | { 143 | int ix = Cols * row + col; 144 | return ix; 145 | } 146 | 147 | private double GetW(int row, int col) 148 | { 149 | return W[GetByIndex(row, col)]; 150 | } 151 | 152 | private void SetW(int row, int col, double val) 153 | { 154 | W[GetByIndex(row, col)] = val; 155 | } 156 | } 157 | } 158 | -------------------------------------------------------------------------------- /src/SharpML.Recurrent/Models/Runnable.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace SharpML.Recurrent.Models 4 | { 5 | public class Runnable : IRunnable 6 | { 7 | public Action Run {get;set;} 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /src/SharpML.Recurrent/Networks/FeedForwardLayer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using SharpML.Recurrent.Activations; 4 | using SharpML.Recurrent.Models; 5 | 6 | namespace SharpML.Recurrent.Networks 7 | { 8 | [Serializable] 9 | public class FeedForwardLayer : ILayer 10 | { 11 | 12 | private static long _serialVersionUid = 1L; 13 | readonly Matrix _w; 14 | readonly Matrix _b; 15 | readonly INonlinearity _f; 16 | 17 | public FeedForwardLayer(int inputDimension, int outputDimension, INonlinearity f, double initParamsStdDev, Random rng) 18 | { 19 | _w = Matrix.Random(outputDimension, inputDimension, initParamsStdDev, rng); 20 | _b = new Matrix(outputDimension); 21 | this._f = f; 22 | } 23 | 24 | public Matrix Activate(Matrix input, Graph g) 25 | { 26 | Matrix sum = g.Add(g.Mul(_w, input), _b); 27 | Matrix returnObj = g.Nonlin(_f, sum); 28 | return returnObj; 29 | } 30 | 31 | public void ResetState() 32 | { 33 | 34 | } 35 | 36 | public List GetParameters() 37 | { 38 | List result = new List(); 39 | result.Add(_w); 40 | result.Add(_b); 41 | return result; 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/SharpML.Recurrent/Networks/GruLayer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using SharpML.Recurrent.Activations; 4 | using SharpML.Recurrent.Models; 5 | 6 | namespace SharpML.Recurrent.Networks 7 | { 8 | [Serializable] 9 | public class GruLayer : ILayer { 10 | 11 | private static long _serialVersionUid = 1L; 12 | int _inputDimension; 13 | readonly int _outputDimension; 14 | 15 | readonly Matrix _hmix; 16 | readonly Matrix _hHmix; 17 | readonly Matrix _bmix; 18 | readonly Matrix _hnew; 19 | readonly Matrix _hHnew; 20 | readonly Matrix _bnew; 21 | readonly Matrix _hreset; 22 | readonly Matrix _hHreset; 23 | readonly Matrix _breset; 24 | 25 | Matrix _context; 26 | 27 | readonly INonlinearity _fMix = new SigmoidUnit(); 28 | readonly INonlinearity _fReset = new SigmoidUnit(); 29 | readonly INonlinearity _fNew = new TanhUnit(); 30 | 31 | public GruLayer(int inputDimension, int outputDimension, double initParamsStdDev, Random rng) { 32 | this._inputDimension = inputDimension; 33 | this._outputDimension = outputDimension; 34 | _hmix = Matrix.Random(outputDimension, inputDimension, initParamsStdDev, rng); 35 | _hHmix = Matrix.Random(outputDimension, outputDimension, initParamsStdDev, rng); 36 | _bmix = new Matrix(outputDimension); 37 | _hnew = Matrix.Random(outputDimension, inputDimension, initParamsStdDev, rng); 38 | _hHnew = Matrix.Random(outputDimension, outputDimension, initParamsStdDev, rng); 39 | _bnew = new Matrix(outputDimension); 40 | _hreset = Matrix.Random(outputDimension, inputDimension, initParamsStdDev, rng); 41 | _hHreset = Matrix.Random(outputDimension, outputDimension, initParamsStdDev, rng); 42 | _breset= new Matrix(outputDimension); 43 | } 44 | 45 | public Matrix Activate(Matrix input, Graph g) { 46 | 47 | Matrix sum0 = g.Mul(_hmix, input); 48 | Matrix sum1 = g.Mul(_hHmix, _context); 49 | Matrix actMix = g.Nonlin(_fMix, g.Add(g.Add(sum0, sum1), _bmix)); 50 | 51 | Matrix sum2 = g.Mul(_hreset, input); 52 | Matrix sum3 = g.Mul(_hHreset, _context); 53 | Matrix actReset = g.Nonlin(_fReset, g.Add(g.Add(sum2, sum3), _breset)); 54 | 55 | Matrix sum4 = g.Mul(_hnew, input); 56 | Matrix gatedContext = g.Elmul(actReset, _context); 57 | Matrix sum5 = g.Mul(_hHnew, gatedContext); 58 | Matrix actNewPlusGatedContext = g.Nonlin(_fNew, g.Add(g.Add(sum4, sum5), _bnew)); 59 | 60 | Matrix memvals = g.Elmul(actMix, _context); 61 | Matrix newvals = g.Elmul(g.OneMinus(actMix), actNewPlusGatedContext); 62 | Matrix output = g.Add(memvals, newvals); 63 | 64 | //rollover activations for next iteration 65 | _context = output; 66 | 67 | return output; 68 | } 69 | 70 | public void ResetState() { 71 | _context = new Matrix(_outputDimension); 72 | } 73 | 74 | public List GetParameters() { 75 | List result = new List(); 76 | result.Add(_hmix); 77 | result.Add(_hHmix); 78 | result.Add(_bmix); 79 | result.Add(_hnew); 80 | result.Add(_hHnew); 81 | result.Add(_bnew); 82 | result.Add(_hreset); 83 | result.Add(_hHreset); 84 | result.Add(_breset); 85 | return result; 86 | } 87 | 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /src/SharpML.Recurrent/Networks/ILayer.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using SharpML.Recurrent.Models; 3 | 4 | namespace SharpML.Recurrent.Networks 5 | { 6 | public interface ILayer 7 | { 8 | Matrix Activate(Matrix input, Graph g); 9 | void ResetState(); 10 | List GetParameters(); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/SharpML.Recurrent/Networks/INetwork.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using SharpML.Recurrent.Models; 3 | 4 | namespace SharpML.Recurrent.Networks 5 | { 6 | public interface INetwork 7 | { 8 | Matrix Activate(Matrix input, Graph g); 9 | void ResetState(); 10 | List GetParameters(); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/SharpML.Recurrent/Networks/LinearLayer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using SharpML.Recurrent.Models; 4 | 5 | namespace SharpML.Recurrent.Networks 6 | { 7 | [Serializable] 8 | public class LinearLayer : ILayer 9 | { 10 | 11 | private static long _serialVersionUid = 1L; 12 | readonly Matrix _w; 13 | //no biases 14 | 15 | public LinearLayer(int inputDimension, int outputDimension, double initParamsStdDev, Random rng) 16 | { 17 | _w = Matrix.Random(outputDimension, inputDimension, initParamsStdDev, rng); 18 | } 19 | 20 | public Matrix Activate(Matrix input, Graph g) 21 | { 22 | Matrix returnObj = g.Mul(_w, input); 23 | return returnObj; 24 | } 25 | 26 | public void ResetState() 27 | { 28 | 29 | } 30 | 31 | public List GetParameters() 32 | { 33 | List result = new List(); 34 | result.Add(_w); 35 | return result; 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/SharpML.Recurrent/Networks/LstmLayer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using SharpML.Recurrent.Activations; 4 | using SharpML.Recurrent.Models; 5 | 6 | namespace SharpML.Recurrent.Networks 7 | { 8 | [Serializable] 9 | public class LstmLayer : ILayer 10 | { 11 | 12 | private static long _serialVersionUid = 1L; 13 | int _inputDimension; 14 | readonly int _outputDimension; 15 | 16 | readonly Matrix _wix; 17 | readonly Matrix _wih; 18 | readonly Matrix _inputBias; 19 | readonly Matrix _wfx; 20 | readonly Matrix _wfh; 21 | readonly Matrix _forgetBias; 22 | readonly Matrix _wox; 23 | readonly Matrix _woh; 24 | readonly Matrix _outputBias; 25 | readonly Matrix _wcx; 26 | readonly Matrix _wch; 27 | readonly Matrix _cellWriteBias; 28 | 29 | Matrix _hiddenContext; 30 | Matrix _cellContext; 31 | 32 | readonly INonlinearity _inputGateActivation = new SigmoidUnit(); 33 | readonly INonlinearity _forgetGateActivation = new SigmoidUnit(); 34 | readonly INonlinearity _outputGateActivation = new SigmoidUnit(); 35 | readonly INonlinearity _cellInputActivation = new TanhUnit(); 36 | readonly INonlinearity _cellOutputActivation = new TanhUnit(); 37 | 38 | public LstmLayer(int inputDimension, int outputDimension, double initParamsStdDev, Random rng) 39 | { 40 | this._inputDimension = inputDimension; 41 | this._outputDimension = outputDimension; 42 | _wix = Matrix.Random(outputDimension, inputDimension, initParamsStdDev, rng); 43 | _wih = Matrix.Random(outputDimension, outputDimension, initParamsStdDev, rng); 44 | _inputBias = new Matrix(outputDimension); 45 | _wfx = Matrix.Random(outputDimension, inputDimension, initParamsStdDev, rng); 46 | _wfh = Matrix.Random(outputDimension, outputDimension, initParamsStdDev, rng); 47 | //set forget bias to 1.0, as described here: http://jmlr.org/proceedings/papers/v37/jozefowicz15.pdf 48 | _forgetBias = Matrix.Ones(outputDimension, 1); 49 | _wox = Matrix.Random(outputDimension, inputDimension, initParamsStdDev, rng); 50 | _woh = Matrix.Random(outputDimension, outputDimension, initParamsStdDev, rng); 51 | _outputBias = new Matrix(outputDimension); 52 | _wcx = Matrix.Random(outputDimension, inputDimension, initParamsStdDev, rng); 53 | _wch = Matrix.Random(outputDimension, outputDimension, initParamsStdDev, rng); 54 | _cellWriteBias = new Matrix(outputDimension); 55 | } 56 | 57 | public Matrix Activate(Matrix input, Graph g) 58 | { 59 | 60 | //input gate 61 | Matrix sum0 = g.Mul(_wix, input); 62 | Matrix sum1 = g.Mul(_wih, _hiddenContext); 63 | Matrix inputGate = g.Nonlin(_inputGateActivation, g.Add(g.Add(sum0, sum1), _inputBias)); 64 | 65 | //forget gate 66 | Matrix sum2 = g.Mul(_wfx, input); 67 | Matrix sum3 = g.Mul(_wfh, _hiddenContext); 68 | Matrix forgetGate = g.Nonlin(_forgetGateActivation, g.Add(g.Add(sum2, sum3), _forgetBias)); 69 | 70 | //output gate 71 | Matrix sum4 = g.Mul(_wox, input); 72 | Matrix sum5 = g.Mul(_woh, _hiddenContext); 73 | Matrix outputGate = g.Nonlin(_outputGateActivation, g.Add(g.Add(sum4, sum5), _outputBias)); 74 | 75 | //write operation on cells 76 | Matrix sum6 = g.Mul(_wcx, input); 77 | Matrix sum7 = g.Mul(_wch, _hiddenContext); 78 | Matrix cellInput = g.Nonlin(_cellInputActivation, g.Add(g.Add(sum6, sum7), _cellWriteBias)); 79 | 80 | //compute new cell activation 81 | Matrix retainCell = g.Elmul(forgetGate, _cellContext); 82 | Matrix writeCell = g.Elmul(inputGate, cellInput); 83 | Matrix cellAct = g.Add(retainCell, writeCell); 84 | 85 | //compute hidden state as gated, saturated cell activations 86 | Matrix output = g.Elmul(outputGate, g.Nonlin(_cellOutputActivation, cellAct)); 87 | 88 | //rollover activations for next iteration 89 | _hiddenContext = output; 90 | _cellContext = cellAct; 91 | 92 | return output; 93 | } 94 | 95 | public void ResetState() 96 | { 97 | _hiddenContext = new Matrix(_outputDimension); 98 | _cellContext = new Matrix(_outputDimension); 99 | } 100 | 101 | public List GetParameters() 102 | { 103 | List result = new List(); 104 | result.Add(_wix); 105 | result.Add(_wih); 106 | result.Add(_inputBias); 107 | result.Add(_wfx); 108 | result.Add(_wfh); 109 | result.Add(_forgetBias); 110 | result.Add(_wox); 111 | result.Add(_woh); 112 | result.Add(_outputBias); 113 | result.Add(_wcx); 114 | result.Add(_wch); 115 | result.Add(_cellWriteBias); 116 | return result; 117 | } 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /src/SharpML.Recurrent/Networks/NeuralNetwork.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using SharpML.Recurrent.Models; 4 | 5 | namespace SharpML.Recurrent.Networks 6 | { 7 | [Serializable] 8 | public class NeuralNetwork : INetwork 9 | { 10 | 11 | private static long _serialVersionUid = 1L; 12 | readonly List _layers; 13 | 14 | public NeuralNetwork(List layers) 15 | { 16 | this._layers = layers; 17 | } 18 | 19 | public Matrix Activate(Matrix input, Graph g) 20 | { 21 | Matrix prev = input; 22 | foreach (ILayer layer in _layers) 23 | { 24 | prev = layer.Activate(prev, g); 25 | } 26 | return prev; 27 | } 28 | 29 | public void ResetState() 30 | { 31 | foreach (ILayer layer in _layers) 32 | { 33 | layer.ResetState(); 34 | } 35 | } 36 | 37 | public List GetParameters() 38 | { 39 | List result = new List(); 40 | foreach (ILayer layer in _layers) 41 | { 42 | result.AddRange(layer.GetParameters()); 43 | } 44 | return result; 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/SharpML.Recurrent/Networks/RnnLayer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using SharpML.Recurrent.Activations; 4 | using SharpML.Recurrent.Models; 5 | 6 | namespace SharpML.Recurrent.Networks 7 | { 8 | [Serializable] 9 | public class RnnLayer : ILayer 10 | { 11 | 12 | private static long _serialVersionUid = 1L; 13 | private int _inputDimension; 14 | private readonly int _outputDimension; 15 | 16 | private readonly Matrix _w; 17 | private readonly Matrix _b; 18 | 19 | private Matrix _context; 20 | 21 | private readonly INonlinearity _f; 22 | 23 | public RnnLayer(int inputDimension, int outputDimension, INonlinearity hiddenUnit, double initParamsStdDev, 24 | Random rng) 25 | { 26 | this._inputDimension = inputDimension; 27 | this._outputDimension = outputDimension; 28 | this._f = hiddenUnit; 29 | _w = Matrix.Random(outputDimension, inputDimension + outputDimension, initParamsStdDev, rng); 30 | _b = new Matrix(outputDimension); 31 | } 32 | 33 | public Matrix Activate(Matrix input, Graph g) 34 | { 35 | Matrix concat = g.ConcatVectors(input, _context); 36 | Matrix sum = g.Mul(_w, concat); sum = g.Add(sum, _b); 37 | Matrix output = g.Nonlin(_f, sum); 38 | 39 | //rollover activations for next iteration 40 | _context = output; 41 | 42 | return output; 43 | } 44 | 45 | 46 | public void ResetState() 47 | { 48 | _context = new Matrix(_outputDimension); 49 | } 50 | 51 | 52 | public List GetParameters() 53 | { 54 | List result = new List(); 55 | result.Add(_w); 56 | result.Add(_b); 57 | return result; 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/SharpML.Recurrent/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.CompilerServices; 3 | using System.Runtime.InteropServices; 4 | 5 | // General Information about an assembly is controlled through the following 6 | // set of attributes. Change these attribute values to modify the information 7 | // associated with an assembly. 8 | [assembly: AssemblyTitle("SharpML.Recurrent")] 9 | [assembly: AssemblyDescription("")] 10 | [assembly: AssemblyConfiguration("")] 11 | [assembly: AssemblyCompany("Microsoft")] 12 | [assembly: AssemblyProduct("SharpML.Recurrent")] 13 | [assembly: AssemblyCopyright("Copyright © Microsoft 2015")] 14 | [assembly: AssemblyTrademark("")] 15 | [assembly: AssemblyCulture("")] 16 | 17 | // Setting ComVisible to false makes the types in this assembly not visible 18 | // to COM components. If you need to access a type in this assembly from 19 | // COM, set the ComVisible attribute to true on that type. 20 | [assembly: ComVisible(false)] 21 | 22 | // The following GUID is for the ID of the typelib if this project is exposed to COM 23 | [assembly: Guid("51f90600-238d-430d-96da-65cd46918173")] 24 | 25 | // Version information for an assembly consists of the following four values: 26 | // 27 | // Major Version 28 | // Minor Version 29 | // Build Number 30 | // Revision 31 | // 32 | // You can specify all the values or you can default the Build and Revision Numbers 33 | // by using the '*' as shown below: 34 | // [assembly: AssemblyVersion("1.0.*")] 35 | [assembly: AssemblyVersion("1.0.0.0")] 36 | [assembly: AssemblyFileVersion("1.0.0.0")] 37 | -------------------------------------------------------------------------------- /src/SharpML.Recurrent/SharpML.Recurrent.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | Debug 6 | AnyCPU 7 | {07F1480B-7CB3-4C18-BA5F-14A40456F4FF} 8 | Library 9 | Properties 10 | SharpML.Recurrent 11 | SharpML.Recurrent 12 | v4.0 13 | 512 14 | 15 | 16 | true 17 | full 18 | false 19 | bin\Debug\ 20 | DEBUG;TRACE 21 | prompt 22 | 4 23 | 24 | 25 | pdbonly 26 | true 27 | bin\Release\ 28 | TRACE 29 | prompt 30 | 4 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | {25231c25-0c69-4837-8f5c-c1a221943932} 78 | SharpML.Helpers 79 | 80 | 81 | 82 | 89 | -------------------------------------------------------------------------------- /src/SharpML.Recurrent/Trainer/Trainer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Linq; 5 | using System.Text; 6 | using SharpML.Recurrent; 7 | using SharpML.Recurrent.DataStructs; 8 | using SharpML.Recurrent.Loss; 9 | using SharpML.Recurrent.Models; 10 | using SharpML.Recurrent.Networks; 11 | using SharpML.Recurrent.Util; 12 | 13 | namespace SharpML.Recurrent.Trainer 14 | { 15 | public class Trainer 16 | { 17 | 18 | public static double DecayRate = 0.999; 19 | public static double SmoothEpsilon = 1e-8; 20 | public static double GradientClipValue = 5; 21 | public static double Regularization = 0.000001; // L2 regularization strength 22 | 23 | public static double train(int trainingEpochs, double learningRate, INetwork network, DataSet data, int reportEveryNthEpoch, Random rng) where T : INetwork 24 | { 25 | return train(trainingEpochs, learningRate, network, data, reportEveryNthEpoch, false, false, null, rng); 26 | } 27 | 28 | public static double train(int trainingEpochs, double learningRate, INetwork network, DataSet data, int reportEveryNthEpoch, bool initFromSaved, bool overwriteSaved, String savePath, Random rng) where T : INetwork 29 | { 30 | Console.WriteLine("--------------------------------------------------------------"); 31 | if (initFromSaved) 32 | { 33 | Console.WriteLine("initializing network from saved state..."); 34 | try 35 | { 36 | network = (INetwork)Binary.ReadFromBinary(savePath); 37 | data.DisplayReport(network, rng); 38 | } 39 | catch (Exception e) 40 | { 41 | Console.WriteLine("Oops. Unable to load from a saved state."); 42 | Console.WriteLine("WARNING: " + e.Message); 43 | Console.WriteLine("Continuing from freshly initialized network instead."); 44 | } 45 | } 46 | double result = 1.0; 47 | for (int epoch = 0; epoch < trainingEpochs; epoch++) 48 | { 49 | 50 | String show = "epoch[" + (epoch + 1) + "/" + trainingEpochs + "]"; 51 | 52 | double reportedLossTrain = Pass(learningRate, network, data.Training, true, data.LossTraining, data.LossReporting); 53 | result = reportedLossTrain; 54 | if (Double.IsNaN(reportedLossTrain) || Double.IsInfinity(reportedLossTrain)) 55 | { 56 | throw new Exception("WARNING: invalid value for training loss. Try lowering learning rate."); 57 | } 58 | double reportedLossValidation = 0; 59 | double reportedLossTesting = 0; 60 | if (data.Validation != null) 61 | { 62 | reportedLossValidation = Pass(learningRate, network, data.Validation, false, data.LossTraining, data.LossReporting); 63 | result = reportedLossValidation; 64 | } 65 | if (data.Testing != null) 66 | { 67 | reportedLossTesting = Pass(learningRate, network, data.Testing, false, data.LossTraining, data.LossReporting); 68 | result = reportedLossTesting; 69 | } 70 | show += "\ttrain loss = " + String.Format("{0:N5}", reportedLossTrain); 71 | if (data.Validation != null) 72 | { 73 | show += "\tvalid loss = " + String.Format("{0:N5}", reportedLossValidation); 74 | } 75 | if (data.Testing != null) 76 | { 77 | show += "\ttest loss = " + String.Format("{0:N5}", reportedLossTesting); 78 | } 79 | Console.WriteLine(show); 80 | 81 | if (epoch % reportEveryNthEpoch == reportEveryNthEpoch - 1) 82 | { 83 | data.DisplayReport(network, rng); 84 | } 85 | 86 | if (overwriteSaved) 87 | { 88 | Binary.WriteToBinary(network, savePath); 89 | } 90 | 91 | if (reportedLossTrain == 0 && reportedLossValidation == 0) 92 | { 93 | Console.WriteLine("--------------------------------------------------------------"); 94 | Console.WriteLine("\nDONE."); 95 | break; 96 | } 97 | } 98 | return result; 99 | } 100 | 101 | public static double Pass(double learningRate, INetwork network, List sequences, 102 | bool applyTraining, ILoss lossTraining, ILoss lossReporting) 103 | { 104 | double numerLoss = 0; 105 | double denomLoss = 0; 106 | 107 | foreach (DataSequence seq in sequences) 108 | { 109 | network.ResetState(); 110 | Graph g = new Graph(applyTraining); 111 | foreach (DataStep step in seq.Steps) 112 | { 113 | Matrix output = network.Activate(step.Input, g); 114 | if (step.TargetOutput != null) 115 | { 116 | double loss = lossReporting.Measure(output, step.TargetOutput); 117 | if (Double.IsNaN(loss) || Double.IsInfinity(loss)) 118 | { 119 | return loss; 120 | } 121 | numerLoss += loss; 122 | denomLoss++; 123 | if (applyTraining) 124 | { 125 | lossTraining.Backward(output, step.TargetOutput); 126 | } 127 | } 128 | } 129 | List thisSequence = new List(); 130 | thisSequence.Add(seq); 131 | if (applyTraining) 132 | { 133 | g.Backward(); //backprop dw values 134 | UpdateModelParams(network, learningRate); //update params 135 | } 136 | } 137 | return numerLoss / denomLoss; 138 | } 139 | 140 | //public static Tuple PassSingleWithResult(double learningRate, INetwork network, DataStep step, bool applyTraining, ILoss lossTraining, ILoss lossReporting) 141 | //{ 142 | // var returnObj = new Tuple(0, null); 143 | // double numerLoss = 0; 144 | // double denomLoss = 0; 145 | 146 | 147 | // network.ResetState(); 148 | // Graph g = new Graph(applyTraining); 149 | // foreach (DataStep step in sequence.Steps) 150 | // { 151 | // Matrix output = network.Activate(step.Input, g); 152 | // if (step.TargetOutput != null) 153 | // { 154 | // double loss = lossReporting.Measure(output, step.TargetOutput); 155 | // if (Double.IsNaN(loss) || Double.IsInfinity(loss)) 156 | // { 157 | // //return loss; 158 | // return new Tuple(loss, output); 159 | // } 160 | // numerLoss += loss; 161 | // denomLoss++; 162 | // if (applyTraining) 163 | // { 164 | // lossTraining.Backward(output, step.TargetOutput); 165 | // } 166 | // } 167 | // } 168 | // List thisSequence = new List(); 169 | // thisSequence.Add(sequence); 170 | // if (applyTraining) 171 | // { 172 | // g.Backward(); //backprop dw values 173 | // UpdateModelParams(network, learningRate); //update params 174 | // } 175 | 176 | // return numerLoss / denomLoss; 177 | // return new Tuple(loss, output); 178 | //} 179 | 180 | public static void UpdateModelParams(INetwork network, double stepSize) 181 | { 182 | foreach (Matrix m in network.GetParameters()) 183 | { 184 | for (int i = 0; i < m.W.Length; i++) 185 | { 186 | 187 | // rmsprop adaptive learning rate 188 | double mdwi = m.Dw[i]; 189 | m.StepCache[i] = m.StepCache[i] * DecayRate + (1 - DecayRate) * mdwi * mdwi; 190 | 191 | // gradient clip 192 | if (mdwi > GradientClipValue) 193 | { 194 | mdwi = GradientClipValue; 195 | } 196 | if (mdwi < -GradientClipValue) 197 | { 198 | mdwi = -GradientClipValue; 199 | } 200 | 201 | // update (and regularize) 202 | m.W[i] += -stepSize * mdwi / Math.Sqrt(m.StepCache[i] + SmoothEpsilon) - Regularization * m.W[i]; 203 | m.Dw[i] = 0; 204 | } 205 | } 206 | } 207 | } 208 | } 209 | -------------------------------------------------------------------------------- /src/SharpML.Recurrent/Util/Binary.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Linq; 5 | using System.Runtime.Serialization.Formatters.Binary; 6 | using System.Text; 7 | 8 | namespace SharpML.Recurrent.Util 9 | { 10 | public static class Binary 11 | { 12 | public static T ReadFromBinary(string filePath) 13 | { 14 | if (File.Exists(filePath)) 15 | { 16 | using (var fs = new FileStream(filePath, FileMode.Open)) 17 | { 18 | BinaryFormatter formatter = new BinaryFormatter(); 19 | formatter.AssemblyFormat = System.Runtime.Serialization.Formatters.FormatterAssemblyStyle.Simple; 20 | if (fs.Length == 0) 21 | return default(T); 22 | 23 | return ((T)formatter.Deserialize(fs)); 24 | } 25 | } 26 | 27 | throw new FileNotFoundException(filePath); 28 | } 29 | 30 | public static void WriteToBinary(object dataToWrite, string filePath) 31 | { 32 | using (Stream stream = File.Open(filePath, FileMode.Create)) 33 | { 34 | BinaryFormatter bformatter = new BinaryFormatter(); 35 | bformatter.AssemblyFormat = System.Runtime.Serialization.Formatters.FormatterAssemblyStyle.Simple; 36 | 37 | var filesToWrite = ((T)dataToWrite); 38 | 39 | bformatter.Serialize(stream, filesToWrite); 40 | } 41 | } 42 | 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/SharpML.Recurrent/Util/NetworkBuilder.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using SharpML.Recurrent.Activations; 6 | using SharpML.Recurrent.Networks; 7 | 8 | namespace SharpML.Recurrent.Util 9 | { 10 | public static class NetworkBuilder 11 | { 12 | 13 | public static NeuralNetwork MakeLstm(int inputDimension, int hiddenDimension, int hiddenLayers, int outputDimension, INonlinearity decoderUnit, double initParamsStdDev, Random rng) 14 | { 15 | List layers = new List(); 16 | for (int h = 0; h < hiddenLayers; h++) 17 | { 18 | if (h == 0) 19 | { 20 | layers.Add(new LstmLayer(inputDimension, hiddenDimension, initParamsStdDev, rng)); 21 | } 22 | else 23 | { 24 | layers.Add(new LstmLayer(hiddenDimension, hiddenDimension, initParamsStdDev, rng)); 25 | } 26 | } 27 | layers.Add(new FeedForwardLayer(hiddenDimension, outputDimension, decoderUnit, initParamsStdDev, rng)); 28 | return new NeuralNetwork(layers); 29 | } 30 | 31 | public static NeuralNetwork MakeLstmWithInputBottleneck(int inputDimension, int bottleneckDimension, int hiddenDimension, int hiddenLayers, int outputDimension, INonlinearity decoderUnit, double initParamsStdDev, Random rng) 32 | { 33 | List layers = new List(); 34 | layers.Add(new LinearLayer(inputDimension, bottleneckDimension, initParamsStdDev, rng)); 35 | for (int h = 0; h < hiddenLayers; h++) 36 | { 37 | if (h == 0) 38 | { 39 | layers.Add(new LstmLayer(bottleneckDimension, hiddenDimension, initParamsStdDev, rng)); 40 | } 41 | else 42 | { 43 | layers.Add(new LstmLayer(hiddenDimension, hiddenDimension, initParamsStdDev, rng)); 44 | } 45 | } 46 | layers.Add(new FeedForwardLayer(hiddenDimension, outputDimension, decoderUnit, initParamsStdDev, rng)); 47 | return new NeuralNetwork(layers); 48 | } 49 | 50 | public static NeuralNetwork MakeFeedForward(int inputDimension, int hiddenDimension, int hiddenLayers, int outputDimension, INonlinearity hiddenUnit, INonlinearity decoderUnit, double initParamsStdDev, Random rng) 51 | { 52 | List layers = new List(); 53 | for (int h = 0; h < hiddenLayers; h++) 54 | { 55 | if (h == 0) 56 | { 57 | layers.Add(new FeedForwardLayer(inputDimension, hiddenDimension, hiddenUnit, initParamsStdDev, rng)); 58 | } 59 | else 60 | { 61 | layers.Add(new FeedForwardLayer(hiddenDimension, hiddenDimension, hiddenUnit, initParamsStdDev, rng)); 62 | } 63 | } 64 | layers.Add(new FeedForwardLayer(hiddenDimension, outputDimension, decoderUnit, initParamsStdDev, rng)); 65 | return new NeuralNetwork(layers); 66 | } 67 | 68 | public static NeuralNetwork MakeGru(int inputDimension, int hiddenDimension, int hiddenLayers, int outputDimension, INonlinearity decoderUnit, double initParamsStdDev, Random rng) 69 | { 70 | List layers = new List(); 71 | for (int h = 0; h < hiddenLayers; h++) 72 | { 73 | if (h == 0) 74 | { 75 | layers.Add(new GruLayer(inputDimension, hiddenDimension, initParamsStdDev, rng)); 76 | } 77 | else 78 | { 79 | layers.Add(new GruLayer(hiddenDimension, hiddenDimension, initParamsStdDev, rng)); 80 | } 81 | } 82 | layers.Add(new FeedForwardLayer(hiddenDimension, outputDimension, decoderUnit, initParamsStdDev, rng)); 83 | return new NeuralNetwork(layers); 84 | } 85 | 86 | public static NeuralNetwork MakeRnn(int inputDimension, int hiddenDimension, int hiddenLayers, int outputDimension, INonlinearity hiddenUnit, INonlinearity decoderUnit, double initParamsStdDev, Random rng) 87 | { 88 | List layers = new List(); 89 | for (int h = 0; h < hiddenLayers; h++) 90 | { 91 | if (h == 0) 92 | { 93 | layers.Add(new RnnLayer(inputDimension, hiddenDimension, hiddenUnit, initParamsStdDev, rng)); 94 | } 95 | else 96 | { 97 | layers.Add(new RnnLayer(hiddenDimension, hiddenDimension, hiddenUnit, initParamsStdDev, rng)); 98 | } 99 | } 100 | layers.Add(new FeedForwardLayer(hiddenDimension, outputDimension, decoderUnit, initParamsStdDev, rng)); 101 | return new NeuralNetwork(layers); 102 | } 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /src/SharpML.Recurrent/Util/Util.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using SharpML.Recurrent.Models; 6 | 7 | namespace SharpML.Recurrent.Util 8 | { 9 | public class Util 10 | { 11 | 12 | public static int PickIndexFromRandomVector(Matrix probs, Random r) 13 | { 14 | double mass = 1.0; 15 | for (int i = 0; i < probs.W.Length; i++) 16 | { 17 | double prob = probs.W[i] / mass; 18 | if (r.NextDouble() < prob) 19 | { 20 | return i; 21 | } 22 | mass -= probs.W[i]; 23 | } 24 | throw new Exception("no target index selected"); 25 | } 26 | 27 | public static double Median(List vals) 28 | { 29 | vals = vals.OrderBy(a => a).ToList(); 30 | int mid = vals.Count / 2; 31 | if (vals.Count % 2 == 1) 32 | { 33 | return vals[mid]; 34 | } 35 | else 36 | { 37 | return (vals[mid - 1] + vals[mid]) / 2; 38 | } 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/SharpML.Recurrent/bin/Debug/SharpML.Recurrent.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrewfry/SharpML-Recurrent/9afeb8189a3c43b321cba39df12fcdfba91d4958/src/SharpML.Recurrent/bin/Debug/SharpML.Recurrent.dll -------------------------------------------------------------------------------- /src/SharpML.Recurrent/bin/Debug/SharpML.Recurrent.pdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrewfry/SharpML-Recurrent/9afeb8189a3c43b321cba39df12fcdfba91d4958/src/SharpML.Recurrent/bin/Debug/SharpML.Recurrent.pdb -------------------------------------------------------------------------------- /src/SharpML.Recurrent/obj/Debug/DesignTimeResolveAssemblyReferencesInput.cache: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrewfry/SharpML-Recurrent/9afeb8189a3c43b321cba39df12fcdfba91d4958/src/SharpML.Recurrent/obj/Debug/DesignTimeResolveAssemblyReferencesInput.cache -------------------------------------------------------------------------------- /src/SharpML.Recurrent/obj/Debug/SharpML.Recurrent.csproj.FileListAbsolute.txt: -------------------------------------------------------------------------------- 1 | C:\Users\afry\Documents\Visual Studio 2013\Projects\SharpML\SharpML.Recurrent\bin\Debug\SharpML.Recurrent.dll 2 | C:\Users\afry\Documents\Visual Studio 2013\Projects\SharpML\SharpML.Recurrent\bin\Debug\SharpML.Recurrent.pdb 3 | C:\Users\afry\Documents\Visual Studio 2013\Projects\SharpML\SharpML.Recurrent\obj\Debug\SharpML.Recurrent.csprojResolveAssemblyReference.cache 4 | C:\Users\afry\Documents\Visual Studio 2013\Projects\SharpML\SharpML.Recurrent\obj\Debug\SharpML.Recurrent.dll 5 | C:\Users\afry\Documents\Visual Studio 2013\Projects\SharpML\SharpML.Recurrent\obj\Debug\SharpML.Recurrent.pdb 6 | C:\Users\afry\Documents\Visual Studio 2013\Projects\SharpML\SharpML.Recurrent\bin\Debug\SharpML.Helpers.dll 7 | C:\Users\afry\Documents\Visual Studio 2013\Projects\SharpML\SharpML.Recurrent\bin\Debug\SharpML.Helpers.pdb 8 | C:\Users\afry\Documents\GitHub\SharpML.Recurrent\SharpML.Recurrent\bin\Debug\SharpML.Recurrent.dll 9 | C:\Users\afry\Documents\GitHub\SharpML.Recurrent\SharpML.Recurrent\bin\Debug\SharpML.Recurrent.pdb 10 | C:\Users\afry\Documents\GitHub\SharpML.Recurrent\SharpML.Recurrent\obj\Debug\SharpML.Recurrent.csprojResolveAssemblyReference.cache 11 | C:\Users\afry\Documents\GitHub\SharpML.Recurrent\SharpML.Recurrent\obj\Debug\SharpML.Recurrent.dll 12 | C:\Users\afry\Documents\GitHub\SharpML.Recurrent\SharpML.Recurrent\obj\Debug\SharpML.Recurrent.pdb 13 | C:\Users\afry\Documents\GitHub\SharpML.Recurrent\SharpML-Recurrent\src\SharpML.Recurrent\bin\Debug\SharpML.Recurrent.dll 14 | C:\Users\afry\Documents\GitHub\SharpML.Recurrent\SharpML-Recurrent\src\SharpML.Recurrent\bin\Debug\SharpML.Recurrent.pdb 15 | C:\Users\afry\Documents\GitHub\SharpML.Recurrent\SharpML-Recurrent\src\SharpML.Recurrent\obj\Debug\SharpML.Recurrent.csprojResolveAssemblyReference.cache 16 | C:\Users\afry\Documents\GitHub\SharpML.Recurrent\SharpML-Recurrent\src\SharpML.Recurrent\obj\Debug\SharpML.Recurrent.dll 17 | C:\Users\afry\Documents\GitHub\SharpML.Recurrent\SharpML-Recurrent\src\SharpML.Recurrent\obj\Debug\SharpML.Recurrent.pdb 18 | -------------------------------------------------------------------------------- /src/SharpML.Recurrent/obj/Debug/SharpML.Recurrent.csprojResolveAssemblyReference.cache: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrewfry/SharpML-Recurrent/9afeb8189a3c43b321cba39df12fcdfba91d4958/src/SharpML.Recurrent/obj/Debug/SharpML.Recurrent.csprojResolveAssemblyReference.cache -------------------------------------------------------------------------------- /src/SharpML.Recurrent/obj/Debug/SharpML.Recurrent.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrewfry/SharpML-Recurrent/9afeb8189a3c43b321cba39df12fcdfba91d4958/src/SharpML.Recurrent/obj/Debug/SharpML.Recurrent.dll -------------------------------------------------------------------------------- /src/SharpML.Recurrent/obj/Debug/SharpML.Recurrent.pdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrewfry/SharpML-Recurrent/9afeb8189a3c43b321cba39df12fcdfba91d4958/src/SharpML.Recurrent/obj/Debug/SharpML.Recurrent.pdb -------------------------------------------------------------------------------- /src/SharpML.Recurrent/obj/Release/DesignTimeResolveAssemblyReferencesInput.cache: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrewfry/SharpML-Recurrent/9afeb8189a3c43b321cba39df12fcdfba91d4958/src/SharpML.Recurrent/obj/Release/DesignTimeResolveAssemblyReferencesInput.cache -------------------------------------------------------------------------------- /src/SharpML.Recurrent/obj/Release/SharpML.Recurrent.csproj.FileListAbsolute.txt: -------------------------------------------------------------------------------- 1 | C:\Users\afry\Documents\Visual Studio 2013\Projects\SharpML\SharpML.Recurrent\bin\Release\SharpML.Recurrent.dll 2 | C:\Users\afry\Documents\Visual Studio 2013\Projects\SharpML\SharpML.Recurrent\bin\Release\SharpML.Recurrent.pdb 3 | C:\Users\afry\Documents\Visual Studio 2013\Projects\SharpML\SharpML.Recurrent\obj\Release\SharpML.Recurrent.csprojResolveAssemblyReference.cache 4 | C:\Users\afry\Documents\Visual Studio 2013\Projects\SharpML\SharpML.Recurrent\obj\Release\SharpML.Recurrent.dll 5 | C:\Users\afry\Documents\Visual Studio 2013\Projects\SharpML\SharpML.Recurrent\obj\Release\SharpML.Recurrent.pdb 6 | -------------------------------------------------------------------------------- /src/SharpML.Recurrent/obj/Release/SharpML.Recurrent.csprojResolveAssemblyReference.cache: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrewfry/SharpML-Recurrent/9afeb8189a3c43b321cba39df12fcdfba91d4958/src/SharpML.Recurrent/obj/Release/SharpML.Recurrent.csprojResolveAssemblyReference.cache -------------------------------------------------------------------------------- /src/SharpML.Recurrent/obj/Release/SharpML.Recurrent.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrewfry/SharpML-Recurrent/9afeb8189a3c43b321cba39df12fcdfba91d4958/src/SharpML.Recurrent/obj/Release/SharpML.Recurrent.dll -------------------------------------------------------------------------------- /src/SharpML.Recurrent/obj/Release/SharpML.Recurrent.pdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrewfry/SharpML-Recurrent/9afeb8189a3c43b321cba39df12fcdfba91d4958/src/SharpML.Recurrent/obj/Release/SharpML.Recurrent.pdb --------------------------------------------------------------------------------