├── .gitignore ├── CONTRIBUTING.md ├── ConvNetSharp.Nuget.sln ├── ConvNetSharp.sln ├── ConvNetSharp.sln.DotSettings ├── Examples ├── Classify2DDemo │ ├── App.config │ ├── Classify2DDemo.csproj │ ├── Program.cs │ └── README.md ├── FlowDemo │ ├── Classify2DDemo.cs │ ├── ExampleCPUDouble.cs │ ├── ExampleCPUSingle.cs │ ├── ExampleGPUSingle.cs │ ├── FlowDemo.csproj │ ├── Program.cs │ └── packages.config ├── FluentMnistDemo │ ├── CircularBuffer.cs │ ├── DataSet.cs │ ├── DataSets.cs │ ├── FluentMnistDemo.csproj │ ├── MnistEntry.cs │ ├── MnistReader.cs │ └── Program.cs ├── MinimalExample │ ├── MinimalExample.csproj │ └── Program.cs ├── MnistDemo.Flow.GPU │ ├── CircularBuffer.cs │ ├── DataSet.cs │ ├── DataSets.cs │ ├── MnistDemo.Flow.GPU.csproj │ ├── MnistEntry.cs │ ├── MnistReader.cs │ ├── Program.cs │ └── packages.config ├── MnistDemo.GPU │ ├── CircularBuffer.cs │ ├── DataSet.cs │ ├── DataSets.cs │ ├── MnistEntry.cs │ ├── MnistReader.cs │ ├── MnitsDemo.GPU.csproj │ └── Program.cs ├── MnistDemo │ ├── CircularBuffer.cs │ ├── DataSet.cs │ ├── DataSets.cs │ ├── MnistDemo.csproj │ ├── MnistEntry.cs │ ├── MnistReader.cs │ └── Program.cs ├── Regression1DDemo │ ├── App.config │ ├── Program.cs │ ├── Properties │ │ └── AssemblyInfo.cs │ └── Regression1DDemo.csproj ├── RnnDemo.Flow │ ├── Program.cs │ ├── RnnDemo.Flow.GPU.csproj │ ├── RnnDemo.cs │ ├── Simple.txt │ ├── TextData.cs │ └── shakespeare │ │ ├── 1kinghenryiv.txt │ │ ├── 1kinghenryvi.txt │ │ ├── 2kinghenryiv.txt │ │ ├── 2kinghenryvi.txt │ │ ├── 3kinghenryvi.txt │ │ ├── allswellthatendswell.txt │ │ ├── antonyandcleopatra.txt │ │ ├── asyoulikeit.txt │ │ ├── comedyoferrors.txt │ │ ├── coriolanus.txt │ │ ├── cymbeline.txt │ │ ├── hamlet.txt │ │ ├── juliuscaesar.txt │ │ ├── kinghenryv.txt │ │ ├── kinghenryviii.txt │ │ ├── kingjohn.txt │ │ ├── kinglear.txt │ │ ├── kingrichardii.txt │ │ ├── kingrichardiii.txt │ │ ├── loverscomplaint.txt │ │ ├── loveslabourslost.txt │ │ ├── macbeth.txt │ │ ├── measureforemeasure.txt │ │ ├── merchantofvenice.txt │ │ ├── merrywivesofwindsor.txt │ │ ├── midsummersnightsdream.txt │ │ ├── muchadoaboutnothing.txt │ │ ├── othello.txt │ │ ├── periclesprinceoftyre.txt │ │ ├── rapeoflucrece.txt │ │ ├── romeoandjuliet.txt │ │ ├── sonnets.txt │ │ ├── tamingoftheshrew.txt │ │ ├── tempest.txt │ │ ├── timonofathens.txt │ │ ├── titusandronicus.txt │ │ ├── troilusandcressida.txt │ │ ├── twelfthnight.txt │ │ ├── twogentlemenofverona.txt │ │ ├── various.txt │ │ ├── venusandadonis.txt │ │ └── winterstale.txt ├── SaveAndLoadDemo.Flow │ ├── Program.cs │ └── SaveAndLoadDemo.Flow.csproj └── SaveAndLoadDemo │ ├── Program.cs │ └── SaveAndLoadDemo.csproj ├── LICENSE ├── README.md ├── clean_bin_obj.bat ├── img ├── ConvNetSharpLogo.png ├── ConvNetSharpLogo.svg ├── graph.png └── structure.png └── src ├── ConvNetSharp.Core.Tests ├── ConvLayerTests.cs ├── ConvNetSharp.Core.Tests.csproj ├── DropoutLayerTests.cs ├── FullyConnLayerTests.cs ├── GradientCheckTools.cs ├── LeakyReluLayerTests.cs ├── PoolLayerTests.cs ├── RegressionLayerTests.cs ├── ReluLayerTests.cs ├── SerializationTests.cs ├── SigmoidLayerTests.cs ├── SoftMaxLayerTests.cs └── TanhLayerTests.cs ├── ConvNetSharp.Core ├── ConvNetSharp.Core.Nuget.csproj ├── ConvNetSharp.Core.csproj ├── Fluent │ ├── Extensions.cs │ └── FluentNet.cs ├── INet.cs ├── Layers │ ├── ConvLayer.cs │ ├── Double │ │ ├── ConvLayer.cs │ │ ├── DropoutLayer.cs │ │ ├── FullyConnLayer.cs │ │ ├── InputLayer.cs │ │ ├── LeakyReluLayer.cs │ │ ├── PoolLayer.cs │ │ ├── RegressionLayer.cs │ │ ├── ReluLayer.cs │ │ ├── SigmoidLayer.cs │ │ ├── SoftmaxLayer.cs │ │ └── TanhLayer.cs │ ├── DropoutLayer.cs │ ├── FullyConnLayer.cs │ ├── IClassificationLayer.cs │ ├── IDotProductLayer.cs │ ├── ILastLayer.cs │ ├── InputLayer.cs │ ├── LastLayerBase.cs │ ├── LayerBase.cs │ ├── LeakyReluLayer.cs │ ├── ParametersAndGradients.cs │ ├── PoolLayer.cs │ ├── RegressionLayer.cs │ ├── ReluLayer.cs │ ├── SigmoidLayer.cs │ ├── Single │ │ ├── ConvLayer.cs │ │ ├── DropoutLayer.cs │ │ ├── FullyConnLayer.cs │ │ ├── InputLayer.cs │ │ ├── LeakyReluLayer.cs │ │ ├── PoolLayer.cs │ │ ├── RegressionLayer.cs │ │ ├── ReluLayer.cs │ │ ├── SigmoidLayer.cs │ │ ├── SoftmaxLayer.cs │ │ └── TanhLayer.cs │ ├── SoftMaxLayer.cs │ └── TanhLayer.cs ├── Net.cs ├── Ops.cs ├── Serialization │ ├── JObjectExtensions.cs │ └── SerializationExtensions.cs ├── Training │ ├── AdamTrainer.cs │ ├── Double │ │ ├── AdamTrainer.cs │ │ └── SgdTrainer.cs │ ├── SgdTrainer.cs │ ├── Single │ │ ├── AdamTrainer.cs │ │ └── SgdTrainer.cs │ └── TrainerBase.cs └── img │ └── icon.png ├── ConvNetSharp.Flow.Tests ├── ConvNetSharp.Flow.Tests.csproj ├── DoubleGpuOpTests.cs ├── DoubleOpTests.cs ├── OpTests.cs ├── OpsTests.cs ├── SerializationTests.cs ├── SingleGpuOpTests.cs ├── SingleOpTests.cs └── VolumeMock.cs ├── ConvNetSharp.Flow ├── ConvNetSharp.Flow.Nuget.csproj ├── ConvNetSharp.Flow.csproj ├── ConvNetSharp.cs ├── Fluent │ └── Extensions.cs ├── Graph │ ├── DifferentiateVisitor.cs │ └── IOpVisitor.cs ├── Layers │ ├── ConvLayer.cs │ ├── DropoutLayer.cs │ ├── FullyConnLayer.cs │ ├── ILastLayer.cs │ ├── InputLayer.cs │ ├── LayerBase.cs │ ├── LeakyReluLayer.cs │ ├── OpWrapperLayer.cs │ ├── PoolLayer.cs │ ├── ReluLayer.cs │ ├── SigmoidLayer.cs │ ├── SoftMaxLayer.cs │ └── TanhLayer.cs ├── Net.cs ├── Ops │ ├── Activation.cs │ ├── ActivationGradient.cs │ ├── Add.cs │ ├── Assign.cs │ ├── Concat.cs │ ├── Const.cs │ ├── Convolution.cs │ ├── ConvolutionFilterGradient.cs │ ├── ConvolutionInputGradient.cs │ ├── Dense.cs │ ├── Div.cs │ ├── Dropout.cs │ ├── DropoutGradient.cs │ ├── Exp.cs │ ├── Extract.cs │ ├── INamedOp.cs │ ├── IPersistable.cs │ ├── LeakyRelu.cs │ ├── LeakyReluGradient.cs │ ├── Log.cs │ ├── MatMult.cs │ ├── Max.cs │ ├── Mult.cs │ ├── Negate.cs │ ├── Op.cs │ ├── OpVisitor.cs │ ├── PlaceHolder.cs │ ├── Pool.cs │ ├── PoolGradient.cs │ ├── Power.cs │ ├── Reshape.cs │ ├── Shape.cs │ ├── SoftMax.cs │ ├── SoftMaxCrossEntropy.cs │ ├── SoftMaxGradient.cs │ ├── Sqrt.cs │ ├── Sum.cs │ ├── Tile.cs │ ├── Transpose.cs │ └── Variable.cs ├── Scope.cs ├── Serialization │ └── SerializationExtensions.cs ├── Session.cs ├── Training │ ├── AdamOptimizer.cs │ ├── GradientDescentOptimizer.cs │ ├── SgdTrainer.cs │ └── TrainerBase.cs └── img │ └── icon.png ├── ConvNetSharp.Performance.Tests ├── ConvNetSharp.Performance.Tests.csproj └── Program.cs ├── ConvNetSharp.Utils ├── ConvNetSharp.Utils.Nuget.csproj ├── ConvNetSharp.Utils.csproj ├── GraphVisualizer │ ├── GraphControl.xaml │ ├── GraphControl.xaml.cs │ ├── OpEdge.cs │ └── ViewModel.cs ├── Image.cs └── img │ └── icon.png ├── ConvNetSharp.Volume.GPU.Tests ├── ConvNetSharp.Volume.GPU.Tests.csproj ├── DoubleVolumeTests.cs ├── GenericVolumeTests.cs └── SingleVolumeTests.cs ├── ConvNetSharp.Volume.GPU ├── ActivationTypeExtension.cs ├── ConvNetSharp.Volume.GPU.Nuget.csproj ├── ConvNetSharp.Volume.GPU.csproj ├── CudaHostMemoryRegion.cs ├── DataLocation.cs ├── Double │ ├── BuilderInstance.cs │ ├── Kernels │ │ ├── concat.cu │ │ ├── div.cu │ │ ├── exp.cu │ │ ├── extract.cu │ │ ├── leakyrelu.cu │ │ ├── leakyrelu_gradient.cu │ │ ├── log.cu │ │ ├── power.cu │ │ └── tile.cu │ ├── Volume.cs │ ├── VolumeBuilder.cs │ └── VolumeStorage.cs ├── GpuContext.cs ├── IVolumeStorage.cs ├── KernelLoader.cs ├── Single │ ├── BuilderInstance.cs │ ├── Kernels │ │ ├── concat.cu │ │ ├── div.cu │ │ ├── exp.cu │ │ ├── extract.cu │ │ ├── leakyrelu.cu │ │ ├── leakyrelu_gradient.cu │ │ ├── log.cu │ │ ├── power.cu │ │ └── tile.cu │ ├── Volume.cs │ ├── VolumeBuilder.cs │ └── VolumeStorage.cs ├── TensorReduceOpExtension.cs └── img │ └── icon.png ├── ConvNetSharp.Volume.Tests ├── AssertNumber.cs ├── ConvNetSharp.Volume.Tests.csproj ├── DoubleVolumeTests.cs ├── GenericVolumeTests.cs ├── ShapeTests.cs ├── SingleVolumeTests.cs └── VolumeTests.cs ├── ConvNetSharp.Volume ├── ActivationType.cs ├── ArrayExtensions.cs ├── BuilderInstance.cs ├── ConvNetSharp.Volume.csproj ├── Double │ ├── BuilderInstance.cs │ ├── Volume.cs │ └── VolumeBuilder.cs ├── NcwhVolumeStorage.cs ├── RandomUtilities.cs ├── Shape.cs ├── ShapeExtension.cs ├── Single │ ├── BuilderInstance.cs │ ├── Volume.cs │ └── VolumeBuilder.cs ├── TensorReduceOp.cs ├── Volume.cs ├── VolumeBuilder.cs ├── VolumeStorage.cs └── img │ └── icon.png └── external └── ManagedCuda ├── CudaBlas.XML ├── CudaBlas.dll ├── CudaDNN.XML ├── CudaDNN.dll ├── ManagedCuda.dll ├── ManagedCuda.xml ├── NVRTC.XML └── NVRTC.dll /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to contribute 2 | 3 | 1) Pick or create some issues and assign them to yourself 4 | 2) Create a feature branch 5 | 3) Commit documented changes along with unit tests 6 | 4) Create a Pull request 7 | 8 | 9 | Contribution is welcome on any aspect of the project but the following points are the priority: 10 | * Better documentation: Fill up missing public methods documentation headers, find a way to easily export it and host it in github 11 | * Better error messages: End user should understand why an exception is raised (e.g. wrong input shape) 12 | * Make the api less verbose (e.g. volume creation is ugly: ```BuilderInstance.Volume.From(new[] { 0.3, -0.5 }, new Shape(2))```) 13 | * Optimization of ConvNetSharp.Flow computation graph (factorization, simple optimization like replace X * 1 by X, ...) 14 | -------------------------------------------------------------------------------- /ConvNetSharp.sln.DotSettings: -------------------------------------------------------------------------------- 1 |  2 | True 3 | True 4 | True 5 | True 6 | True 7 | True 8 | True 9 | True 10 | True 11 | True -------------------------------------------------------------------------------- /Examples/Classify2DDemo/App.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /Examples/Classify2DDemo/Classify2DDemo.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netcoreapp3.1 5 | false 6 | Classify2DDemo.Program 7 | 8 | Exe 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /Examples/Classify2DDemo/README.md: -------------------------------------------------------------------------------- 1 | This example shows how to train a neural network to classify a two dimensions vector (x1, x2) into one of two classes (y). 2 | This is therefore supervised training (because the label is given in the training set) 3 | 4 | We want to learn `(x1, x2) -> y where y = 0 OR y = 1` 5 | 6 | ### Trainin set 7 | 8 | | x1 | x2 | y (label / class) | 9 | | ------------- | ------------- | ------------- | 10 | |-0.4326| 1.1909| 1| 11 | |3.0| 4.0|1| 12 | |0.1253| -0.0376|1| 13 | |0.2877| 0.3273|1| 14 | |-1.1465| 0.1746|1| 15 | |1.8133| 1.0139|0| 16 | |2.7258| 1.0668|0| 17 | |1.4117| 0.5593|0| 18 | |4.1832| 0.3044|0| 19 | |1.8636| 0.1677|0| 20 | |0.5| 3.2|1| 21 | |0.8| 3.2|1| 22 | |1.0| -2.2|1| 23 | 24 | ### Code 25 | 26 | #### Learning 27 | This example shows how to input a batch of data (**n** examples at the time) rather than one example at the time. 28 | This means that the network will learn from several example at the time rather than learning by looking at one example at the time. This is more efficient. 29 | 30 | We first try to teach the network `(x1, x2) -> (y1, y2)` where `(y1, y2)` is the one-hot encoded label. 31 | ``` 32 | y = 1 -> (y1, y2) = (1, 0) 33 | y = 2 -> (y1, y2) = (0, 1) 34 | ``` 35 | 36 | In *Classify2DUpdate*, a Volume of shape [1, 1, 2, n] is created to store input data `(x1, x2)` and another Volume of shape [1, 1, 2, n] is created to store the one-hot encoded labels where n is the batchsize. 37 | 38 | #### Inference 39 | 40 | An input volume **netx** of shape [1, 1, 2, n] is created. For each input (n inputs here), we want to guess the associated label. 41 | This is done with those lines: 42 | 43 | ```c# 44 | var result = net.Forward(netx); 45 | var c = net.GetPrediction(); 46 | ``` 47 | **Result** will contain the one-hot encoded predicted labels. 48 | **GetPrediction** is a convenience method that will translate from one-hot encoded labels to the labels. 49 | ``` 50 | e.g. 51 | (1, 0) -> 0 52 | (0, 1) -> 1 53 | ``` 54 | *note:* Learning was done using batch of **n** elements but you can do the inference on any batch size (i.e. the 'n' of inference doesnt have to be the same as the 'n' used during training) 55 | -------------------------------------------------------------------------------- /Examples/FlowDemo/FlowDemo.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netcoreapp3.1 5 | false 6 | FlowDemo.Program 7 | 8 | Exe 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /Examples/FlowDemo/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace FlowDemo 4 | { 5 | internal class Program 6 | { 7 | [STAThread] 8 | private static void Main() 9 | { 10 | //ExampleCpuSingle.Example1(); 11 | //ExampleCpuDouble.Example2(); 12 | //ExampleGpuSingle.Example1(); 13 | NetExampleSingle.Classify2DDemo(); 14 | } 15 | } 16 | } -------------------------------------------------------------------------------- /Examples/FlowDemo/packages.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | -------------------------------------------------------------------------------- /Examples/FluentMnistDemo/CircularBuffer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | 4 | namespace FluentMnistDemo 5 | { 6 | public class CircularBuffer 7 | { 8 | private readonly T[] _buffer; 9 | private int _nextFree; 10 | 11 | public CircularBuffer(int capacity) 12 | { 13 | this.Capacity = capacity; 14 | this.Count = 0; 15 | this._buffer = new T[capacity]; 16 | } 17 | 18 | public int Capacity { get; } 19 | 20 | public int Count { get; private set; } 21 | 22 | public IEnumerable Items => this._buffer; 23 | 24 | public void Add(T o) 25 | { 26 | this._buffer[this._nextFree] = o; 27 | this._nextFree = (this._nextFree + 1) % this._buffer.Length; 28 | this.Count = Math.Min(this.Count + 1, this.Capacity); 29 | } 30 | } 31 | } -------------------------------------------------------------------------------- /Examples/FluentMnistDemo/FluentMnistDemo.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netcoreapp3.1 5 | false 6 | FluentMnistDemo.Program 7 | 8 | Exe 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /Examples/FluentMnistDemo/MnistEntry.cs: -------------------------------------------------------------------------------- 1 | namespace FluentMnistDemo 2 | { 3 | public class MnistEntry 4 | { 5 | public byte[] Image { get; set; } 6 | 7 | public int Label { get; set; } 8 | 9 | public override string ToString() 10 | { 11 | return "Label: " + this.Label; 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /Examples/MinimalExample/MinimalExample.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netcoreapp3.1 5 | false 6 | MinimalExample.Program 7 | 8 | Exe 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /Examples/MinimalExample/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using ConvNetSharp.Core; 3 | using ConvNetSharp.Core.Layers.Double; 4 | using ConvNetSharp.Core.Training.Double; 5 | using ConvNetSharp.Volume; 6 | using ConvNetSharp.Volume.Double; 7 | 8 | namespace MinimalExample 9 | { 10 | internal class Program 11 | { 12 | private static void Main() 13 | { 14 | // species a 2-layer neural network with one hidden layer of 20 neurons 15 | var net = new Net(); 16 | 17 | // input layer declares size of input. here: 2-D data 18 | // ConvNetJS works on 3-Dimensional volumes (width, height, depth), but if you're not dealing with images 19 | // then the first two dimensions (width, height) will always be kept at size 1 20 | net.AddLayer(new InputLayer(1, 1, 2)); 21 | 22 | // declare 20 neurons 23 | net.AddLayer(new FullyConnLayer(20)); 24 | 25 | // declare a ReLU (rectified linear unit non-linearity) 26 | net.AddLayer(new ReluLayer()); 27 | 28 | // declare a fully connected layer that will be used by the softmax layer 29 | net.AddLayer(new FullyConnLayer(10)); 30 | 31 | // declare the linear classifier on top of the previous hidden layer 32 | net.AddLayer(new SoftmaxLayer(10)); 33 | 34 | // forward a random data point through the network 35 | var x = BuilderInstance.Volume.From(new[] { 0.3, -0.5 }, new Shape(2)); 36 | 37 | var prob = net.Forward(x); 38 | 39 | // prob is a Volume. Volumes have a property Weights that stores the raw data, and WeightGradients that stores gradients 40 | Console.WriteLine("probability that x is class 0: " + prob.Get(0)); // prints e.g. 0.50101 41 | 42 | var trainer = new SgdTrainer(net) { LearningRate = 0.01 }; 43 | trainer.Train(x, BuilderInstance.Volume.From(new[] { 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 }, new Shape(1, 1, 10, 1))); // train the network, specifying that x is class zero 44 | 45 | var prob2 = net.Forward(x); 46 | Console.WriteLine("probability that x is class 0: " + prob2.Get(0)); 47 | // now prints 0.50374, slightly higher than previous 0.50101: the networks 48 | // weights have been adjusted by the Trainer to give a higher probability to 49 | // the class we trained the network with (zero) 50 | } 51 | } 52 | } -------------------------------------------------------------------------------- /Examples/MnistDemo.Flow.GPU/CircularBuffer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | 4 | namespace MnistDemo.GPU 5 | { 6 | public class CircularBuffer 7 | { 8 | private readonly T[] _buffer; 9 | private int _nextFree; 10 | 11 | public CircularBuffer(int capacity) 12 | { 13 | this.Capacity = capacity; 14 | this.Count = 0; 15 | this._buffer = new T[capacity]; 16 | } 17 | 18 | public int Capacity { get; } 19 | 20 | public int Count { get; private set; } 21 | 22 | public IEnumerable Items => this._buffer; 23 | 24 | public void Add(T o) 25 | { 26 | this._buffer[this._nextFree] = o; 27 | this._nextFree = (this._nextFree + 1) % this._buffer.Length; 28 | this.Count = Math.Min(this.Count + 1, this.Capacity); 29 | } 30 | } 31 | } -------------------------------------------------------------------------------- /Examples/MnistDemo.Flow.GPU/MnistDemo.Flow.GPU.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netcoreapp3.1 5 | false 6 | MnistDemo.GPU.Program 7 | 8 | Exe 9 | 10 | 11 | 12 | x64 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /Examples/MnistDemo.Flow.GPU/MnistEntry.cs: -------------------------------------------------------------------------------- 1 | namespace MnistDemo.Flow.GPU 2 | { 3 | public class MnistEntry 4 | { 5 | public byte[] Image { get; set; } 6 | 7 | public int Label { get; set; } 8 | 9 | public override string ToString() 10 | { 11 | return "Label: " + this.Label; 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /Examples/MnistDemo.Flow.GPU/packages.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | -------------------------------------------------------------------------------- /Examples/MnistDemo.GPU/CircularBuffer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | 4 | namespace MnistDemo.GPU 5 | { 6 | public class CircularBuffer 7 | { 8 | private readonly T[] _buffer; 9 | private int _nextFree; 10 | 11 | public CircularBuffer(int capacity) 12 | { 13 | this.Capacity = capacity; 14 | this.Count = 0; 15 | this._buffer = new T[capacity]; 16 | } 17 | 18 | public int Capacity { get; } 19 | 20 | public int Count { get; private set; } 21 | 22 | public IEnumerable Items => this._buffer; 23 | 24 | public void Add(T o) 25 | { 26 | this._buffer[this._nextFree] = o; 27 | this._nextFree = (this._nextFree + 1) % this._buffer.Length; 28 | this.Count = Math.Min(this.Count + 1, this.Capacity); 29 | } 30 | } 31 | } -------------------------------------------------------------------------------- /Examples/MnistDemo.GPU/MnistEntry.cs: -------------------------------------------------------------------------------- 1 | namespace MnistDemo.GPU 2 | { 3 | public class MnistEntry 4 | { 5 | public byte[] Image { get; set; } 6 | 7 | public int Label { get; set; } 8 | 9 | public override string ToString() 10 | { 11 | return "Label: " + this.Label; 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /Examples/MnistDemo.GPU/MnitsDemo.GPU.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netcoreapp3.1 5 | false 6 | MnistDemo.GPU.Program 7 | 8 | Exe 9 | 10 | 11 | 12 | x64 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /Examples/MnistDemo/CircularBuffer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | 4 | namespace MnistDemo 5 | { 6 | public class CircularBuffer 7 | { 8 | private readonly T[] _buffer; 9 | private int _nextFree; 10 | 11 | public CircularBuffer(int capacity) 12 | { 13 | this.Capacity = capacity; 14 | this.Count = 0; 15 | this._buffer = new T[capacity]; 16 | } 17 | 18 | public int Capacity { get; } 19 | 20 | public int Count { get; private set; } 21 | 22 | public IEnumerable Items => this._buffer; 23 | 24 | public void Add(T o) 25 | { 26 | this._buffer[this._nextFree] = o; 27 | this._nextFree = (this._nextFree + 1) % this._buffer.Length; 28 | this.Count = Math.Min(this.Count + 1, this.Capacity); 29 | } 30 | } 31 | } -------------------------------------------------------------------------------- /Examples/MnistDemo/MnistDemo.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netcoreapp3.1 5 | false 6 | MnistDemo.Program 7 | 8 | Exe 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /Examples/MnistDemo/MnistEntry.cs: -------------------------------------------------------------------------------- 1 | namespace MnistDemo 2 | { 3 | public class MnistEntry 4 | { 5 | public byte[] Image { get; set; } 6 | 7 | public int Label { get; set; } 8 | 9 | public override string ToString() 10 | { 11 | return "Label: " + this.Label; 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /Examples/Regression1DDemo/App.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /Examples/Regression1DDemo/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using ConvNetSharp.Core; 3 | using ConvNetSharp.Core.Layers.Double; 4 | using ConvNetSharp.Core.Training; 5 | using ConvNetSharp.Core.Training.Double; 6 | using ConvNetSharp.Volume; 7 | using ConvNetSharp.Volume.Double; 8 | 9 | namespace Regression1DDemo 10 | { 11 | internal class Program 12 | { 13 | private static void Main() 14 | { 15 | Regression1DDemo(); 16 | } 17 | 18 | private static void Regression1DDemo() 19 | { 20 | var net = new Net(); 21 | net.AddLayer(new InputLayer(1, 1, 1)); 22 | net.AddLayer(new FullyConnLayer(20)); 23 | net.AddLayer(new ReluLayer()); 24 | net.AddLayer(new FullyConnLayer(20)); 25 | net.AddLayer(new SigmoidLayer()); 26 | net.AddLayer(new FullyConnLayer(1)); 27 | net.AddLayer(new RegressionLayer()); 28 | 29 | var trainer = new SgdTrainer(net) { LearningRate = 0.01, Momentum = 0.0, BatchSize = 1, L2Decay = 0.001 }; 30 | 31 | // Function we want to learn 32 | double[] x = { 0.0, 0.5, 1.0 }; 33 | double[] y = { 0.0, 0.1, 0.2 }; 34 | var n = x.Length; 35 | 36 | // Training 37 | do 38 | { 39 | RegressionUpdate(n, x, trainer, y); 40 | } while (!Console.KeyAvailable); 41 | 42 | // Testing 43 | var netx = BuilderInstance.Volume.SameAs(new Shape(1, 1, 1)); 44 | for (var ix = 0; ix < n; ix++) 45 | { 46 | netx.Set(0, 0, 0, x[ix]); 47 | var result = net.Forward(netx); 48 | } 49 | } 50 | 51 | private static void RegressionUpdate(int n, double[] x, TrainerBase trainer, double[] y) 52 | { 53 | var netx = BuilderInstance.Volume.SameAs(new Shape(1, 1, 1, n)); 54 | var nety = BuilderInstance.Volume.SameAs(new Shape(1, 1, 1, n)); 55 | var avloss = 0.0; 56 | 57 | for (var ix = 0; ix < n; ix++) 58 | { 59 | netx.Set(0, 0, 0, ix, x[ix]); 60 | nety.Set(0, 0, 0, ix, y[ix]); 61 | } 62 | 63 | for (var iters = 0; iters < 50; iters++) 64 | { 65 | trainer.Train(netx, nety); 66 | avloss += trainer.Loss; 67 | } 68 | 69 | avloss /= n * 50.0; 70 | Console.WriteLine("Loss:" + avloss); 71 | } 72 | } 73 | } -------------------------------------------------------------------------------- /Examples/Regression1DDemo/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.CompilerServices; 3 | using System.Runtime.InteropServices; 4 | 5 | // General Information about an assembly is controlled through the following 6 | // set of attributes. Change these attribute values to modify the information 7 | // associated with an assembly. 8 | [assembly: AssemblyTitle("Regression1DDemo")] 9 | [assembly: AssemblyDescription("")] 10 | [assembly: AssemblyConfiguration("")] 11 | [assembly: AssemblyCompany("")] 12 | [assembly: AssemblyProduct("Regression1DDemo")] 13 | [assembly: AssemblyCopyright("Copyright © 2017")] 14 | [assembly: AssemblyTrademark("")] 15 | [assembly: AssemblyCulture("")] 16 | 17 | // Setting ComVisible to false makes the types in this assembly not visible 18 | // to COM components. If you need to access a type in this assembly from 19 | // COM, set the ComVisible attribute to true on that type. 20 | [assembly: ComVisible(false)] 21 | 22 | // The following GUID is for the ID of the typelib if this project is exposed to COM 23 | [assembly: Guid("ebb6ceb5-1390-405c-b0f6-1e9b24306079")] 24 | 25 | // Version information for an assembly consists of the following four values: 26 | // 27 | // Major Version 28 | // Minor Version 29 | // Build Number 30 | // Revision 31 | // 32 | // You can specify all the values or you can default the Build and Revision Numbers 33 | // by using the '*' as shown below: 34 | // [assembly: AssemblyVersion("1.0.*")] 35 | [assembly: AssemblyVersion("1.0.0.0")] 36 | [assembly: AssemblyFileVersion("1.0.0.0")] 37 | -------------------------------------------------------------------------------- /Examples/RnnDemo.Flow/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | 4 | namespace RnnDemo.Flow.GPU 5 | { 6 | internal class Program 7 | { 8 | /// 9 | /// This demo is has some issues: 10 | /// - Network never manages to get every word right (not matter the size of hidden state) 11 | /// - When starting a new epoch, loss increases. This is probably due to discontinuity in the state. 12 | /// - Text generation is not impressive at all. Some real words are sometimes generated but it's mostly garbage. 13 | /// 14 | /// TODO: use LSTM 15 | /// 16 | private static void Main() 17 | { 18 | // Load Shakespeare data 19 | //var book = new StringBuilder(); 20 | //foreach (var file in Directory.EnumerateFiles(@"./shakespeare/", "*.txt")) 21 | //{ 22 | // using (var sr = new StreamReader(new FileStream(file, FileMode.Open, FileAccess.Read))) 23 | // { 24 | // book.Append(sr.ReadToEnd()); 25 | // } 26 | //} 27 | 28 | //var textData = new TextData(book.ToString()); 29 | var textData = new TextData(File.ReadAllText("Simple.txt")); 30 | 31 | // Start learning 32 | var rnnDemo = new RnnDemo(textData, 3, 100, 300, 0.01f); 33 | 34 | while (true) 35 | { 36 | Console.Clear(); 37 | Console.WriteLine(Environment.NewLine + "---- Rnn demo ----"); 38 | Console.WriteLine("0) Learning"); 39 | Console.WriteLine("1) Text generation"); 40 | Console.WriteLine("2) Exit"); 41 | 42 | var c = Console.ReadKey(); 43 | switch (c.KeyChar) 44 | { 45 | case '2': 46 | return; 47 | case '0': 48 | rnnDemo.Learn(); 49 | break; 50 | case '1': 51 | rnnDemo.GenerateText(); 52 | break; 53 | } 54 | } 55 | } 56 | } 57 | } -------------------------------------------------------------------------------- /Examples/RnnDemo.Flow/RnnDemo.Flow.GPU.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netcoreapp3.1 5 | false 6 | RnnDemo.Flow.GPU.Program 7 | 8 | Exe 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | x64 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | PreserveNewest 29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /Examples/RnnDemo.Flow/Simple.txt: -------------------------------------------------------------------------------- 1 | Poncho is a police dog. He is from Madrid. He is eight years old. He is a very good dog. He performs at a show for police dogs. People especially love that he can perform CPR. His partner falls to the ground. He runs to him and tries to help him. He starts jumping on his chest. After doing this for a while, the officer gets up. -------------------------------------------------------------------------------- /Examples/RnnDemo.Flow/TextData.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Linq; 3 | 4 | namespace RnnDemo.Flow.GPU 5 | { 6 | internal class TextData 7 | { 8 | public TextData(string rawData) 9 | { 10 | this.RawData = rawData; 11 | this.Vocabulary = rawData.Select(c => c).Distinct().OrderBy(c => c).ToList(); 12 | } 13 | 14 | public string RawData { get; } 15 | 16 | public List Vocabulary { get; } 17 | } 18 | } -------------------------------------------------------------------------------- /Examples/SaveAndLoadDemo.Flow/SaveAndLoadDemo.Flow.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netcoreapp3.1 5 | false 6 | SaveAndLoadDemo.Flow.Program 7 | 8 | Exe 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /Examples/SaveAndLoadDemo/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using ConvNetSharp.Core; 3 | using ConvNetSharp.Core.Layers.Double; 4 | using ConvNetSharp.Core.Serialization; 5 | using ConvNetSharp.Core.Training.Double; 6 | using ConvNetSharp.Volume; 7 | using ConvNetSharp.Volume.Double; 8 | 9 | namespace SaveAndLoadDemo 10 | { 11 | internal class Program 12 | { 13 | /// 14 | /// This sample shows how to serialize and deserialize a ConvNetSharp.Core network 15 | /// 1) Network creation 16 | /// 2) Dummy Training (only use a single data point) 17 | /// 3) Serialization 18 | /// 4) Deserialization 19 | /// 20 | private static void Main() 21 | { 22 | // 1) Network creation 23 | var net = new Net(); 24 | 25 | net.AddLayer(new InputLayer(1, 1, 2)); 26 | net.AddLayer(new FullyConnLayer(20)); 27 | net.AddLayer(new ReluLayer()); 28 | net.AddLayer(new FullyConnLayer(10)); 29 | net.AddLayer(new SoftmaxLayer(10)); 30 | 31 | // 2) Dummy Training (only use a single data point) 32 | var x = BuilderInstance.Volume.From(new[] {0.3, -0.5}, new Shape(2)); 33 | var y = BuilderInstance.Volume.From(new[] {1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, new Shape(10)); 34 | 35 | var count = 0; 36 | var trainer = new SgdTrainer(net) {LearningRate = 0.01}; 37 | do 38 | { 39 | trainer.Train(x, y); // train the network, specifying that x is class zero 40 | Console.WriteLine($"Loss: {trainer.Loss}"); 41 | count++; 42 | } while (trainer.Loss > 1e-2); 43 | 44 | Console.WriteLine($"{count}"); 45 | 46 | // Forward pass with original network 47 | var prob1 = net.Forward(x); 48 | Console.WriteLine("probability that x is class 0: " + prob1.Get(0)); 49 | 50 | // 3) Serialization 51 | var json = net.ToJson(); 52 | 53 | // 4) Deserialization 54 | var deserialized = SerializationExtensions.FromJson(json); 55 | 56 | // Forward pass with deserialized network 57 | var prob2 = deserialized.Forward(x); 58 | Console.WriteLine("probability that x is class 0: " + prob2.Get(0)); // This should give exactly the same result as previous network evaluation 59 | 60 | Console.ReadLine(); 61 | } 62 | } 63 | } -------------------------------------------------------------------------------- /Examples/SaveAndLoadDemo/SaveAndLoadDemo.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netcoreapp3.1 5 | false 6 | SaveAndLoadDemo.Program 7 | 8 | Exe 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License 2 | 3 | Copyright (c) 2018 Cédric Bovar 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /clean_bin_obj.bat: -------------------------------------------------------------------------------- 1 | rd /s /q .\src\ConvNetSharp.Core\bin 2 | rd /s /q .\src\ConvNetSharp.Core.Tests\bin 3 | rd /s /q .\src\ConvNetSharp.Volume\bin 4 | rd /s /q .\src\ConvNetSharp.Volume.GPU\bin 5 | rd /s /q .\src\ConvNetSharp.Volume.GPU.Tests\bin 6 | rd /s /q .\src\ConvNetSharp.Volume.Tests\bin 7 | rd /s /q .\src\ConvNetSharp.Flow\bin 8 | rd /s /q .\src\ConvNetSharp.Flow.Tests\bin 9 | rd /s /q .\src\ConvNetSharp.Performance.Tests\bin 10 | rd /s /q .\Examples\Classify2DDemo\bin 11 | rd /s /q .\Examples\ConvNetSharp.Example\bin 12 | rd /s /q .\Examples\FluentMnistDemo\bin 13 | rd /s /q .\Examples\MinimalExample\bin 14 | rd /s /q .\Examples\MnistDemo\bin 15 | rd /s /q .\Examples\MnistDemo.GPU\bin 16 | rd /s /q .\src\ConvNetSharp.Utils\bin 17 | rd /s /q .\Examples\MnistDemo.Flow.GPU\bin 18 | rd /s /q .\Examples\FlowDemo\bin 19 | rd /s /q .\Examples\RnnDemo.Flow\bin 20 | rd /s /q .\Examples\SaveAndLoadDemo\bin 21 | rd /s /q .\Examples\SaveAndLoadDemo.Flow\bin 22 | 23 | rd /s /q .\src\ConvNetSharp.Core\obj 24 | rd /s /q .\src\ConvNetSharp.Core.Tests\obj 25 | rd /s /q .\src\ConvNetSharp.Volume\obj 26 | rd /s /q .\src\ConvNetSharp.Volume.GPU\obj 27 | rd /s /q .\src\ConvNetSharp.Volume.GPU.Tests\obj 28 | rd /s /q .\src\ConvNetSharp.Volume.Tests\obj 29 | rd /s /q .\src\ConvNetSharp.Flow\obj 30 | rd /s /q .\src\ConvNetSharp.Flow.Tests\obj 31 | rd /s /q .\src\ConvNetSharp.Performance.Tests\obj 32 | rd /s /q .\Examples\Classify2DDemo\obj 33 | rd /s /q .\Examples\ConvNetSharp.Example\obj 34 | rd /s /q .\Examples\FluentMnistDemo\obj 35 | rd /s /q .\Examples\MinimalExample\obj 36 | rd /s /q .\Examples\MnistDemo\obj 37 | rd /s /q .\Examples\MnistDemo.GPU\obj 38 | rd /s /q .\src\ConvNetSharp.Utils\obj 39 | rd /s /q .\Examples\MnistDemo.Flow.GPU\obj 40 | rd /s /q .\Examples\FlowDemo\obj 41 | rd /s /q .\Examples\RnnDemo.Flow\obj 42 | rd /s /q .\Examples\SaveAndLoadDemo\obj 43 | rd /s /q .\Examples\SaveAndLoadDemo.Flow\obj -------------------------------------------------------------------------------- /img/ConvNetSharpLogo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbovar/ConvNetSharp/ccaf6f6959c12026c4e1a0e396a7711654c92551/img/ConvNetSharpLogo.png -------------------------------------------------------------------------------- /img/graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbovar/ConvNetSharp/ccaf6f6959c12026c4e1a0e396a7711654c92551/img/graph.png -------------------------------------------------------------------------------- /img/structure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbovar/ConvNetSharp/ccaf6f6959c12026c4e1a0e396a7711654c92551/img/structure.png -------------------------------------------------------------------------------- /src/ConvNetSharp.Core.Tests/ConvNetSharp.Core.Tests.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netcoreapp3.1 5 | false 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | all 14 | runtime; build; native; contentfiles; analyzers; buildtransitive 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /src/ConvNetSharp.Core.Tests/DropoutLayerTests.cs: -------------------------------------------------------------------------------- 1 | using System.Linq; 2 | using ConvNetSharp.Core.Layers; 3 | using ConvNetSharp.Volume; 4 | using ConvNetSharp.Volume.Double; 5 | using NUnit.Framework; 6 | 7 | namespace ConvNetSharp.Core.Tests 8 | { 9 | [TestFixture] 10 | public class DropoutLayerTests 11 | { 12 | [Test] 13 | public void Learning() 14 | { 15 | var n = 1000000; 16 | var dropProbability = 0.2; 17 | var layer = new DropoutLayer(dropProbability); 18 | layer.Init(1, 1, n); 19 | 20 | var input = BuilderInstance.Volume.From(new double[n].Populate(1.0), new Shape(1, 1, n, 1)); 21 | var result = layer.DoForward(input, true); 22 | 23 | var val = result.ToArray().First(o => o != 0.0); 24 | var scalingFactor = 1.0 / (1.0 - dropProbability); 25 | Assert.AreEqual(scalingFactor, val); // Make sure output is scaled during learning 26 | 27 | var average = result.ToArray().Average(); 28 | var measuredProba = average * dropProbability; 29 | Assert.AreEqual(dropProbability, measuredProba, 0.001); // Make sure dropout really happened 30 | } 31 | 32 | [Test] 33 | public void NotLearning() 34 | { 35 | var n = 1000000; 36 | var dropProbability = 0.2; 37 | var layer = new DropoutLayer(dropProbability); 38 | layer.Init(1, 1, n); 39 | 40 | var input = BuilderInstance.Volume.From(new double[n].Populate(1.0), new Shape(1, 1, n, 1)); 41 | var result = layer.DoForward(input); 42 | 43 | var average = result.ToArray().Average(); 44 | Assert.AreEqual(1.0, average); // Let everything go through 45 | } 46 | } 47 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core.Tests/LeakyReluLayerTests.cs: -------------------------------------------------------------------------------- 1 | using System.Linq; 2 | using ConvNetSharp.Core.Layers.Double; 3 | using ConvNetSharp.Volume; 4 | using ConvNetSharp.Volume.Double; 5 | using NUnit.Framework; 6 | 7 | namespace ConvNetSharp.Core.Tests 8 | { 9 | [TestFixture] 10 | public class LeakyReluLayerTests 11 | { 12 | [Test] 13 | public void ComputeTwiceGradientShouldYieldTheSameResult() 14 | { 15 | const int inputWidth = 20; 16 | const int inputHeight = 20; 17 | const int inputDepth = 2; 18 | 19 | var layer = new LeakyReluLayer(0.01); 20 | layer.Init(inputWidth, inputHeight, inputDepth); 21 | 22 | // Forward pass 23 | var input = BuilderInstance.Volume.Random(new Shape(inputWidth, inputHeight, inputDepth)); 24 | var output = layer.DoForward(input, true); 25 | 26 | // Set output gradients to 1 27 | var outputGradient = BuilderInstance.Volume.From(new double[output.Shape.TotalLength].Populate(1.0), output.Shape); 28 | 29 | // Backward pass to retrieve gradients 30 | layer.Backward(outputGradient); 31 | var step1 = ((Volume.Double.Volume)layer.InputActivationGradients.Clone()).ToArray(); 32 | 33 | layer.Backward(outputGradient); 34 | var step2 = ((Volume.Double.Volume)layer.InputActivationGradients.Clone()).ToArray(); 35 | 36 | Assert.IsTrue(step1.SequenceEqual(step2)); 37 | } 38 | 39 | [Test] 40 | public void GradientWrtInputCheck() 41 | { 42 | const int inputWidth = 20; 43 | const int inputHeight = 20; 44 | const int inputDepth = 2; 45 | 46 | const int batchSize = 3; 47 | 48 | // Create layer 49 | var layer = new LeakyReluLayer(0.01); 50 | 51 | GradientCheckTools.GradientCheck(layer, inputWidth, inputHeight, inputDepth, batchSize, 1e-6); 52 | } 53 | } 54 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core.Tests/RegressionLayerTests.cs: -------------------------------------------------------------------------------- 1 | using ConvNetSharp.Core.Layers.Double; 2 | using NUnit.Framework; 3 | 4 | namespace ConvNetSharp.Core.Tests 5 | { 6 | [TestFixture] 7 | public class RegressionLayerTests 8 | { 9 | [Test] 10 | public void Instantiation() 11 | { 12 | const int inputWidth = 20; 13 | const int inputHeight = 20; 14 | const int inputDepth = 2; 15 | 16 | var layer = new ReluLayer(); 17 | layer.Init(inputWidth, inputHeight, inputDepth); 18 | 19 | Assert.AreEqual(20, layer.OutputWidth); 20 | Assert.AreEqual(20, layer.OutputHeight); 21 | Assert.AreEqual(2, layer.OutputDepth); 22 | } 23 | } 24 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core.Tests/ReluLayerTests.cs: -------------------------------------------------------------------------------- 1 | using System.Linq; 2 | using ConvNetSharp.Core.Layers.Double; 3 | using ConvNetSharp.Volume; 4 | using ConvNetSharp.Volume.Double; 5 | using NUnit.Framework; 6 | 7 | namespace ConvNetSharp.Core.Tests 8 | { 9 | [TestFixture] 10 | public class ReluLayerTests 11 | { 12 | [Test] 13 | public void ComputeTwiceGradientShouldYieldTheSameResult() 14 | { 15 | const int inputWidth = 20; 16 | const int inputHeight = 20; 17 | const int inputDepth = 2; 18 | 19 | var layer = new ReluLayer(); 20 | layer.Init(inputWidth, inputHeight, inputDepth); 21 | 22 | // Forward pass 23 | var input = BuilderInstance.Volume.Random(new Shape(inputWidth, inputHeight, inputDepth)); 24 | var output = layer.DoForward(input, true); 25 | 26 | // Set output gradients to 1 27 | var outputGradient = BuilderInstance.Volume.From(new double[output.Shape.TotalLength].Populate(1.0), output.Shape); 28 | 29 | // Backward pass to retrieve gradients 30 | layer.Backward(outputGradient); 31 | var step1 = ((Volume.Double.Volume)layer.InputActivationGradients.Clone()).ToArray(); 32 | 33 | layer.Backward(outputGradient); 34 | var step2 = ((Volume.Double.Volume)layer.InputActivationGradients.Clone()).ToArray(); 35 | 36 | Assert.IsTrue(step1.SequenceEqual(step2)); 37 | } 38 | 39 | [Test] 40 | public void GradientWrtInputCheck() 41 | { 42 | const int inputWidth = 20; 43 | const int inputHeight = 20; 44 | const int inputDepth = 2; 45 | 46 | const int batchSize = 3; 47 | 48 | // Create layer 49 | var layer = new ReluLayer(); 50 | 51 | GradientCheckTools.GradientCheck(layer, inputWidth, inputHeight, inputDepth, batchSize, 1e-6); 52 | } 53 | } 54 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core.Tests/SoftMaxLayerTests.cs: -------------------------------------------------------------------------------- 1 | using ConvNetSharp.Core.Layers.Double; 2 | using ConvNetSharp.Volume; 3 | using ConvNetSharp.Volume.Double; 4 | using NUnit.Framework; 5 | 6 | namespace ConvNetSharp.Core.Tests 7 | { 8 | [TestFixture] 9 | public class SoftmaxLayerTests 10 | { 11 | private readonly SoftmaxLayer layer; 12 | private readonly Volume input; 13 | 14 | public VolumeBuilder Volume => BuilderInstance.Volume; 15 | 16 | public SoftmaxLayerTests() 17 | { 18 | this.layer = new SoftmaxLayer(4); 19 | this.layer.Init(1, 1, 4); 20 | 21 | this.input = this.Volume.From(new[] 22 | { 23 | 0.1, 0.1, 0.1, 0.1, 24 | 1000, 2000, 3000, 4000, 25 | 0, 0, 0, 0 26 | }, new Shape(1, 1, 4, 3)); 27 | } 28 | 29 | [Test] 30 | public void OutputIsNormalized() 31 | { 32 | var output = this.layer.DoForward(this.input, true); 33 | Assert.AreEqual(1, output.Shape.Dimensions[0]); 34 | Assert.AreEqual(1, output.Shape.Dimensions[1]); 35 | Assert.AreEqual(4, output.Shape.Dimensions[2]); 36 | Assert.AreEqual(3, output.Shape.Dimensions[3]); 37 | 38 | var values = output.ToArray(); 39 | Assert.AreEqual(0.25, values[0]); 40 | Assert.AreEqual(0.25, values[1]); 41 | Assert.AreEqual(0.25, values[2]); 42 | Assert.AreEqual(0.25, values[3]); 43 | 44 | Assert.AreEqual(0, values[4]); 45 | Assert.AreEqual(0, values[5]); 46 | Assert.AreEqual(0, values[6]); 47 | Assert.AreEqual(1, values[7]); 48 | 49 | Assert.AreEqual(0.25, values[8]); 50 | Assert.AreEqual(0.25, values[9]); 51 | Assert.AreEqual(0.25, values[10]); 52 | Assert.AreEqual(0.25, values[11]); 53 | } 54 | 55 | [Test] 56 | public void StorageIsReusedIfPossible() 57 | { 58 | var output1 = this.layer.DoForward(this.input, true); 59 | var output2 = this.layer.DoForward(this.input, true); 60 | Assert.AreSame(output1, output2, "Storage is reused if possible."); 61 | } 62 | } 63 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/ConvNetSharp.Core.Nuget.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netstandard2.1 5 | 6 | 7 | 8 | 0.4.14 9 | 0.4.14.0 10 | Cedric Bovar 11 | Deep learning in C# 12 | Copyright (c) 2020 13 | 14 | 15 | 16 | Cognitio.ConvNetSharp.Core 17 | 0.4.14 18 | false 19 | Deep Learning Convolution AI 20 | https://github.com/cbovar/ConvNetSharp/blob/master/LICENSE 21 | https://github.com/cbovar/ConvNetSharp 22 | ConvNetSharp.Core 23 | ConvNetSharp.Core 24 | https://github.com/cbovar/ConvNetSharp/releases/tag/v0.4.14 25 | True 26 | 0.4.14.0 27 | icon.png 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/ConvNetSharp.Core.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netstandard2.1 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/INet.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using ConvNetSharp.Core.Layers; 4 | using ConvNetSharp.Volume; 5 | 6 | namespace ConvNetSharp.Core 7 | { 8 | public interface INet where T : struct, IEquatable, IFormattable 9 | { 10 | T Backward(Volume y); 11 | 12 | Volume Forward(Volume input, bool isTraining = false); 13 | 14 | T GetCostLoss(Volume input, Volume y); 15 | 16 | List> GetParametersAndGradients(); 17 | 18 | int[] GetPrediction(); 19 | } 20 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/Layers/Double/ConvLayer.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace ConvNetSharp.Core.Layers.Double 4 | { 5 | public class ConvLayer : ConvLayer 6 | { 7 | public ConvLayer(Dictionary data) : base(data) 8 | { 9 | } 10 | 11 | public ConvLayer(int width, int height, int filterCount) : base(width, height, filterCount) 12 | { 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/Layers/Double/DropoutLayer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | 4 | namespace ConvNetSharp.Core.Layers.Double 5 | { 6 | public class DropoutLayer : DropoutLayer 7 | { 8 | public DropoutLayer(double dropoutProbability) : base(dropoutProbability) 9 | { 10 | } 11 | 12 | public DropoutLayer(Dictionary data) : base(data) 13 | { 14 | this.DropProbability = Convert.ToDouble(data["DropProbability"]); 15 | } 16 | } 17 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/Layers/Double/FullyConnLayer.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace ConvNetSharp.Core.Layers.Double 4 | { 5 | public class FullyConnLayer : FullyConnLayer 6 | { 7 | public FullyConnLayer(Dictionary data) : base(data) 8 | { 9 | } 10 | 11 | public FullyConnLayer(int neuronCount) : base(neuronCount) 12 | { 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/Layers/Double/InputLayer.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace ConvNetSharp.Core.Layers.Double 4 | { 5 | public class InputLayer : InputLayer 6 | { 7 | public InputLayer(Dictionary data) : base(data) 8 | { 9 | } 10 | 11 | public InputLayer(int inputWidth, int inputHeight, int inputDepth) : base(inputWidth, inputHeight, inputDepth) 12 | { 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/Layers/Double/LeakyReluLayer.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace ConvNetSharp.Core.Layers.Double 4 | { 5 | public class LeakyReluLayer : LeakyReluLayer 6 | { 7 | public LeakyReluLayer(double alpha) : base(alpha) 8 | { 9 | } 10 | 11 | public LeakyReluLayer(Dictionary data) : base(data) 12 | { 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/Layers/Double/PoolLayer.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace ConvNetSharp.Core.Layers.Double 4 | { 5 | public class PoolLayer : PoolLayer 6 | { 7 | public PoolLayer(Dictionary data) : base(data) 8 | { 9 | } 10 | 11 | public PoolLayer(int width, int height) : base(width, height) 12 | { 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/Layers/Double/RegressionLayer.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace ConvNetSharp.Core.Layers.Double 4 | { 5 | public class RegressionLayer : RegressionLayer 6 | { 7 | public RegressionLayer(Dictionary data) : base(data) 8 | { 9 | } 10 | 11 | public RegressionLayer() 12 | { 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/Layers/Double/ReluLayer.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace ConvNetSharp.Core.Layers.Double 4 | { 5 | public class ReluLayer : ReluLayer 6 | { 7 | public ReluLayer() 8 | { 9 | } 10 | 11 | public ReluLayer(Dictionary data) : base(data) 12 | { 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/Layers/Double/SigmoidLayer.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace ConvNetSharp.Core.Layers.Double 4 | { 5 | public class SigmoidLayer : SigmoidLayer 6 | { 7 | public SigmoidLayer(Dictionary data) : base(data) 8 | { 9 | } 10 | 11 | public SigmoidLayer() 12 | { 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/Layers/Double/SoftmaxLayer.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace ConvNetSharp.Core.Layers.Double 4 | { 5 | public class SoftmaxLayer : SoftmaxLayer 6 | { 7 | public SoftmaxLayer(Dictionary data) : base(data) 8 | { 9 | } 10 | 11 | public SoftmaxLayer(int classCount) : base(classCount) 12 | { 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/Layers/Double/TanhLayer.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace ConvNetSharp.Core.Layers.Double 4 | { 5 | public class TanhLayer : TanhLayer 6 | { 7 | public TanhLayer() 8 | { 9 | } 10 | 11 | public TanhLayer(Dictionary data) : base(data) 12 | { 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/Layers/DropoutLayer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using ConvNetSharp.Volume; 4 | 5 | namespace ConvNetSharp.Core.Layers 6 | { 7 | public class DropoutLayer : LayerBase where T : struct, IEquatable, IFormattable 8 | { 9 | public DropoutLayer(Dictionary data) : base(data) 10 | { 11 | this.DropProbability = (T)Convert.ChangeType(data["DropProbability"], typeof(T)); 12 | } 13 | 14 | public DropoutLayer(T dropProbability) 15 | { 16 | this.DropProbability = dropProbability; 17 | } 18 | 19 | public T DropProbability { get; set; } 20 | 21 | public override void Backward(Volume outputGradient) 22 | { 23 | this.OutputActivationGradients = outputGradient; 24 | 25 | this.InputActivationGradients.Clear(); 26 | 27 | this.OutputActivation.DropoutGradient(this.InputActivation, this.OutputActivationGradients, this.DropProbability, this.InputActivationGradients); 28 | } 29 | 30 | protected override Volume Forward(Volume input, bool isTraining = false) 31 | { 32 | input.Dropout(isTraining ? this.DropProbability : Ops.Zero, this.OutputActivation); 33 | return this.OutputActivation; 34 | } 35 | 36 | public override Dictionary GetData() 37 | { 38 | var dico = base.GetData(); 39 | dico["DropProbability"] = this.DropProbability; 40 | return dico; 41 | } 42 | 43 | public override void Init(int inputWidth, int inputHeight, int inputDepth) 44 | { 45 | base.Init(inputWidth, inputHeight, inputDepth); 46 | 47 | this.OutputDepth = inputDepth; 48 | this.OutputWidth = inputWidth; 49 | this.OutputHeight = inputHeight; 50 | } 51 | } 52 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/Layers/IClassificationLayer.cs: -------------------------------------------------------------------------------- 1 | namespace ConvNetSharp.Core.Layers 2 | { 3 | public interface IClassificationLayer 4 | { 5 | int ClassCount { get; set; } 6 | } 7 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/Layers/IDotProductLayer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace ConvNetSharp.Core.Layers 4 | { 5 | public interface IDotProductLayer where T : struct, IEquatable, IFormattable 6 | { 7 | T BiasPref { get; set; } 8 | } 9 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/Layers/ILastLayer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using ConvNetSharp.Volume; 3 | 4 | namespace ConvNetSharp.Core.Layers 5 | { 6 | public interface ILastLayer where T : struct, IEquatable, IFormattable 7 | { 8 | void Backward(Volume y, out T loss); 9 | } 10 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/Layers/InputLayer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using ConvNetSharp.Volume; 4 | 5 | namespace ConvNetSharp.Core.Layers 6 | { 7 | public class InputLayer : LayerBase where T : struct, IEquatable, IFormattable 8 | { 9 | public InputLayer(Dictionary data) : base(data) 10 | { 11 | this.OutputWidth = this.InputWidth; 12 | this.OutputHeight = this.InputHeight; 13 | this.OutputDepth = this.InputDepth; 14 | } 15 | 16 | public InputLayer(int inputWidth, int inputHeight, int inputDepth) 17 | { 18 | Init(inputWidth, inputHeight, inputDepth); 19 | 20 | this.OutputWidth = inputWidth; 21 | this.OutputHeight = inputHeight; 22 | this.OutputDepth = inputDepth; 23 | } 24 | 25 | public override void Backward(Volume outputGradient) 26 | { 27 | } 28 | 29 | protected override Volume Forward(Volume input, bool isTraining = false) 30 | { 31 | this.OutputActivation = input; 32 | return this.OutputActivation; 33 | } 34 | 35 | public override Volume Forward(bool isTraining) 36 | { 37 | return this.OutputActivation; 38 | } 39 | } 40 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/Layers/LastLayerBase.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using ConvNetSharp.Volume; 4 | 5 | namespace ConvNetSharp.Core.Layers 6 | { 7 | public abstract class LastLayerBase : LayerBase, ILastLayer where T : struct, IEquatable, IFormattable 8 | { 9 | protected LastLayerBase() 10 | { 11 | } 12 | 13 | protected LastLayerBase(Dictionary data) : base(data) 14 | { 15 | } 16 | 17 | public abstract void Backward(Volume y, out T loss); 18 | } 19 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/Layers/LeakyReluLayer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using ConvNetSharp.Volume; 4 | 5 | namespace ConvNetSharp.Core.Layers 6 | { 7 | /// 8 | /// Implements LeakyReLU nonlinearity elementwise 9 | /// x -> x > 0, x, otherwise alpha * x 10 | /// 11 | public class LeakyReluLayer : LayerBase where T : struct, IEquatable, IFormattable 12 | { 13 | public LeakyReluLayer(T alpha) 14 | { 15 | this.Alpha = alpha; 16 | } 17 | 18 | public LeakyReluLayer(Dictionary data) : base(data) 19 | { 20 | this.Alpha = (T)Convert.ChangeType(data["Alpha"], typeof(T)); 21 | } 22 | 23 | public T Alpha { get; set; } 24 | 25 | public override Dictionary GetData() 26 | { 27 | var dico = base.GetData(); 28 | 29 | dico["Alpha"] = this.Alpha; 30 | 31 | return dico; 32 | } 33 | 34 | public override void Backward(Volume outputGradient) 35 | { 36 | this.OutputActivationGradients = outputGradient; 37 | this.OutputActivation.LeakyReluGradient(this.OutputActivationGradients, this.InputActivationGradients, this.Alpha); 38 | } 39 | 40 | protected override Volume Forward(Volume input, bool isTraining = false) 41 | { 42 | input.LeakyRelu(this.Alpha, this.OutputActivation); 43 | return this.OutputActivation; 44 | } 45 | 46 | public override void Init(int inputWidth, int inputHeight, int inputDepth) 47 | { 48 | base.Init(inputWidth, inputHeight, inputDepth); 49 | 50 | this.OutputDepth = inputDepth; 51 | this.OutputWidth = inputWidth; 52 | this.OutputHeight = inputHeight; 53 | } 54 | } 55 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/Layers/ParametersAndGradients.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using ConvNetSharp.Volume; 3 | 4 | namespace ConvNetSharp.Core.Layers 5 | { 6 | public class ParametersAndGradients where T : struct, IEquatable, IFormattable 7 | { 8 | public Volume Volume { get; set; } 9 | 10 | public Volume Gradient { get; set; } 11 | } 12 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/Layers/RegressionLayer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using ConvNetSharp.Volume; 4 | 5 | namespace ConvNetSharp.Core.Layers 6 | { 7 | /// 8 | /// implements an L2 regression cost layer, 9 | /// so penalizes \sum_i(||x_i - y_i||^2), where x is its input 10 | /// and y is the user-provided array of "correct" values. 11 | /// Input should have a shape of [1, 1, 1, n] where n is the batch size 12 | /// 13 | public class RegressionLayer : LastLayerBase where T : struct, IEquatable, IFormattable 14 | { 15 | private Volume _result; 16 | private Volume _sum; 17 | 18 | public RegressionLayer() 19 | { 20 | } 21 | 22 | public RegressionLayer(Dictionary data) : base(data) 23 | { 24 | } 25 | 26 | public override void Init(int inputWidth, int inputHeight, int inputDepth) 27 | { 28 | base.Init(inputWidth, inputHeight, inputDepth); 29 | 30 | this.OutputWidth = inputWidth; 31 | this.OutputHeight = inputHeight; 32 | this.OutputDepth = inputDepth; 33 | } 34 | 35 | public override void Backward(Volume outputGradient) 36 | { 37 | throw new NotImplementedException(); 38 | } 39 | 40 | public override void Backward(Volume y, out T loss) 41 | { 42 | var reshape = y.ReShape(new Shape(1, 1, -1, Shape.Keep)); 43 | var dy = this.InputActivationGradients.ReShape(this.OutputActivation.Shape.Dimensions); 44 | reshape.SubtractFrom(this.OutputActivation, dy); 45 | 46 | if (this._result == null) 47 | { 48 | this._result = BuilderInstance.Volume.SameAs(this.OutputActivation.Shape); 49 | this._sum = BuilderInstance.Volume.SameAs(new Shape(1)); 50 | } 51 | 52 | this._sum.Clear(); 53 | dy.Multiply(dy, this._result); // dy * dy 54 | var half = (T)Convert.ChangeType(0.5, typeof(T)); 55 | this._result.Multiply(half, this._result); // dy * dy * 0.5 56 | this._result.Sum(this._sum); // sum over all batch 57 | var batchSize = y.Shape.Dimensions[3]; 58 | loss = Ops.Divide(this._sum.Get(0), Ops.Cast(batchSize)); // average 59 | } 60 | 61 | protected override Volume Forward(Volume input, bool isTraining = false) 62 | { 63 | return input; 64 | } 65 | } 66 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/Layers/ReluLayer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using ConvNetSharp.Volume; 4 | 5 | namespace ConvNetSharp.Core.Layers 6 | { 7 | /// 8 | /// Implements ReLU nonlinearity elementwise 9 | /// x -> max(0, x) 10 | /// the output is in [0, inf) 11 | /// 12 | public class ReluLayer : LayerBase where T : struct, IEquatable, IFormattable 13 | { 14 | public ReluLayer() 15 | { 16 | } 17 | 18 | public ReluLayer(Dictionary data) : base(data) 19 | { 20 | } 21 | 22 | public override void Backward(Volume outputGradient) 23 | { 24 | this.OutputActivationGradients = outputGradient; 25 | 26 | this.OutputActivation.ReluGradient(this.InputActivation, 27 | this.OutputActivationGradients, 28 | this.InputActivationGradients); 29 | } 30 | 31 | protected override Volume Forward(Volume input, bool isTraining = false) 32 | { 33 | input.Relu(this.OutputActivation); 34 | return this.OutputActivation; 35 | } 36 | 37 | public override void Init(int inputWidth, int inputHeight, int inputDepth) 38 | { 39 | base.Init(inputWidth, inputHeight, inputDepth); 40 | 41 | this.OutputDepth = inputDepth; 42 | this.OutputWidth = inputWidth; 43 | this.OutputHeight = inputHeight; 44 | } 45 | } 46 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/Layers/SigmoidLayer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using ConvNetSharp.Volume; 4 | 5 | namespace ConvNetSharp.Core.Layers 6 | { 7 | public class SigmoidLayer : LayerBase where T : struct, IEquatable, IFormattable 8 | { 9 | public SigmoidLayer(Dictionary data) : base(data) 10 | { 11 | } 12 | 13 | public SigmoidLayer() 14 | { 15 | } 16 | 17 | public override void Backward(Volume outputGradient) 18 | { 19 | this.OutputActivationGradients = outputGradient; 20 | this.OutputActivation.SigmoidGradient(this.InputActivation, this.OutputActivationGradients, this.InputActivationGradients); 21 | } 22 | 23 | protected override Volume Forward(Volume input, bool isTraining = false) 24 | { 25 | input.Sigmoid(this.OutputActivation); 26 | return this.OutputActivation; 27 | } 28 | 29 | public override void Init(int inputWidth, int inputHeight, int inputDepth) 30 | { 31 | base.Init(inputWidth, inputHeight, inputDepth); 32 | 33 | this.OutputDepth = inputDepth; 34 | this.OutputWidth = inputWidth; 35 | this.OutputHeight = inputHeight; 36 | } 37 | } 38 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/Layers/Single/ConvLayer.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace ConvNetSharp.Core.Layers.Single 4 | { 5 | public class ConvLayer : ConvLayer 6 | { 7 | public ConvLayer(Dictionary data) : base(data) 8 | { 9 | } 10 | 11 | public ConvLayer(int width, int height, int filterCount) : base(width, height, filterCount) 12 | { 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/Layers/Single/DropoutLayer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | 4 | namespace ConvNetSharp.Core.Layers.Single 5 | { 6 | public class DropoutLayer : DropoutLayer 7 | { 8 | public DropoutLayer(float dropoutProbability) : base(dropoutProbability) 9 | { 10 | } 11 | 12 | public DropoutLayer(Dictionary data) : base(data) 13 | { 14 | this.DropProbability = Convert.ToSingle(data["DropProbability"]); 15 | } 16 | } 17 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/Layers/Single/FullyConnLayer.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace ConvNetSharp.Core.Layers.Single 4 | { 5 | public class FullyConnLayer : FullyConnLayer 6 | { 7 | public FullyConnLayer(Dictionary data) : base(data) 8 | { 9 | } 10 | 11 | public FullyConnLayer(int neuronCount) : base(neuronCount) 12 | { 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/Layers/Single/InputLayer.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace ConvNetSharp.Core.Layers.Single 4 | { 5 | public class InputLayer : InputLayer 6 | { 7 | public InputLayer(Dictionary data) : base(data) 8 | { 9 | } 10 | 11 | public InputLayer(int inputWidth, int inputHeight, int inputDepth) : base(inputWidth, inputHeight, inputDepth) 12 | { 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/Layers/Single/LeakyReluLayer.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace ConvNetSharp.Core.Layers.Single 4 | { 5 | public class LeakyReluLayer : LeakyReluLayer 6 | { 7 | public LeakyReluLayer(float alpha) : base(alpha) 8 | { 9 | } 10 | 11 | public LeakyReluLayer(Dictionary data) : base(data) 12 | { 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/Layers/Single/PoolLayer.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace ConvNetSharp.Core.Layers.Single 4 | { 5 | public class PoolLayer : PoolLayer 6 | { 7 | public PoolLayer(Dictionary data) : base(data) 8 | { 9 | } 10 | 11 | public PoolLayer(int width, int height) : base(width, height) 12 | { 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/Layers/Single/RegressionLayer.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace ConvNetSharp.Core.Layers.Single 4 | { 5 | public class RegressionLayer : RegressionLayer 6 | { 7 | public RegressionLayer(Dictionary data) : base(data) 8 | { 9 | } 10 | 11 | public RegressionLayer() 12 | { 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/Layers/Single/ReluLayer.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace ConvNetSharp.Core.Layers.Single 4 | { 5 | public class ReluLayer : ReluLayer 6 | { 7 | public ReluLayer() 8 | { 9 | 10 | } 11 | 12 | public ReluLayer(Dictionary data) : base(data) 13 | { 14 | } 15 | } 16 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/Layers/Single/SigmoidLayer.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace ConvNetSharp.Core.Layers.Single 4 | { 5 | public class SigmoidLayer : SigmoidLayer 6 | { 7 | public SigmoidLayer(Dictionary data) : base(data) 8 | { 9 | } 10 | 11 | public SigmoidLayer() 12 | { 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/Layers/Single/SoftmaxLayer.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace ConvNetSharp.Core.Layers.Single 4 | { 5 | public class SoftmaxLayer : SoftmaxLayer 6 | { 7 | public SoftmaxLayer(Dictionary data) : base(data) 8 | { 9 | } 10 | 11 | public SoftmaxLayer(int classCount) : base(classCount) 12 | { 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/Layers/Single/TanhLayer.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace ConvNetSharp.Core.Layers.Single 4 | { 5 | public class TanhLayer : TanhLayer 6 | { 7 | public TanhLayer() 8 | { 9 | } 10 | 11 | public TanhLayer(Dictionary data) : base(data) 12 | { 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/Layers/TanhLayer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using ConvNetSharp.Volume; 4 | 5 | namespace ConvNetSharp.Core.Layers 6 | { 7 | public class TanhLayer : LayerBase where T : struct, IEquatable, IFormattable 8 | { 9 | public TanhLayer(Dictionary data) : base(data) 10 | { 11 | } 12 | 13 | public TanhLayer() 14 | { 15 | } 16 | 17 | public override void Backward(Volume outputGradient) 18 | { 19 | this.OutputActivationGradients = outputGradient; 20 | this.OutputActivation.TanhGradient(this.InputActivation, this.OutputActivationGradients, this.InputActivationGradients); 21 | } 22 | 23 | protected override Volume Forward(Volume input, bool isTraining = false) 24 | { 25 | input.Tanh(this.OutputActivation); 26 | return this.OutputActivation; 27 | } 28 | 29 | public override void Init(int inputWidth, int inputHeight, int inputDepth) 30 | { 31 | base.Init(inputWidth, inputHeight, inputDepth); 32 | 33 | this.OutputDepth = inputDepth; 34 | this.OutputWidth = inputWidth; 35 | this.OutputHeight = inputHeight; 36 | } 37 | } 38 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/Serialization/JObjectExtensions.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Linq; 3 | using Newtonsoft.Json.Linq; 4 | 5 | namespace ConvNetSharp.Core.Serialization 6 | { 7 | public static class JObjectExtensions 8 | { 9 | /// 10 | /// JObject to nested dictionaries 11 | /// 12 | /// 13 | /// 14 | public static Dictionary ToDictionary(this JObject jobject) 15 | { 16 | var dico = new Dictionary(); 17 | 18 | foreach (var o in jobject) 19 | { 20 | var oValue = o.Value; 21 | 22 | if (o.Value is JArray jArray) 23 | { 24 | var first = jArray[0]; // use first element to guess if we are dealing with a list of dico or an array 25 | var isValueArray = first is JValue; 26 | 27 | if (isValueArray) 28 | { 29 | var array = jArray.Values().Select(x => ((JValue) x).Value).ToArray(); 30 | dico[o.Key] = array; 31 | } 32 | else 33 | { 34 | var list = new List>(); 35 | foreach (var token in jArray) 36 | { 37 | var elt = ((JObject) token).ToDictionary(); 38 | list.Add(elt); 39 | } 40 | 41 | dico[o.Key] = list; 42 | } 43 | } 44 | else 45 | { 46 | dico[o.Key] = ((JValue) oValue).Value; 47 | } 48 | } 49 | 50 | return dico; 51 | } 52 | } 53 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/Serialization/SerializationExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Linq; 3 | using ConvNetSharp.Core.Fluent; 4 | using Newtonsoft.Json; 5 | using Newtonsoft.Json.Linq; 6 | 7 | namespace ConvNetSharp.Core.Serialization 8 | { 9 | public static class SerializationExtensions 10 | { 11 | public static Net FromJson(string json) where T : struct, IEquatable, IFormattable 12 | { 13 | var data = JsonConvert.DeserializeObject(json); 14 | var dico = data.ToDictionary(); 15 | var net = Net.FromData(dico); 16 | return net; 17 | } 18 | 19 | public static T[] ToArrayOfT(this object obj) 20 | { 21 | if (obj is T[] arrayofT) 22 | { 23 | return arrayofT; 24 | } 25 | 26 | if (obj is JArray jarray) 27 | { 28 | return jarray.ToObject(); 29 | } 30 | 31 | return ((object[]) obj).Select(o => (T) Convert.ChangeType(o, typeof(T), null)).ToArray(); 32 | } 33 | 34 | public static string ToJson(this Net net) where T : struct, IEquatable, IFormattable 35 | { 36 | var data = net.GetData(); 37 | var json = JsonConvert.SerializeObject(data); 38 | return json; 39 | } 40 | 41 | public static string ToJson(this FluentNet net) where T : struct, IEquatable, IFormattable 42 | { 43 | var data = net.GetData(); 44 | var json = JsonConvert.SerializeObject(data); 45 | return json; 46 | } 47 | } 48 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/Training/Double/AdamTrainer.cs: -------------------------------------------------------------------------------- 1 | namespace ConvNetSharp.Core.Training.Double 2 | { 3 | public class AdamTrainer : AdamTrainer 4 | { 5 | public AdamTrainer(INet net) : base(net) 6 | { 7 | } 8 | } 9 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/Training/Double/SgdTrainer.cs: -------------------------------------------------------------------------------- 1 | namespace ConvNetSharp.Core.Training.Double 2 | { 3 | public class SgdTrainer : SgdTrainer 4 | { 5 | public SgdTrainer(INet net) : base(net) 6 | { 7 | } 8 | } 9 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/Training/Single/AdamTrainer.cs: -------------------------------------------------------------------------------- 1 | namespace ConvNetSharp.Core.Training.Single 2 | { 3 | public class AdamTrainer : AdamTrainer 4 | { 5 | public AdamTrainer(INet net) : base(net) 6 | { 7 | } 8 | } 9 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/Training/Single/SgdTrainer.cs: -------------------------------------------------------------------------------- 1 | namespace ConvNetSharp.Core.Training.Single 2 | { 3 | public class SgdTrainer : SgdTrainer 4 | { 5 | public SgdTrainer(INet net) : base(net) 6 | { 7 | } 8 | } 9 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/Training/TrainerBase.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Diagnostics; 3 | using ConvNetSharp.Volume; 4 | 5 | namespace ConvNetSharp.Core.Training 6 | { 7 | public abstract class TrainerBase where T : struct, IEquatable, IFormattable 8 | { 9 | protected TrainerBase(INet net) 10 | { 11 | this.Net = net; 12 | } 13 | 14 | public INet Net { get; } 15 | 16 | public double BackwardTimeMs { get; protected set; } 17 | 18 | public double ForwardTimeMs { get; protected set; } 19 | 20 | public double UpdateWeightsTimeMs { get; private set; } 21 | 22 | public virtual T Loss { get; protected set; } 23 | 24 | public int BatchSize { get; set; } = 1; 25 | 26 | protected virtual void Backward(Volume y) 27 | { 28 | var chrono = Stopwatch.StartNew(); 29 | 30 | var batchSize = y.Shape.Dimensions[3]; 31 | this.Loss = Ops.Divide(this.Net.Backward(y), Ops.Cast(batchSize)); 32 | this.BackwardTimeMs = chrono.Elapsed.TotalMilliseconds / batchSize; 33 | } 34 | 35 | private void Forward(Volume x) 36 | { 37 | var chrono = Stopwatch.StartNew(); 38 | var batchSize = x.Shape.Dimensions[3]; 39 | this.Net.Forward(x, true); // also set the flag that lets the net know we're just training 40 | this.ForwardTimeMs = chrono.Elapsed.TotalMilliseconds / batchSize; 41 | } 42 | 43 | public virtual void Train(Volume x, Volume y) 44 | { 45 | this.Forward(x); 46 | 47 | this.Backward(y); 48 | 49 | var batchSize = x.Shape.Dimensions[3]; 50 | var chrono = Stopwatch.StartNew(); 51 | this.TrainImplem(); 52 | this.UpdateWeightsTimeMs = chrono.Elapsed.TotalMilliseconds / batchSize; 53 | } 54 | 55 | protected abstract void TrainImplem(); 56 | } 57 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Core/img/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbovar/ConvNetSharp/ccaf6f6959c12026c4e1a0e396a7711654c92551/src/ConvNetSharp.Core/img/icon.png -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow.Tests/ConvNetSharp.Flow.Tests.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netcoreapp3.1 5 | false 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | all 15 | runtime; build; native; contentfiles; analyzers; buildtransitive 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow.Tests/DoubleGpuOpTests.cs: -------------------------------------------------------------------------------- 1 | using ConvNetSharp.Flow.Ops; 2 | using ConvNetSharp.Volume; 3 | using ConvNetSharp.Volume.GPU.Double; 4 | using NUnit.Framework; 5 | 6 | namespace ConvNetSharp.Flow.Tests 7 | { 8 | [TestFixture] 9 | public class DoubleGpuOpTests : OpTests 10 | { 11 | public DoubleGpuOpTests() 12 | { 13 | Op.Count = 1; 14 | BuilderInstance.Volume = new VolumeBuilder(); 15 | } 16 | 17 | protected override Volume NewVolume(double[] values, Shape shape) 18 | { 19 | return new Volume.GPU.Double.Volume(values, shape); 20 | } 21 | } 22 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow.Tests/DoubleOpTests.cs: -------------------------------------------------------------------------------- 1 | using ConvNetSharp.Flow.Ops; 2 | using ConvNetSharp.Volume; 3 | using ConvNetSharp.Volume.Double; 4 | using NUnit.Framework; 5 | 6 | namespace ConvNetSharp.Flow.Tests 7 | { 8 | [TestFixture] 9 | public class DoubleOpTests : OpTests 10 | { 11 | public DoubleOpTests() 12 | { 13 | Op.Count = 1; 14 | BuilderInstance.Volume = new VolumeBuilder(); 15 | } 16 | 17 | protected override Volume NewVolume(double[] values, Shape shape) 18 | { 19 | return BuilderInstance.Volume.From(values, shape); 20 | } 21 | } 22 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow.Tests/SingleGpuOpTests.cs: -------------------------------------------------------------------------------- 1 | using System.Linq; 2 | using ConvNetSharp.Flow.Ops; 3 | using ConvNetSharp.Volume; 4 | using ConvNetSharp.Volume.GPU.Single; 5 | using NUnit.Framework; 6 | 7 | namespace ConvNetSharp.Flow.Tests 8 | { 9 | [TestFixture] 10 | public class SingleGpuOpTests : OpTests 11 | { 12 | public SingleGpuOpTests() 13 | { 14 | Op.Count = 1; 15 | BuilderInstance.Volume = new VolumeBuilder(); 16 | } 17 | 18 | protected override Volume NewVolume(double[] values, Shape shape) 19 | { 20 | var converted = values.Select(i => (float)i).ToArray(); 21 | return BuilderInstance.Volume.From(converted, shape); 22 | } 23 | } 24 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow.Tests/SingleOpTests.cs: -------------------------------------------------------------------------------- 1 | using System.Linq; 2 | using ConvNetSharp.Flow.Ops; 3 | using ConvNetSharp.Volume; 4 | using ConvNetSharp.Volume.Single; 5 | using NUnit.Framework; 6 | 7 | namespace ConvNetSharp.Flow.Tests 8 | { 9 | [TestFixture] 10 | public class SingleOpTests : OpTests 11 | { 12 | public SingleOpTests() 13 | { 14 | Op.Count = 1; 15 | BuilderInstance.Volume = new VolumeBuilder(); 16 | } 17 | 18 | protected override Volume NewVolume(double[] values, Shape shape) 19 | { 20 | var converted = values.Select(i => (float)i).ToArray(); 21 | return BuilderInstance.Volume.From(converted, shape); 22 | } 23 | } 24 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/ConvNetSharp.Flow.Nuget.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netstandard2.1 5 | 6 | 7 | 8 | 0.4.14 9 | 0.4.14.0 10 | Cedric Bovar 11 | Deep learning in C# 12 | Copyright (c) 2020 13 | 14 | 15 | 16 | Cognitio.ConvNetSharp.Flow 17 | 0.4.14 18 | false 19 | Deep Learning Convolution AI 20 | https://github.com/cbovar/ConvNetSharp/blob/master/LICENSE 21 | https://github.com/cbovar/ConvNetSharp 22 | ConvNetSharp.Flow 23 | ConvNetSharp.Flow 24 | https://github.com/cbovar/ConvNetSharp/releases/tag/v0.4.14 25 | True 26 | ConvNetSharp.Flow 27 | 0.4.14.0 28 | icon.png 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/ConvNetSharp.Flow.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netstandard2.1 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Graph/DifferentiateVisitor.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using ConvNetSharp.Flow.Ops; 3 | 4 | namespace ConvNetSharp.Flow.Graph 5 | { 6 | internal class DifferentiateVisitor : IOpVisitor where T : struct, IEquatable, IFormattable 7 | { 8 | public void Visit(Op op) 9 | { 10 | if (op.Derivate != null) 11 | { 12 | op.Differentiate(); 13 | } 14 | 15 | foreach (var parent in op.Parents) 16 | { 17 | var diff = new DifferentiateVisitor(); 18 | diff.Visit(parent); 19 | } 20 | } 21 | } 22 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Graph/IOpVisitor.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using ConvNetSharp.Flow.Ops; 3 | 4 | namespace ConvNetSharp.Flow.Graph 5 | { 6 | public interface IOpVisitor where T : struct, IEquatable, IFormattable 7 | { 8 | void Visit(Op op); 9 | } 10 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Layers/ConvLayer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using ConvNetSharp.Flow.Ops; 3 | using ConvNetSharp.Volume; 4 | 5 | namespace ConvNetSharp.Flow.Layers 6 | { 7 | public class ConvLayer : LayerBase where T : struct, IEquatable, IFormattable 8 | { 9 | private readonly int _filterCount; 10 | private readonly int _height; 11 | private readonly int _width; 12 | private Variable _bias; 13 | private T _biasPref; 14 | private Convolution _conv; 15 | private bool _initialized; 16 | private int _pad; 17 | private int _stride = 1; 18 | 19 | public ConvLayer(int width, int height, int filterCount) 20 | { 21 | this._width = width; 22 | this._height = height; 23 | this._filterCount = filterCount; 24 | } 25 | 26 | public T BiasPref 27 | { 28 | get => this._biasPref; 29 | set 30 | { 31 | this._biasPref = value; 32 | 33 | if (this._initialized) 34 | { 35 | this._bias.Result = new T[this._filterCount].Populate(this.BiasPref); 36 | } 37 | } 38 | } 39 | 40 | public int Stride 41 | { 42 | get => this._stride; 43 | set 44 | { 45 | this._stride = value; 46 | if (this._initialized) 47 | { 48 | this._conv.Stride = value; 49 | } 50 | } 51 | } 52 | 53 | public int Pad 54 | { 55 | get => this._pad; 56 | set 57 | { 58 | this._pad = value; 59 | if (this._initialized) 60 | { 61 | this._conv.Pad = value; 62 | } 63 | } 64 | } 65 | 66 | public Op Filter => this._conv.Filter; 67 | 68 | public override void AcceptParent(LayerBase parent) 69 | { 70 | base.AcceptParent(parent); 71 | 72 | var cns = parent.Op.Graph; 73 | 74 | using (cns.Scope($"ConvLayer_{this.Id}")) 75 | { 76 | var content = new T[this._filterCount].Populate(this.BiasPref); 77 | this._bias = cns.Variable(BuilderInstance.Volume.From(content, new Shape(1, 1, this._filterCount, 1)), "Bias", true); 78 | this._conv = cns.Conv(parent.Op, this._width, this._height, this._filterCount, this.Stride, this.Pad); 79 | this.Op = this._conv + this._bias; 80 | } 81 | 82 | this._initialized = true; 83 | } 84 | } 85 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Layers/DropoutLayer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace ConvNetSharp.Flow.Layers 4 | { 5 | public class DropoutLayer : LayerBase where T : struct, IEquatable, IFormattable 6 | { 7 | private readonly T _dropoutProbability; 8 | 9 | public DropoutLayer(T dropoutProbability) 10 | { 11 | this._dropoutProbability = dropoutProbability; 12 | } 13 | 14 | public override void AcceptParent(LayerBase parent) 15 | { 16 | base.AcceptParent(parent); 17 | this.Op = parent.Op.Graph.Dropout(parent.Op, this._dropoutProbability); 18 | } 19 | } 20 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Layers/FullyConnLayer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using ConvNetSharp.Flow.Ops; 3 | using ConvNetSharp.Volume; 4 | 5 | namespace ConvNetSharp.Flow.Layers 6 | { 7 | public class FullyConnLayer : LayerBase where T : struct, IEquatable, IFormattable 8 | { 9 | private readonly int _neuronCount; 10 | private Variable _bias; 11 | private T _biasPref; 12 | private Dense _conv; 13 | private bool _initialized; 14 | 15 | public FullyConnLayer(int neuronCount) 16 | { 17 | this._neuronCount = neuronCount; 18 | } 19 | 20 | public T BiasPref 21 | { 22 | get => this._biasPref; 23 | set 24 | { 25 | this._biasPref = value; 26 | 27 | if (this._initialized) 28 | { 29 | this._bias.Result = new T[this._neuronCount].Populate(this.BiasPref); 30 | } 31 | } 32 | } 33 | 34 | public Op Filter => this._conv.Filter; 35 | 36 | public override void AcceptParent(LayerBase parent) 37 | { 38 | base.AcceptParent(parent); 39 | 40 | var cns = parent.Op.Graph; 41 | 42 | using (cns.Scope($"FullConnLayer_{this.Id}")) 43 | { 44 | this._bias = cns.Variable(BuilderInstance.Volume.SameAs(new Shape(1, 1, this._neuronCount, 1)), "Bias", true); 45 | this._conv = cns.Dense(cns.Reshape(parent.Op, new Shape(1, 1, -1, Shape.Keep)), this._neuronCount); 46 | this.Op = this._conv + this._bias; 47 | } 48 | 49 | this._initialized = true; 50 | } 51 | } 52 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Layers/ILastLayer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using ConvNetSharp.Flow.Ops; 3 | 4 | namespace ConvNetSharp.Flow.Layers 5 | { 6 | public interface ILastLayer where T : struct, IEquatable, IFormattable 7 | { 8 | Op Cost { get; } 9 | } 10 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Layers/InputLayer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace ConvNetSharp.Flow.Layers 4 | { 5 | public class InputLayer : LayerBase where T : struct, IEquatable, IFormattable 6 | { 7 | public InputLayer() 8 | { 9 | var cns = new ConvNetSharp(); 10 | this.Op = cns.PlaceHolder("input"); 11 | } 12 | } 13 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Layers/LayerBase.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using ConvNetSharp.Flow.Ops; 3 | 4 | namespace ConvNetSharp.Flow.Layers 5 | { 6 | public abstract class LayerBase where T : struct, IEquatable, IFormattable 7 | { 8 | public Op Op { get; set; } 9 | 10 | public int Id { get; private set; } 11 | 12 | public virtual void AcceptParent(LayerBase parent) 13 | { 14 | this.Id = parent.Id + 1; 15 | } 16 | } 17 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Layers/LeakyReluLayer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace ConvNetSharp.Flow.Layers 4 | { 5 | public class LeakyReluLayer : LayerBase where T : struct, IEquatable, IFormattable 6 | { 7 | public LeakyReluLayer(T alpha) 8 | { 9 | this.Alpha = alpha; 10 | } 11 | 12 | public T Alpha { get; set; } 13 | 14 | public override void AcceptParent(LayerBase parent) 15 | { 16 | base.AcceptParent(parent); 17 | this.Op = parent.Op.Graph.LeakyRelu(parent.Op, this.Alpha); 18 | } 19 | } 20 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Layers/OpWrapperLayer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using ConvNetSharp.Flow.Ops; 3 | 4 | namespace ConvNetSharp.Flow.Layers 5 | { 6 | public class OpWrapperLayer : LayerBase, ILastLayer where T : struct, IEquatable, IFormattable 7 | { 8 | public OpWrapperLayer(Op root, Op costOp) 9 | { 10 | this.Op = root; 11 | this.Cost = costOp; 12 | } 13 | 14 | public Op Cost { get; } 15 | } 16 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Layers/PoolLayer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using ConvNetSharp.Flow.Ops; 3 | 4 | namespace ConvNetSharp.Flow.Layers 5 | { 6 | public class PoolLayer : LayerBase where T : struct, IEquatable, IFormattable 7 | { 8 | private readonly int _height; 9 | private readonly int _width; 10 | private bool _initialized; 11 | private int _pad; 12 | private Pool _pool; 13 | private int _stride = 1; 14 | 15 | public PoolLayer(int width, int height) 16 | { 17 | this._width = width; 18 | this._height = height; 19 | } 20 | 21 | public int Stride 22 | { 23 | get => this._stride; 24 | set 25 | { 26 | this._stride = value; 27 | if (this._initialized) 28 | { 29 | this._pool.HorizontalStride = value; 30 | this._pool.VerticalStride = value; 31 | } 32 | } 33 | } 34 | 35 | public int Pad 36 | { 37 | get => this._pad; 38 | set 39 | { 40 | this._pad = value; 41 | if (this._initialized) 42 | { 43 | this._pool.HorizontalPad = value; 44 | this._pool.VerticalPad = value; 45 | } 46 | } 47 | } 48 | 49 | public override void AcceptParent(LayerBase parent) 50 | { 51 | base.AcceptParent(parent); 52 | this._pool = parent.Op.Graph.Pool(parent.Op, this._width, this._height, this.Pad, this.Pad, this.Stride, this.Stride); 53 | this.Op = this._pool; 54 | 55 | this._initialized = true; 56 | } 57 | } 58 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Layers/ReluLayer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace ConvNetSharp.Flow.Layers 4 | { 5 | public class ReluLayer : LayerBase where T : struct, IEquatable, IFormattable 6 | { 7 | public override void AcceptParent(LayerBase parent) 8 | { 9 | base.AcceptParent(parent); 10 | this.Op = parent.Op.Graph.Relu(parent.Op); 11 | } 12 | } 13 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Layers/SigmoidLayer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace ConvNetSharp.Flow.Layers 4 | { 5 | public class SigmoidLayer : LayerBase where T : struct, IEquatable, IFormattable 6 | { 7 | public override void AcceptParent(LayerBase parent) 8 | { 9 | base.AcceptParent(parent); 10 | this.Op = parent.Op.Graph.Sigmoid(parent.Op); 11 | } 12 | } 13 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Layers/SoftMaxLayer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using ConvNetSharp.Flow.Ops; 3 | 4 | namespace ConvNetSharp.Flow.Layers 5 | { 6 | public class SoftmaxLayer : LayerBase, ILastLayer where T : struct, IEquatable, IFormattable 7 | { 8 | public PlaceHolder Y { get; private set; } 9 | 10 | public Op Cost { get; private set; } 11 | 12 | public override void AcceptParent(LayerBase parent) 13 | { 14 | base.AcceptParent(parent); 15 | 16 | var cns = parent.Op.Graph; 17 | this.Y = cns.PlaceHolder("Y"); 18 | this.Op = cns.Softmax(parent.Op); 19 | this.Cost = cns.CrossEntropyLoss(this.Op, this.Y); 20 | } 21 | } 22 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Layers/TanhLayer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace ConvNetSharp.Flow.Layers 4 | { 5 | public class TanhLayer : LayerBase where T : struct, IEquatable, IFormattable 6 | { 7 | public override void AcceptParent(LayerBase parent) 8 | { 9 | base.AcceptParent(parent); 10 | this.Op = parent.Op.Graph.Tanh(parent.Op); 11 | } 12 | } 13 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Ops/Activation.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using ConvNetSharp.Volume; 4 | 5 | namespace ConvNetSharp.Flow.Ops 6 | { 7 | /// 8 | /// y = f(x) where f can be Sigmoid ,Relu,Tanh or ClippedRelu 9 | /// 10 | /// 11 | public class Activation : Op where T : struct, IEquatable, IFormattable 12 | { 13 | public Activation(ConvNetSharp graph, Dictionary data) : base(graph) 14 | { 15 | Enum.TryParse((string)data["ActivationType"], out ActivationType result); 16 | 17 | this.Type = result; 18 | } 19 | 20 | public Activation(ConvNetSharp graph, Op x, ActivationType type) : base(graph) 21 | { 22 | this.AddParent(x); 23 | this.Type = type; 24 | } 25 | 26 | public ActivationType Type { get; set; } 27 | 28 | public override string Representation => $"{this.Type}"; 29 | 30 | public override void Differentiate() 31 | { 32 | this.Parents[0].RegisterDerivate(new ActivationGradient(this.Graph, this.Parents[0], this, this.Derivate, this.Type)); 33 | } 34 | 35 | protected override void Dispose(bool disposing) 36 | { 37 | if (disposing) 38 | { 39 | this.Result?.Dispose(); 40 | } 41 | 42 | base.Dispose(disposing); 43 | } 44 | 45 | public override Volume Evaluate(Session session) 46 | { 47 | if (!this.IsDirty) 48 | { 49 | return base.Evaluate(session); 50 | } 51 | 52 | this.IsDirty = false; 53 | 54 | var x = this.Parents[0].Evaluate(session); 55 | 56 | if (this.Result == null || !Equals(this.Result.Shape, x.Shape)) 57 | { 58 | this.Result?.Dispose(); 59 | this.Result = BuilderInstance.Volume.SameAs(x.Shape); 60 | } 61 | 62 | x.Activation(this.Type, this.Result); 63 | return base.Evaluate(session); 64 | } 65 | 66 | public override Dictionary GetData() 67 | { 68 | var data = base.GetData(); 69 | data["ActivationType"] = this.Type; 70 | return data; 71 | } 72 | 73 | public override string ToString() 74 | { 75 | return $"{this.Type}({this.Parents[0]})"; 76 | } 77 | } 78 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Ops/ActivationGradient.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using ConvNetSharp.Volume; 3 | 4 | namespace ConvNetSharp.Flow.Ops 5 | { 6 | internal class ActivationGradient : Op where T : struct, IEquatable, IFormattable 7 | { 8 | public ActivationGradient(ConvNetSharp graph, Op input, Op output, Op outputGradient, ActivationType type) : base(graph) 9 | { 10 | this.AddParent(input); 11 | this.AddParent(output); 12 | this.AddParent(outputGradient); 13 | 14 | this.Type = type; 15 | } 16 | 17 | public ActivationType Type { get; set; } 18 | 19 | public override string Representation => $"{this.Type} Gradient"; 20 | 21 | public override void Differentiate() 22 | { 23 | throw new NotImplementedException(); 24 | } 25 | 26 | protected override void Dispose(bool disposing) 27 | { 28 | if (disposing) 29 | { 30 | this.Result?.Dispose(); 31 | } 32 | 33 | base.Dispose(disposing); 34 | } 35 | 36 | public override Volume Evaluate(Session session) 37 | { 38 | if (!this.IsDirty) 39 | { 40 | return base.Evaluate(session); 41 | } 42 | 43 | this.IsDirty = false; 44 | 45 | var input = this.Parents[0].Evaluate(session); 46 | var output = this.Parents[1].Evaluate(session); 47 | var outputGradient = this.Parents[2].Evaluate(session); 48 | 49 | if (this.Result == null || !Equals(this.Result.Shape, input.Shape)) 50 | { 51 | this.Result?.Dispose(); 52 | this.Result = BuilderInstance.Volume.SameAs(input.Shape); 53 | } 54 | 55 | output.ActivationGradient(input, outputGradient, this.Type, this.Result); 56 | 57 | return base.Evaluate(session); 58 | } 59 | 60 | public override string ToString() 61 | { 62 | return $"{this.Type} Gradient({this.Parents[0]})"; 63 | } 64 | } 65 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Ops/Add.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using ConvNetSharp.Volume; 4 | 5 | namespace ConvNetSharp.Flow.Ops 6 | { 7 | /// 8 | /// y = a + b 9 | /// 10 | /// 11 | public class Add : Op where T : struct, IEquatable, IFormattable 12 | { 13 | public Add(ConvNetSharp graph, Dictionary data) : base(graph) 14 | { 15 | } 16 | 17 | public Add(ConvNetSharp graph, Op left, Op right) : base(graph) 18 | { 19 | this.AddParent(left); 20 | this.AddParent(right); 21 | } 22 | 23 | public override string Representation => "+"; 24 | 25 | public override void Differentiate() 26 | { 27 | this.Parents[0].RegisterDerivate(this.Graph.Sum(this.Derivate, this.Graph.Shape(this.Parents[0]))); 28 | this.Parents[1].RegisterDerivate(this.Graph.Sum(this.Derivate, this.Graph.Shape(this.Parents[1]))); 29 | } 30 | 31 | protected override void Dispose(bool disposing) 32 | { 33 | if (disposing) 34 | { 35 | this.Result?.Dispose(); 36 | } 37 | 38 | base.Dispose(disposing); 39 | } 40 | 41 | public override Volume Evaluate(Session session) 42 | { 43 | if (!this.IsDirty) 44 | { 45 | return base.Evaluate(session); 46 | } 47 | 48 | this.IsDirty = false; 49 | 50 | var left = this.Parents[0].Evaluate(session); 51 | var right = this.Parents[1].Evaluate(session); 52 | 53 | var shape = right.Shape.TotalLength > left.Shape.TotalLength ? right.Shape : left.Shape; 54 | 55 | if (this.Result == null || !Equals(this.Result.Shape, shape)) 56 | { 57 | this.Result?.Dispose(); 58 | this.Result = BuilderInstance.Volume.SameAs(shape); 59 | } 60 | 61 | left.Add(right, this.Result); 62 | 63 | return base.Evaluate(session); 64 | } 65 | 66 | public override string ToString() 67 | { 68 | return $"{this.Parents[0]} + {this.Parents[1]}"; 69 | } 70 | } 71 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Ops/Assign.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using ConvNetSharp.Volume; 3 | 4 | namespace ConvNetSharp.Flow.Ops 5 | { 6 | /// 7 | /// Assignment: valueOp = op 8 | /// 9 | /// 10 | public class Assign : Op where T : struct, IEquatable, IFormattable 11 | { 12 | public Assign(ConvNetSharp graph, Op valueOp, Op op) : base(graph) 13 | { 14 | if (!(valueOp is Variable)) 15 | { 16 | throw new ArgumentException("Assigned Op should be a Variable", nameof(valueOp)); 17 | } 18 | 19 | this.AddParent(valueOp); 20 | this.AddParent(op); 21 | } 22 | 23 | public override string Representation => "->"; 24 | 25 | public override void Differentiate() 26 | { 27 | throw new NotImplementedException(); 28 | } 29 | 30 | public override Volume Evaluate(Session session) 31 | { 32 | if (!this.IsDirty) 33 | { 34 | return base.Evaluate(session); 35 | } 36 | 37 | this.IsDirty = false; 38 | 39 | this.Result = this.Parents[1].Evaluate(session); 40 | ((Variable)this.Parents[0]).SetValue(this.Result); 41 | 42 | return base.Evaluate(session); 43 | } 44 | 45 | public override string ToString() 46 | { 47 | return $"({this.Parents[0]} <- {this.Parents[1]})"; 48 | } 49 | } 50 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Ops/Concat.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using ConvNetSharp.Volume; 4 | 5 | namespace ConvNetSharp.Flow.Ops 6 | { 7 | public class Concat : Op where T : struct, IEquatable, IFormattable 8 | { 9 | private readonly int lastTotalLength = 0; 10 | 11 | public Concat(ConvNetSharp graph, Dictionary data) : base(graph) 12 | { 13 | } 14 | 15 | public Concat(ConvNetSharp graph, Op left, Op right) : base(graph) 16 | { 17 | this.AddParent(left); 18 | this.AddParent(right); 19 | } 20 | 21 | public override string Representation => "Concat"; 22 | 23 | public override void Differentiate() 24 | { 25 | var flattenShape = new Shape(1, 1, -1, Shape.Keep); 26 | var lengthLeft = new Shape(this.Graph, new Reshape(this.Graph, this.Parents[0], flattenShape), 2); 27 | var lengthRight = new Shape(this.Graph, new Reshape(this.Graph, this.Parents[1], flattenShape), 2); 28 | 29 | var extractLeft = this.Graph.Extract(this.Derivate, lengthLeft, ConvNetSharp.Zero); 30 | var extractRight = this.Graph.Extract(this.Derivate, lengthRight, lengthLeft); 31 | 32 | this.Parents[0].RegisterDerivate(new Reshape(this.Graph, extractLeft, this.Graph.Shape(this.Parents[0]))); 33 | this.Parents[1].RegisterDerivate(new Reshape(this.Graph, extractRight, this.Graph.Shape(this.Parents[1]))); 34 | } 35 | 36 | public override Volume Evaluate(Session session) 37 | { 38 | if (!this.IsDirty) 39 | { 40 | return base.Evaluate(session); 41 | } 42 | 43 | this.IsDirty = false; 44 | 45 | var left = this.Parents[0].Evaluate(session); 46 | var right = this.Parents[1].Evaluate(session); 47 | 48 | var batchSize = Math.Max(left.Shape.Dimensions[3], right.Shape.Dimensions[3]); 49 | 50 | var totalLength = (int)(left.Shape.TotalLength / left.Shape.Dimensions[3] + right.Shape.TotalLength / right.Shape.Dimensions[3]); 51 | if (this.Result == null || this.lastTotalLength != totalLength) 52 | { 53 | this.Result?.Dispose(); 54 | this.Result = BuilderInstance.Volume.SameAs(new Shape(1, 1, totalLength, batchSize)); 55 | } 56 | 57 | left.Concat(right, this.Result); 58 | 59 | return base.Evaluate(session); 60 | } 61 | 62 | public override string ToString() 63 | { 64 | return $"Concat({this.Parents[0]}, {this.Parents[1]})"; 65 | } 66 | } 67 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Ops/ConvolutionFilterGradient.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using ConvNetSharp.Volume; 3 | 4 | namespace ConvNetSharp.Flow.Ops 5 | { 6 | internal class ConvolutionFilterGradient : Op where T : struct, IEquatable, IFormattable 7 | { 8 | private readonly Convolution _convolution; 9 | 10 | public ConvolutionFilterGradient(ConvNetSharp graph, Convolution convolution, Op derivate) : base(graph) 11 | { 12 | this._convolution = convolution; 13 | 14 | this.AddParent(convolution); 15 | this.AddParent(derivate); 16 | } 17 | 18 | public override string Representation => "ConvolutionFilterGradient"; 19 | 20 | public override void Differentiate() 21 | { 22 | throw new NotImplementedException(); 23 | } 24 | 25 | public override Volume Evaluate(Session session) 26 | { 27 | if (!this.IsDirty) 28 | { 29 | return this._convolution.FilterGradient; 30 | } 31 | 32 | this.IsDirty = false; 33 | 34 | this._convolution.EvaluateGradient(session); 35 | return this._convolution.FilterGradient; 36 | } 37 | } 38 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Ops/ConvolutionInputGradient.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using ConvNetSharp.Volume; 3 | 4 | namespace ConvNetSharp.Flow.Ops 5 | { 6 | internal class ConvolutionInputGradient : Op where T : struct, IEquatable, IFormattable 7 | { 8 | private readonly Convolution _convolution; 9 | 10 | public ConvolutionInputGradient(ConvNetSharp graph, Convolution convolution, Op derivate) : base(graph) 11 | { 12 | this._convolution = convolution; 13 | 14 | this.AddParent(convolution); 15 | this.AddParent(derivate); 16 | } 17 | 18 | public override string Representation => "ConvolutionInputGradient"; 19 | 20 | public override void Differentiate() 21 | { 22 | throw new NotImplementedException(); 23 | } 24 | 25 | public override Volume Evaluate(Session session) 26 | { 27 | if (!this.IsDirty) 28 | { 29 | return this._convolution.InputGradient; 30 | } 31 | 32 | this.IsDirty = false; 33 | 34 | this._convolution.EvaluateGradient(session); 35 | return this._convolution.InputGradient; 36 | } 37 | } 38 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Ops/Dense.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | 4 | namespace ConvNetSharp.Flow.Ops 5 | { 6 | /// 7 | /// Dense / Fully connected layer is just a convolution of 'neuronCount' 1x1 filters 8 | /// 9 | /// 10 | public class Dense : Convolution where T : struct, IEquatable, IFormattable 11 | { 12 | public Dense(ConvNetSharp graph, Dictionary data) : base(graph, data) 13 | { 14 | } 15 | 16 | public Dense(ConvNetSharp graph, Op x, int neuronCount) : base(graph, x, 1, 1, neuronCount) 17 | { 18 | } 19 | } 20 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Ops/Div.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using ConvNetSharp.Volume; 5 | 6 | namespace ConvNetSharp.Flow.Ops 7 | { 8 | /// 9 | /// Element wise division 10 | /// y = left / right 11 | /// 12 | /// 13 | public class Div : Op where T : struct, IEquatable, IFormattable 14 | { 15 | public Div(ConvNetSharp graph, Dictionary data) : base(graph) 16 | { 17 | } 18 | 19 | public Div(ConvNetSharp graph, Op left, Op right) : base(graph) 20 | { 21 | this.AddParent(left); 22 | this.AddParent(right); 23 | } 24 | 25 | public override string Representation => "/"; 26 | 27 | public override void Differentiate() 28 | { 29 | this.Parents[0].RegisterDerivate(this.Derivate / this.Parents[1]); 30 | this.Parents[1].RegisterDerivate(this.Derivate * -this.Parents[0] / (this.Parents[1] * this.Parents[1])); 31 | } 32 | 33 | protected override void Dispose(bool disposing) 34 | { 35 | if (disposing) 36 | { 37 | this.Result?.Dispose(); 38 | } 39 | 40 | base.Dispose(disposing); 41 | } 42 | 43 | public override Volume Evaluate(Session session) 44 | { 45 | if (!this.IsDirty) 46 | { 47 | return base.Evaluate(session); 48 | } 49 | 50 | this.IsDirty = false; 51 | 52 | var left = this.Parents[0].Evaluate(session); 53 | var right = this.Parents[1].Evaluate(session); 54 | 55 | if (this.Result == null || !Equals(this.Result.Shape, left.Shape)) 56 | { 57 | this.Result?.Dispose(); 58 | this.Result = BuilderInstance.Volume.SameAs(left.Shape); 59 | } 60 | 61 | left.Divide(right, this.Result); 62 | 63 | return base.Evaluate(session); 64 | } 65 | 66 | public override string ToString() 67 | { 68 | var addParenthesis = this.Parents[0].Parents.Any(); 69 | var leftStr = addParenthesis ? $"({this.Parents[0]})" : $"{this.Parents[0]}"; 70 | 71 | addParenthesis = this.Parents[1].Parents.Any(); 72 | var rightStr = addParenthesis ? $"({this.Parents[1]})" : $"{this.Parents[1]}"; 73 | 74 | return $"{leftStr} / {rightStr}"; 75 | } 76 | } 77 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Ops/Dropout.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using ConvNetSharp.Volume; 4 | 5 | namespace ConvNetSharp.Flow.Ops 6 | { 7 | public class Dropout : Op where T : struct, IEquatable, IFormattable 8 | { 9 | private Shape _lastInputShape; 10 | 11 | public Dropout(ConvNetSharp graph, Op x, Op dropoutProbability) : base(graph) 12 | { 13 | this.AddParent(x); 14 | this.AddParent(dropoutProbability); 15 | } 16 | 17 | public Dropout(ConvNetSharp graph, Dictionary data) : base(graph) 18 | { 19 | } 20 | 21 | public override string Representation => $"Dropout({this.DropoutProbability})"; 22 | 23 | public Op DropoutProbability => this?.Parents[1]; 24 | 25 | public override void Differentiate() 26 | { 27 | this.Parents[0].RegisterDerivate(new DropoutGradient(this.Graph, this, this.Derivate)); 28 | } 29 | 30 | public override Volume Evaluate(Session session) 31 | { 32 | if (!this.IsDirty) 33 | { 34 | return base.Evaluate(session); 35 | } 36 | 37 | this.IsDirty = false; 38 | 39 | var x = this.Parents[0].Evaluate(session); 40 | var dropoutProb = this.Parents[1].Evaluate(session); 41 | 42 | if (this.Result == null || !Equals(this._lastInputShape, x.Shape)) 43 | { 44 | this._lastInputShape = new Shape(x.Shape); 45 | 46 | this.Result?.Dispose(); 47 | this.Result = BuilderInstance.Volume.SameAs(x.Shape); 48 | } 49 | 50 | x.Dropout(dropoutProb, this.Result); 51 | 52 | return base.Evaluate(session); 53 | } 54 | 55 | public override Dictionary GetData() 56 | { 57 | var data = base.GetData(); 58 | data["DropoutProbability"] = this.DropoutProbability; 59 | return data; 60 | } 61 | 62 | public override string ToString() 63 | { 64 | return $"Dropout({this.Parents[0]})"; 65 | } 66 | } 67 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Ops/DropoutGradient.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using ConvNetSharp.Volume; 3 | 4 | namespace ConvNetSharp.Flow.Ops 5 | { 6 | public class DropoutGradient : Op where T : struct, IEquatable, IFormattable 7 | { 8 | private readonly Dropout _dropout; 9 | private Shape _lastInputShape; 10 | 11 | public DropoutGradient(ConvNetSharp graph, Dropout dropout, Op derivate) : base(graph) 12 | { 13 | this._dropout = dropout; 14 | this.AddParent(dropout); 15 | this.AddParent(derivate); 16 | } 17 | 18 | public override string Representation => "DropoutGradient"; 19 | 20 | public override void Differentiate() 21 | { 22 | throw new NotImplementedException(); 23 | } 24 | 25 | public override Volume Evaluate(Session session) 26 | { 27 | if (!this.IsDirty) 28 | { 29 | return base.Evaluate(session); 30 | } 31 | 32 | this.IsDirty = false; 33 | 34 | var dropoutOutput = this._dropout.Evaluate(session); 35 | var dropoutInput = this._dropout.Parents[0].Evaluate(session); 36 | var dropoutOutputGradient = this._dropout.Derivate.Evaluate(session); 37 | var dropoutProb = this._dropout.DropoutProbability.Evaluate(session); 38 | 39 | if (this.Result == null || !Equals(this._lastInputShape, dropoutInput.Shape)) 40 | { 41 | this._lastInputShape = new Shape(dropoutInput.Shape); 42 | 43 | this.Result?.Dispose(); 44 | this.Result = BuilderInstance.Volume.SameAs(dropoutInput.Shape); 45 | } 46 | 47 | dropoutOutput.DropoutGradient(dropoutInput, dropoutOutputGradient, dropoutProb, this.Result); 48 | 49 | return base.Evaluate(session); 50 | } 51 | } 52 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Ops/Exp.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using ConvNetSharp.Volume; 4 | 5 | namespace ConvNetSharp.Flow.Ops 6 | { 7 | public class Exp : Op where T : struct, IEquatable, IFormattable 8 | { 9 | public Exp(ConvNetSharp graph, Dictionary data) : base(graph) 10 | { 11 | } 12 | 13 | public Exp(ConvNetSharp graph, Op x) : base(graph) 14 | { 15 | this.AddParent(x); 16 | } 17 | 18 | public override string Representation => "Exp"; 19 | 20 | public override void Differentiate() 21 | { 22 | this.Parents[0].RegisterDerivate(this.Derivate * this.Graph.Exp(this.Parents[0])); 23 | } 24 | 25 | public override Volume Evaluate(Session session) 26 | { 27 | if (!this.IsDirty) 28 | { 29 | return base.Evaluate(session); 30 | } 31 | 32 | this.IsDirty = false; 33 | 34 | var x = this.Parents[0].Evaluate(session); 35 | 36 | if (this.Result == null || !Equals(this.Result.Shape, x.Shape)) 37 | { 38 | this.Result?.Dispose(); 39 | this.Result = BuilderInstance.Volume.SameAs(x.Shape); 40 | } 41 | 42 | x.Exp(this.Result); 43 | return base.Evaluate(session); 44 | } 45 | 46 | public override string ToString() 47 | { 48 | return $"Exp({this.Parents[0]})"; 49 | } 50 | } 51 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Ops/Extract.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using ConvNetSharp.Volume; 4 | 5 | namespace ConvNetSharp.Flow.Ops 6 | { 7 | public class Extract : Op where T : struct, IEquatable, IFormattable 8 | { 9 | private readonly int lastTotalLength = 0; 10 | 11 | public Extract(ConvNetSharp graph, Dictionary data) : base(graph) 12 | { 13 | } 14 | 15 | public Extract(ConvNetSharp graph, Op x, Op length, Op offset) : base(graph) 16 | { 17 | this.AddParent(x); 18 | this.AddParent(length); 19 | this.AddParent(offset); 20 | } 21 | 22 | public override string Representation => "Extract"; 23 | 24 | public override string ToString() 25 | { 26 | return $"Extract({this.Parents[0]}, {this.Parents[1]}, {this.Parents[2]})"; 27 | } 28 | 29 | public override Volume Evaluate(Session session) 30 | { 31 | if (!this.IsDirty) 32 | { 33 | return base.Evaluate(session); 34 | } 35 | 36 | this.IsDirty = false; 37 | 38 | var x = this.Parents[0].Evaluate(session); 39 | var length = (int)Convert.ChangeType(this.Parents[1].Evaluate(session).Get(0), typeof(int)); // TODO: Find a way to keep this on host 40 | var offset = (int)Convert.ChangeType(this.Parents[2].Evaluate(session).Get(0), typeof(int)); // TODO: Find a way to keep this on host 41 | 42 | var batchSize = x.Shape.Dimensions[3]; 43 | 44 | var totalLength = length * batchSize; 45 | if (this.Result == null || this.lastTotalLength != totalLength) 46 | { 47 | this.Result?.Dispose(); 48 | this.Result = BuilderInstance.Volume.SameAs(new Shape(1, 1, length, batchSize)); 49 | } 50 | 51 | var isScalar = x.Shape.TotalLength == 1; 52 | 53 | if (isScalar) 54 | { 55 | x.Tile(this.Result.Shape.ToVolume(), this.Result); 56 | } 57 | else 58 | { 59 | x.Extract(length, offset, this.Result); 60 | } 61 | 62 | return base.Evaluate(session); 63 | } 64 | } 65 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Ops/INamedOp.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace ConvNetSharp.Flow.Ops 4 | { 5 | public interface INamedOp where T : struct, IEquatable, IFormattable 6 | { 7 | string Name { get; set; } 8 | } 9 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Ops/IPersistable.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using ConvNetSharp.Volume; 3 | 4 | namespace ConvNetSharp.Flow.Ops 5 | { 6 | public interface IPersistable : INamedOp where T : struct, IEquatable, IFormattable 7 | { 8 | Volume Result { get; set; } 9 | } 10 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Ops/LeakyRelu.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using ConvNetSharp.Volume; 4 | 5 | namespace ConvNetSharp.Flow.Ops 6 | { 7 | /// 8 | /// Implements LeakyReLU non-linearity element-wise 9 | /// x -> x > 0, x, otherwise alpha * x 10 | /// 11 | /// 12 | public class LeakyRelu : Op where T : struct, IEquatable, IFormattable 13 | { 14 | public LeakyRelu(ConvNetSharp graph, Dictionary data) : base(graph) 15 | { 16 | this.Alpha = (T)Convert.ChangeType(data["Alpha"], typeof(T)); 17 | } 18 | 19 | public LeakyRelu(ConvNetSharp graph, Op x, T alpha) : base(graph) 20 | { 21 | this.Alpha = alpha; 22 | this.AddParent(x); 23 | } 24 | 25 | public T Alpha { get; set; } 26 | 27 | public override string Representation => "LeakyRelu"; 28 | 29 | public override void Differentiate() 30 | { 31 | var x = this.Parents[0]; 32 | 33 | x.RegisterDerivate(this.Graph.LeakyReluGradient(this, this.Derivate, this.Alpha)); 34 | } 35 | 36 | public override Volume Evaluate(Session session) 37 | { 38 | if (!this.IsDirty) 39 | { 40 | return base.Evaluate(session); 41 | } 42 | 43 | this.IsDirty = false; 44 | 45 | var x = this.Parents[0].Evaluate(session); 46 | 47 | if (this.Result == null || !Equals(this.Result.Shape, x.Shape)) 48 | { 49 | this.Result?.Dispose(); 50 | this.Result = BuilderInstance.Volume.SameAs(x.Shape); 51 | } 52 | 53 | x.LeakyRelu(this.Alpha, this.Result); 54 | 55 | return base.Evaluate(session); 56 | } 57 | 58 | public override Dictionary GetData() 59 | { 60 | var data = base.GetData(); 61 | data["Alpha"] = this.Alpha; 62 | return data; 63 | } 64 | 65 | public override string ToString() 66 | { 67 | return $"LeakyRelu({this.Parents[0]})"; 68 | } 69 | } 70 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Ops/LeakyReluGradient.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using ConvNetSharp.Volume; 4 | 5 | namespace ConvNetSharp.Flow.Ops 6 | { 7 | /// 8 | /// Implements LeakyReLU gradient 9 | /// 10 | /// 11 | public class LeakyReluGradient : Op where T : struct, IEquatable, IFormattable 12 | { 13 | public LeakyReluGradient(ConvNetSharp graph, Dictionary data) : base(graph) 14 | { 15 | this.Alpha = (T)Convert.ChangeType(data["Alpha"], typeof(T)); 16 | } 17 | 18 | public LeakyReluGradient(ConvNetSharp graph, Op y, Op derivate, T alpha) : base(graph) 19 | { 20 | this.Alpha = alpha; 21 | this.AddParent(y); 22 | this.AddParent(derivate); 23 | } 24 | 25 | public T Alpha { get; set; } 26 | 27 | public override string Representation => "LeakyReluGradient"; 28 | 29 | public override Volume Evaluate(Session session) 30 | { 31 | if (!this.IsDirty) 32 | { 33 | return base.Evaluate(session); 34 | } 35 | 36 | this.IsDirty = false; 37 | 38 | var y = this.Parents[0].Evaluate(session); 39 | var derivate = this.Parents[1].Evaluate(session); 40 | 41 | if (this.Result == null || !Equals(this.Result.Shape, y.Shape)) 42 | { 43 | this.Result?.Dispose(); 44 | this.Result = BuilderInstance.Volume.SameAs(y.Shape); 45 | } 46 | 47 | y.LeakyReluGradient(derivate, this.Result, this.Alpha); 48 | return base.Evaluate(session); 49 | } 50 | 51 | public override Dictionary GetData() 52 | { 53 | var data = base.GetData(); 54 | data["Alpha"] = this.Alpha; 55 | return data; 56 | } 57 | 58 | public override string ToString() 59 | { 60 | return $"LeakyReluGradient({this.Parents[0]})"; 61 | } 62 | } 63 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Ops/Log.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using ConvNetSharp.Volume; 4 | 5 | namespace ConvNetSharp.Flow.Ops 6 | { 7 | public class Log : Op where T : struct, IEquatable, IFormattable 8 | { 9 | public Log(ConvNetSharp graph, Dictionary data) : base(graph) 10 | { 11 | } 12 | 13 | public Log(ConvNetSharp graph, Op x) : base(graph) 14 | { 15 | this.AddParent(x); 16 | } 17 | 18 | public override string Representation => "Log"; 19 | 20 | public override void Differentiate() 21 | { 22 | this.Parents[0].RegisterDerivate(this.Derivate / this.Parents[0]); 23 | } 24 | 25 | public override Volume Evaluate(Session session) 26 | { 27 | if (!this.IsDirty) 28 | { 29 | return base.Evaluate(session); 30 | } 31 | 32 | this.IsDirty = false; 33 | 34 | var x = this.Parents[0].Evaluate(session); 35 | 36 | if (this.Result == null || !Equals(this.Result.Shape, x.Shape)) 37 | { 38 | this.Result?.Dispose(); 39 | this.Result = BuilderInstance.Volume.SameAs(x.Shape); 40 | } 41 | 42 | x.Log(this.Result); 43 | return base.Evaluate(session); 44 | } 45 | 46 | public override string ToString() 47 | { 48 | return $"Log({this.Parents[0]})"; 49 | } 50 | } 51 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Ops/Max.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using ConvNetSharp.Volume; 4 | 5 | namespace ConvNetSharp.Flow.Ops 6 | { 7 | public class Max : Op where T : struct, IEquatable, IFormattable 8 | { 9 | public Max(ConvNetSharp graph, Dictionary data) : base(graph) 10 | { 11 | } 12 | 13 | public Max(ConvNetSharp graph, Op x) : base(graph) 14 | { 15 | this.AddParent(x); 16 | } 17 | 18 | public override string Representation => "Max"; 19 | 20 | public override void Differentiate() 21 | { 22 | throw new NotImplementedException(); 23 | } 24 | 25 | protected override void Dispose(bool disposing) 26 | { 27 | if (disposing) 28 | { 29 | this.Result?.Dispose(); 30 | } 31 | 32 | base.Dispose(disposing); 33 | } 34 | 35 | public override Volume Evaluate(Session session) 36 | { 37 | if (!this.IsDirty) 38 | { 39 | return base.Evaluate(session); 40 | } 41 | 42 | this.IsDirty = false; 43 | 44 | var x = this.Parents[0].Evaluate(session); 45 | var reshape = x.ReShape(-1, x.Shape.Dimensions[3]); 46 | var targetShape = new Shape(reshape.Shape); 47 | targetShape.SetDimension(0, 1); 48 | 49 | if (this.Result == null || !Equals(this.Result.Shape, targetShape)) 50 | { 51 | this.Result?.Dispose(); 52 | this.Result = BuilderInstance.Volume.SameAs(targetShape); 53 | } 54 | 55 | reshape.Reduce(TensorReduceOp.Max, this.Result); 56 | 57 | return base.Evaluate(session); 58 | } 59 | 60 | public override string ToString() 61 | { 62 | return $"Max({this.Parents[0]})"; 63 | } 64 | } 65 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Ops/Mult.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using ConvNetSharp.Volume; 5 | 6 | namespace ConvNetSharp.Flow.Ops 7 | { 8 | /// 9 | /// Element wise multiplication 10 | /// y = left * right 11 | /// 12 | /// 13 | public class Mult : Op where T : struct, IEquatable, IFormattable 14 | { 15 | public Mult(ConvNetSharp graph, Dictionary data) : base(graph) 16 | { 17 | } 18 | 19 | public Mult(ConvNetSharp graph, Op left, Op right) : base(graph) 20 | { 21 | this.AddParent(left); 22 | this.AddParent(right); 23 | } 24 | 25 | public override string Representation => "*"; 26 | 27 | public override void Differentiate() 28 | { 29 | // dA = GB, dB = AG 30 | this.Parents[0].RegisterDerivate(this.Derivate * this.Parents[1]); 31 | this.Parents[1].RegisterDerivate(this.Derivate * this.Parents[0]); 32 | } 33 | 34 | protected override void Dispose(bool disposing) 35 | { 36 | if (disposing) 37 | { 38 | this.Result?.Dispose(); 39 | } 40 | 41 | base.Dispose(disposing); 42 | } 43 | 44 | public override Volume Evaluate(Session session) 45 | { 46 | if (!this.IsDirty) 47 | { 48 | return base.Evaluate(session); 49 | } 50 | 51 | this.IsDirty = false; 52 | 53 | var left = this.Parents[0].Evaluate(session); 54 | var right = this.Parents[1].Evaluate(session); 55 | 56 | var shape = right.Shape.TotalLength > left.Shape.TotalLength ? right.Shape : left.Shape; 57 | 58 | if (this.Result == null || !Equals(this.Result.Shape, shape)) 59 | { 60 | this.Result?.Dispose(); 61 | this.Result = BuilderInstance.Volume.SameAs(shape); 62 | } 63 | 64 | left.Multiply(right, this.Result); 65 | 66 | return base.Evaluate(session); 67 | } 68 | 69 | public override string ToString() 70 | { 71 | var addParenthesis = this.Parents[0].Parents.Any(); 72 | var leftStr = addParenthesis ? $"({this.Parents[0]})" : $"{this.Parents[0]}"; 73 | 74 | addParenthesis = this.Parents[1].Parents.Any(); 75 | var rightStr = addParenthesis ? $"({this.Parents[1]})" : $"{this.Parents[1]}"; 76 | 77 | return $"{leftStr} * {rightStr}"; 78 | } 79 | } 80 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Ops/Negate.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using ConvNetSharp.Volume; 5 | 6 | namespace ConvNetSharp.Flow.Ops 7 | { 8 | /// 9 | /// y = -x 10 | /// 11 | /// 12 | public class Negate : Op where T : struct, IEquatable, IFormattable 13 | { 14 | public Negate(ConvNetSharp graph, Dictionary data) : base(graph) 15 | { 16 | } 17 | 18 | public Negate(ConvNetSharp graph, Op x) : base(graph) 19 | { 20 | this.AddParent(x); 21 | } 22 | 23 | public override string Representation => "Neg"; 24 | 25 | public override void Differentiate() 26 | { 27 | this.Parents[0].RegisterDerivate(-this.Derivate); 28 | } 29 | 30 | protected override void Dispose(bool disposing) 31 | { 32 | if (disposing) 33 | { 34 | this.Result?.Dispose(); 35 | } 36 | 37 | base.Dispose(disposing); 38 | } 39 | 40 | public override Volume Evaluate(Session session) 41 | { 42 | if (!this.IsDirty) 43 | { 44 | return base.Evaluate(session); 45 | } 46 | 47 | this.IsDirty = false; 48 | 49 | var y = this.Parents[0].Evaluate(session); 50 | 51 | if (this.Result == null || !Equals(this.Result.Shape, y.Shape)) 52 | { 53 | this.Result?.Dispose(); 54 | this.Result = BuilderInstance.Volume.SameAs(y.Shape); 55 | } 56 | 57 | y.Negate(this.Result); 58 | 59 | return base.Evaluate(session); 60 | } 61 | 62 | public override string ToString() 63 | { 64 | var addParenthesis = this.Parents[0].Parents.Any(); 65 | if (addParenthesis) 66 | { 67 | return $"(-{this.Parents[0]})"; 68 | } 69 | 70 | return $"-{this.Parents[0]}"; 71 | } 72 | } 73 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Ops/OpVisitor.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using ConvNetSharp.Flow.Graph; 4 | 5 | namespace ConvNetSharp.Flow.Ops 6 | { 7 | /// 8 | /// Walk the computation graph and execute func on each node 9 | /// 10 | /// 11 | public class OpVisitor : IOpVisitor where T : struct, IEquatable, IFormattable 12 | { 13 | private readonly bool _bothWays; 14 | private readonly Action> _func; 15 | private readonly HashSet> visited = new HashSet>(); 16 | 17 | public OpVisitor(Action> func, bool bothWays = false) 18 | { 19 | this._bothWays = bothWays; 20 | this._func = func; 21 | } 22 | 23 | public void Visit(Op op) 24 | { 25 | if (this.visited.Contains(op)) 26 | { 27 | return; 28 | } 29 | 30 | this._func(op); 31 | 32 | this.visited.Add(op); 33 | 34 | foreach (var parents in op.Parents) 35 | { 36 | parents.Accept(this); 37 | } 38 | 39 | if (this._bothWays) 40 | { 41 | foreach (var child in op.Children) 42 | { 43 | child.Accept(this); 44 | } 45 | } 46 | } 47 | } 48 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Ops/PlaceHolder.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Diagnostics; 4 | using ConvNetSharp.Volume; 5 | 6 | namespace ConvNetSharp.Flow.Ops 7 | { 8 | /// 9 | /// A PlaceHolder is an entry point in the computation graph. Its value is set at every run of the graph using the 10 | /// provided dictionary (See ConvNetSharp.Flow.Session.UpdatePlaceHolder method) 11 | /// 12 | [DebuggerDisplay("{" + nameof(Name) + "}")] 13 | public class PlaceHolder : Op, INamedOp where T : struct, IEquatable, IFormattable 14 | { 15 | public PlaceHolder(ConvNetSharp graph, Dictionary data) : base(graph) 16 | { 17 | this.Name = (string) data["Name"]; 18 | } 19 | 20 | public PlaceHolder(ConvNetSharp graph, string name) : base(graph) 21 | { 22 | this.Name = name; 23 | } 24 | 25 | public override string Representation => this.Name; 26 | 27 | public string Name { get; set; } 28 | 29 | public override void Differentiate() 30 | { 31 | } 32 | 33 | protected override void Dispose(bool disposing) 34 | { 35 | if (disposing) 36 | { 37 | this.Result?.Dispose(); 38 | } 39 | 40 | base.Dispose(disposing); 41 | } 42 | 43 | public override Dictionary GetData() 44 | { 45 | var data = base.GetData(); 46 | data["Name"] = this.Name; 47 | return data; 48 | } 49 | 50 | public void SetValue(Volume value) 51 | { 52 | this.Result = value; 53 | this.SetDirty(); 54 | } 55 | 56 | public override string ToString() 57 | { 58 | return this.Name; 59 | } 60 | } 61 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Ops/PoolGradient.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using ConvNetSharp.Volume; 3 | 4 | namespace ConvNetSharp.Flow.Ops 5 | { 6 | public class PoolGradient : Op where T : struct, IEquatable, IFormattable 7 | { 8 | private readonly Pool _pool; 9 | 10 | public PoolGradient(ConvNetSharp graph, Pool pool, Op derivate) : base(graph) 11 | { 12 | this._pool = pool; 13 | 14 | this.AddParent(pool); 15 | this.AddParent(derivate); 16 | } 17 | 18 | public override string Representation => "PoolGradient"; 19 | 20 | public override void Differentiate() 21 | { 22 | throw new NotImplementedException(); 23 | } 24 | 25 | public override Volume Evaluate(Session session) 26 | { 27 | if (!this.IsDirty) 28 | { 29 | return this._pool.InputGradient; 30 | } 31 | 32 | this.IsDirty = false; 33 | 34 | this._pool.EvaluateGradient(session); 35 | return this._pool.InputGradient; 36 | } 37 | } 38 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Ops/Power.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using ConvNetSharp.Volume; 4 | 5 | namespace ConvNetSharp.Flow.Ops 6 | { 7 | /// 8 | /// Computes u^v 9 | /// 10 | /// 11 | public class Power : Op where T : struct, IEquatable, IFormattable 12 | { 13 | public Power(ConvNetSharp graph, Dictionary data) : base(graph) 14 | { 15 | } 16 | 17 | public Power(ConvNetSharp graph, Op u, Op v) : base(graph) 18 | { 19 | this.AddParent(u); 20 | this.AddParent(v); 21 | } 22 | 23 | public override string Representation => "Pow"; 24 | 25 | public override void Differentiate() 26 | { 27 | var u = this.Parents[0]; 28 | var v = this.Parents[1]; 29 | 30 | // d(u^v)/d(u) = v.u^(v-1) 31 | u.RegisterDerivate(this.Derivate * v * (u ^ (v - this.Graph.Const(ConvNetSharp.One, "one")))); 32 | 33 | // d(u^v)/d(v) = u^v.log(u) 34 | v.RegisterDerivate(this.Derivate * (u ^ v) * this.Graph.Log(u)); 35 | } 36 | 37 | public override Volume Evaluate(Session session) 38 | { 39 | if (!this.IsDirty) 40 | { 41 | return base.Evaluate(session); 42 | } 43 | 44 | this.IsDirty = false; 45 | 46 | var u = this.Parents[0].Evaluate(session); 47 | var v = this.Parents[1].Evaluate(session); 48 | 49 | if (this.Result == null || !Equals(this.Result.Shape, u.Shape)) 50 | { 51 | this.Result?.Dispose(); 52 | this.Result = BuilderInstance.Volume.SameAs(u.Shape); 53 | } 54 | 55 | u.Power(v, this.Result); 56 | return base.Evaluate(session); 57 | } 58 | 59 | public override string ToString() 60 | { 61 | return $"Pow({this.Parents[0]}, {this.Parents[1]})"; 62 | } 63 | } 64 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Ops/SoftMax.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using ConvNetSharp.Volume; 4 | 5 | namespace ConvNetSharp.Flow.Ops 6 | { 7 | public class Softmax : Op where T : struct, IEquatable, IFormattable 8 | { 9 | private long _lastGradientComputeStep = -1; 10 | 11 | public Softmax(ConvNetSharp graph, Op x) : base(graph) 12 | { 13 | this.AddParent(x); 14 | } 15 | 16 | public Softmax(ConvNetSharp graph, Dictionary data) : base(graph) 17 | { 18 | } 19 | 20 | public override string Representation => "Softmax"; 21 | 22 | public Volume InputGradient { get; set; } 23 | 24 | public override void Differentiate() 25 | { 26 | this.Parents[0].RegisterDerivate(new SoftmaxGradient(this.Graph, this)); 27 | } 28 | 29 | public override Volume Evaluate(Session session) 30 | { 31 | if (!this.IsDirty) 32 | { 33 | return base.Evaluate(session); 34 | } 35 | 36 | this.IsDirty = false; 37 | 38 | var x = this.Parents[0].Evaluate(session); 39 | 40 | if (this.Result == null || !Equals(this.Result.Shape, x.Shape)) 41 | { 42 | this.Result?.Dispose(); 43 | this.Result = BuilderInstance.Volume.SameAs(x.Shape); 44 | } 45 | 46 | x.Softmax(this.Result); 47 | 48 | return base.Evaluate(session); 49 | } 50 | 51 | public void EvaluateGradient(Session session) 52 | { 53 | if (this._lastGradientComputeStep == session.Step) 54 | { 55 | return; 56 | } 57 | 58 | this._lastGradientComputeStep = session.Step; 59 | 60 | var x = this.Parents[0].Evaluate(session); 61 | 62 | if (this.InputGradient == null || !Equals(x.Shape, this.InputGradient.Shape)) 63 | { 64 | this.InputGradient = BuilderInstance.Volume.SameAs(x.Shape); 65 | } 66 | 67 | if (this.Derivate != null) 68 | { 69 | x.SoftmaxGradient(this.Derivate.Evaluate(session), this.InputGradient); 70 | } 71 | } 72 | 73 | public override string ToString() 74 | { 75 | return $"Softmax({this.Parents[0]})"; 76 | } 77 | } 78 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Ops/SoftMaxGradient.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using ConvNetSharp.Volume; 3 | 4 | namespace ConvNetSharp.Flow.Ops 5 | { 6 | internal class SoftmaxGradient : Op where T : struct, IEquatable, IFormattable 7 | { 8 | private readonly Softmax _softmax; 9 | 10 | public SoftmaxGradient(ConvNetSharp graph, Softmax softmax) : base(graph) 11 | { 12 | this._softmax = softmax; 13 | this.AddParent(softmax); 14 | } 15 | 16 | public override string Representation => "SoftmaxGradient"; 17 | 18 | public override void Differentiate() 19 | { 20 | throw new NotImplementedException(); 21 | } 22 | 23 | public override Volume Evaluate(Session session) 24 | { 25 | if (!this.IsDirty) 26 | { 27 | return base.Evaluate(session); 28 | } 29 | 30 | this.IsDirty = false; 31 | 32 | this._softmax.EvaluateGradient(session); 33 | 34 | this.Result = this._softmax.InputGradient; 35 | return base.Evaluate(session); 36 | } 37 | } 38 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Ops/Sqrt.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using ConvNetSharp.Volume; 4 | 5 | namespace ConvNetSharp.Flow.Ops 6 | { 7 | /// 8 | /// Computes square root of x element-wise 9 | /// 10 | /// 11 | public class Sqrt : Op where T : struct, IEquatable, IFormattable 12 | { 13 | public Sqrt(ConvNetSharp graph, Dictionary data) : base(graph) 14 | { 15 | } 16 | 17 | public Sqrt(ConvNetSharp graph, Op x) : base(graph) 18 | { 19 | this.AddParent(x); 20 | } 21 | 22 | public override string Representation => "Sqrt"; 23 | 24 | public override void Differentiate() 25 | { 26 | var u = this.Parents[0]; 27 | 28 | // d(sqrt(u))/du = 1 / (2*sqrt(u)) 29 | u.RegisterDerivate(this.Derivate / (this.Graph.Const((T)Convert.ChangeType(2.0, typeof(T)), "two") * this.Graph.Sqrt(u))); 30 | } 31 | 32 | public override Volume Evaluate(Session session) 33 | { 34 | if (!this.IsDirty) 35 | { 36 | return base.Evaluate(session); 37 | } 38 | 39 | this.IsDirty = false; 40 | 41 | var x = this.Parents[0].Evaluate(session); 42 | 43 | if (this.Result == null || !Equals(this.Result.Shape, x.Shape)) 44 | { 45 | this.Result?.Dispose(); 46 | this.Result = BuilderInstance.Volume.SameAs(x.Shape); 47 | } 48 | 49 | x.Sqrt(this.Result); 50 | return base.Evaluate(session); 51 | } 52 | 53 | public override string ToString() 54 | { 55 | return $"Sqrt({this.Parents[0]})"; 56 | } 57 | } 58 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Ops/Tile.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using ConvNetSharp.Volume; 4 | 5 | namespace ConvNetSharp.Flow.Ops 6 | { 7 | /// 8 | /// Construct a volume by repeating x of times given by reps 9 | /// 10 | /// 11 | public class Tile : Op where T : struct, IEquatable, IFormattable 12 | { 13 | private int _lastBatchSize; 14 | private Shape _tempShape; 15 | 16 | 17 | public Tile(ConvNetSharp graph, Op x, Op reps) : base(graph) 18 | { 19 | this.AddParent(x); 20 | this.AddParent(reps); 21 | } 22 | 23 | public Tile(ConvNetSharp graph, Dictionary data) : base(graph) 24 | { 25 | } 26 | 27 | public override string Representation => "Tile"; 28 | 29 | public override void Differentiate() 30 | { 31 | throw new NotImplementedException(); 32 | } 33 | 34 | public override Volume Evaluate(Session session) 35 | { 36 | if (!this.IsDirty) 37 | { 38 | return base.Evaluate(session); 39 | } 40 | 41 | this.IsDirty = false; 42 | 43 | var x = this.Parents[0].Evaluate(session); 44 | var reps = this.Parents[1].Evaluate(session); 45 | 46 | if (this._tempShape == null || session.BatchSize != this._lastBatchSize) 47 | { 48 | var dim0 = Math.Max(x.Shape.Dimensions[0] * Convert.ToInt32(reps.Get(0)), 1); 49 | var dim1 = Math.Max(x.Shape.Dimensions[1] * Convert.ToInt32(reps.Get(1)), 1); 50 | var dim2 = Math.Max(x.Shape.Dimensions[2] * Convert.ToInt32(reps.Get(2)), 1); 51 | var dim3 = Math.Max(x.Shape.Dimensions[3] * Convert.ToInt32(reps.Get(3)), 1); 52 | 53 | this._tempShape = new Shape(dim0, dim1, dim2, dim3); 54 | this._lastBatchSize = session.BatchSize; 55 | this.Result = BuilderInstance.Volume.SameAs(this._tempShape); 56 | } 57 | 58 | this.Result.Clear(); 59 | 60 | x.Tile(reps, this.Result); 61 | 62 | return base.Evaluate(session); 63 | } 64 | } 65 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Ops/Transpose.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using ConvNetSharp.Volume; 4 | 5 | namespace ConvNetSharp.Flow.Ops 6 | { 7 | public class Transpose : Op where T : struct, IEquatable, IFormattable 8 | { 9 | public Transpose(ConvNetSharp graph, Dictionary data) : base(graph) 10 | { 11 | } 12 | 13 | public Transpose(ConvNetSharp graph, Op x) : base(graph) 14 | { 15 | this.AddParent(x); 16 | } 17 | 18 | public override string Representation => "Transpose"; 19 | 20 | public override void Differentiate() 21 | { 22 | // TODO 23 | } 24 | 25 | public override Volume Evaluate(Session session) 26 | { 27 | if (!this.IsDirty) 28 | { 29 | return base.Evaluate(session); 30 | } 31 | 32 | this.IsDirty = false; 33 | 34 | var x = this.Parents[0].Evaluate(session); 35 | 36 | var expectedShape = new Shape(x.Shape.Dimensions[1], x.Shape.Dimensions[0], 1, x.Shape.Dimensions[3]); 37 | 38 | if (this.Result == null || !Equals(this.Result.Shape, expectedShape)) 39 | { 40 | this.Result?.Dispose(); 41 | this.Result = BuilderInstance.Volume.SameAs(expectedShape); 42 | } 43 | 44 | x.Transpose(this.Result); 45 | return base.Evaluate(session); 46 | } 47 | 48 | public override string ToString() 49 | { 50 | if (this.Parents[0].Parents.Count <= 1) 51 | { 52 | return $"{this.Parents[0]}ᵀ"; 53 | } 54 | 55 | return $"({this.Parents[0]})ᵀ"; 56 | } 57 | } 58 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Ops/Variable.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Diagnostics; 4 | using ConvNetSharp.Volume; 5 | 6 | namespace ConvNetSharp.Flow.Ops 7 | { 8 | /// 9 | /// Variable hold a named Volume that can change over time. 10 | /// 11 | [DebuggerDisplay("{" + nameof(Name) + "}")] 12 | public class Variable : Op, IPersistable where T : struct, IEquatable, IFormattable 13 | { 14 | public Variable(ConvNetSharp graph, Volume v, string name, bool isLearnable = false) : base(graph) 15 | { 16 | this.Name = name; 17 | this.Result = v; 18 | this.IsLearnable = isLearnable; 19 | } 20 | 21 | public Variable(ConvNetSharp graph, Dictionary data) : base(graph) 22 | { 23 | this.Name = (string)data["Name"]; 24 | this.IsLearnable = (string)data["IsLearnable"] == "True"; 25 | } 26 | 27 | public override string Representation => this.Name; 28 | 29 | /// 30 | /// If set to true optimizer will try to reduce cost by updating this variable 31 | /// 32 | public bool IsLearnable { get; } 33 | 34 | public string Name { get; set; } 35 | 36 | protected override void Dispose(bool disposing) 37 | { 38 | if (disposing) 39 | { 40 | this.Result?.Dispose(); 41 | } 42 | 43 | base.Dispose(disposing); 44 | } 45 | 46 | public override Dictionary GetData() 47 | { 48 | var data = base.GetData(); 49 | data["Name"] = this.Name; 50 | data["IsLearnable"] = this.IsLearnable; 51 | return data; 52 | } 53 | 54 | public void SetValue(Volume value) 55 | { 56 | this.Result = value; 57 | this.SetDirty(); 58 | } 59 | 60 | public override string ToString() 61 | { 62 | return this.Name; 63 | } 64 | } 65 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Scope.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace ConvNetSharp.Flow 4 | { 5 | public class Scope : IDisposable where T : struct, IEquatable, IFormattable 6 | { 7 | private readonly ConvNetSharp _cns; 8 | private readonly string _name; 9 | 10 | public Scope(string name, ConvNetSharp cns) 11 | { 12 | this._name = name; 13 | this._cns = cns; 14 | } 15 | 16 | public void Dispose() 17 | { 18 | this._cns.ReleaseScope(this._name); 19 | } 20 | } 21 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Training/SgdTrainer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace ConvNetSharp.Flow.Training 4 | { 5 | public class SgdTrainer : TrainerBase, IDisposable where T : struct, IEquatable, IFormattable 6 | { 7 | public SgdTrainer(Net net, T learningRate) : base(net) 8 | { 9 | this.LearningRate = learningRate; 10 | this.Optimizer = new GradientDescentOptimizer(net.Op.Graph, learningRate); 11 | } 12 | 13 | public T LearningRate { get; } 14 | 15 | public void Dispose() 16 | { 17 | this.Optimizer.Dispose(); 18 | } 19 | } 20 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/Training/TrainerBase.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Diagnostics; 4 | using ConvNetSharp.Core; 5 | using ConvNetSharp.Flow.Ops; 6 | using ConvNetSharp.Volume; 7 | 8 | namespace ConvNetSharp.Flow.Training 9 | { 10 | public abstract class TrainerBase : Core.Training.TrainerBase where T : struct, IEquatable, IFormattable 11 | { 12 | private readonly Dictionary> _dico = new Dictionary>(); 13 | protected readonly Net _net; 14 | 15 | protected TrainerBase(INet net) : base(net) 16 | { 17 | this._net = net as Net; 18 | } 19 | 20 | public Op Optimizer { get; set; } 21 | 22 | protected override void TrainImplem() 23 | { 24 | throw new NotImplementedException(); 25 | } 26 | 27 | public override void Train(Volume x, Volume y) 28 | { 29 | var batchSize = x.Shape.Dimensions[3]; 30 | 31 | this._dico["Y"] = y; 32 | this._dico["input"] = x; 33 | 34 | var chrono = Stopwatch.StartNew(); 35 | this.Loss = this._net.Session.Run(this._net.Cost, this._dico).Get(0); 36 | this.ForwardTimeMs = chrono.Elapsed.TotalMilliseconds / batchSize; 37 | 38 | chrono = Stopwatch.StartNew(); 39 | this._net.Session.Run(this.Optimizer, this._dico); 40 | this.BackwardTimeMs = chrono.Elapsed.TotalMilliseconds / batchSize; 41 | } 42 | } 43 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Flow/img/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbovar/ConvNetSharp/ccaf6f6959c12026c4e1a0e396a7711654c92551/src/ConvNetSharp.Flow/img/icon.png -------------------------------------------------------------------------------- /src/ConvNetSharp.Performance.Tests/ConvNetSharp.Performance.Tests.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netcoreapp3.1 5 | false 6 | ConvNetSharp.Performance.Tests.Program 7 | 8 | 9 | 10 | x64 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | all 20 | runtime; build; native; contentfiles; analyzers; buildtransitive 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /src/ConvNetSharp.Utils/ConvNetSharp.Utils.Nuget.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netstandard2.1 5 | 6 | 7 | 8 | 0.4.13 9 | 0.4.13.0 10 | Cedric Bovar 11 | Deep learning in C# 12 | Copyright (c) 2020 13 | 14 | 15 | 16 | Cognitio.ConvNetSharp.Utils 17 | 0.4.13 18 | false 19 | Deep Learning Convolution AI 20 | https://github.com/cbovar/ConvNetSharp/blob/master/LICENSE 21 | https://github.com/cbovar/ConvNetSharp 22 | Cognitio.ConvNetSharp.Utils 23 | Cognitio.ConvNetSharp.Utils 24 | https://github.com/cbovar/ConvNetSharp/releases/tag/v0.4.13 25 | True 26 | Cognitio.ConvNetSharp.Utils 27 | 0.4.13.0 28 | icon.png 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /src/ConvNetSharp.Utils/ConvNetSharp.Utils.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | WinExe 5 | netcoreapp3.1 6 | true 7 | true 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /src/ConvNetSharp.Utils/GraphVisualizer/GraphControl.xaml.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | using System.Windows; 7 | using System.Windows.Controls; 8 | using System.Windows.Data; 9 | using System.Windows.Documents; 10 | using System.Windows.Input; 11 | using System.Windows.Media; 12 | using System.Windows.Media.Imaging; 13 | using System.Windows.Navigation; 14 | using System.Windows.Shapes; 15 | 16 | namespace ConvNetSharp.Utils.GraphVisualizer 17 | { 18 | /// 19 | /// Interaction logic for GraphControl.xaml 20 | /// 21 | public partial class GraphControl : Window 22 | { 23 | public GraphControl() 24 | { 25 | InitializeComponent(); 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/ConvNetSharp.Utils/GraphVisualizer/OpEdge.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Windows.Data; 3 | using System.Windows.Media; 4 | using QuickGraph; 5 | 6 | namespace ConvNetSharp.Utils.GraphVisualizer 7 | { 8 | public class OpVertex 9 | { 10 | public string Name { get; set; } 11 | 12 | public string Shape { get; set; } 13 | 14 | public string Type { get; set; } 15 | 16 | public Color Color { get; set; } 17 | } 18 | 19 | public class OpEdge : Edge 20 | { 21 | public OpEdge(OpVertex source, OpVertex target) : base(source, target) 22 | { 23 | } 24 | } 25 | 26 | public class EdgeColorConverter : IValueConverter 27 | { 28 | public object Convert(object value, Type targetType, object parameter, System.Globalization.CultureInfo culture) 29 | { 30 | return new SolidColorBrush((Color)value); 31 | } 32 | 33 | public object ConvertBack(object value, Type targetType, object parameter, System.Globalization.CultureInfo culture) 34 | { 35 | throw new NotImplementedException(); 36 | } 37 | } 38 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Utils/Image.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Drawing; 3 | 4 | namespace ConvNetSharp.Utils 5 | { 6 | public static class Image 7 | { 8 | public static Bitmap ToBitmap(Volume.Volume vol, int n = 0) where T : struct, IEquatable, IFormattable 9 | { 10 | var bmp = new Bitmap(vol.Shape.Dimensions[0], vol.Shape.Dimensions[1]); 11 | bool isBlackAndWhite = vol.Shape.Dimensions[2] == 1; 12 | 13 | for (int y = 0; y < bmp.Height; y++) 14 | { 15 | for (int x = 0; x < bmp.Width; x++) 16 | { 17 | Color color; 18 | if (isBlackAndWhite) 19 | { 20 | byte c = ToColorComponent(vol.Get(x, y, 0, n)); 21 | color = Color.FromArgb(c, c, c); 22 | } 23 | else 24 | { 25 | byte r = ToColorComponent(vol.Get(x, y, 0,n)); 26 | byte g = ToColorComponent(vol.Get(x, y, 1,n)); 27 | byte b = ToColorComponent(vol.Get(x, y, 2,n)); 28 | color = Color.FromArgb(r, g, b); 29 | } 30 | bmp.SetPixel(x, y, color); 31 | } 32 | } 33 | 34 | 35 | return bmp; 36 | } 37 | 38 | private static byte ToColorComponent(T val) where T : struct, IEquatable, IFormattable 39 | { 40 | var d = Math.Abs((double)Convert.ChangeType(val, typeof(double))) * 255.0; 41 | d = d > 255.0 ? 255.0 : d; 42 | 43 | return (byte)d; 44 | } 45 | } 46 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Utils/img/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbovar/ConvNetSharp/ccaf6f6959c12026c4e1a0e396a7711654c92551/src/ConvNetSharp.Utils/img/icon.png -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume.GPU.Tests/ConvNetSharp.Volume.GPU.Tests.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netcoreapp3.1 5 | false 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | all 15 | runtime; build; native; contentfiles; analyzers; buildtransitive 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume.GPU.Tests/DoubleVolumeTests.cs: -------------------------------------------------------------------------------- 1 | using ConvNetSharp.Volume.GPU.Double; 2 | using ConvNetSharp.Volume.Tests; 3 | using NUnit.Framework; 4 | 5 | namespace ConvNetSharp.Volume.GPU.Tests 6 | { 7 | [TestFixture] 8 | public class DoubleVolumeTests : VolumeTests 9 | { 10 | public DoubleVolumeTests() 11 | { 12 | BuilderInstance.Volume = new VolumeBuilder(); 13 | } 14 | 15 | protected override Volume NewVolume(double[] values, Shape shape) 16 | { 17 | return new Double.Volume(values, shape); 18 | } 19 | } 20 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume.GPU.Tests/SingleVolumeTests.cs: -------------------------------------------------------------------------------- 1 | using System.Linq; 2 | using ConvNetSharp.Volume.GPU.Single; 3 | using ConvNetSharp.Volume.Tests; 4 | using NUnit.Framework; 5 | 6 | namespace ConvNetSharp.Volume.GPU.Tests 7 | { 8 | [TestFixture] 9 | public class SingleVolumeTests : VolumeTests 10 | { 11 | public SingleVolumeTests() 12 | { 13 | BuilderInstance.Volume = new VolumeBuilder(); 14 | } 15 | 16 | protected override Volume NewVolume(double[] values, Shape shape) 17 | { 18 | var converted = values.Select(i => (float)i).ToArray(); 19 | return BuilderInstance.Volume.From(converted, shape); 20 | } 21 | } 22 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume.GPU/ActivationTypeExtension.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using ManagedCuda.CudaDNN; 3 | 4 | namespace ConvNetSharp.Volume.GPU 5 | { 6 | internal static class ActivationTypeExtension 7 | { 8 | public static cudnnActivationMode ToCudnn(this ActivationType type) 9 | { 10 | switch (type) 11 | { 12 | case ActivationType.Sigmoid: 13 | return cudnnActivationMode.Sigmoid; 14 | case ActivationType.Relu: 15 | return cudnnActivationMode.Relu; 16 | case ActivationType.Tanh: 17 | return cudnnActivationMode.Tanh; 18 | case ActivationType.ClippedRelu: 19 | return cudnnActivationMode.ClippedRelu; 20 | } 21 | 22 | throw new NotImplementedException(); 23 | } 24 | } 25 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume.GPU/CudaHostMemoryRegion.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using ManagedCuda; 3 | using ManagedCuda.BasicTypes; 4 | 5 | namespace ConvNetSharp.Volume.GPU 6 | { 7 | internal class CudaHostMemoryRegion : IDisposable 8 | { 9 | private readonly IntPtr _startPointer; 10 | private bool _releasedCudaMemory; 11 | 12 | public CudaHostMemoryRegion(long byteCount) 13 | { 14 | if (byteCount < 0) 15 | { 16 | throw new ArgumentOutOfRangeException(nameof(byteCount)); 17 | } 18 | 19 | this._releasedCudaMemory = true; 20 | this.ByteCount = byteCount; 21 | var result = DriverAPINativeMethods.MemoryManagement.cuMemAllocHost_v2(ref this._startPointer, byteCount); 22 | 23 | if (result != CUResult.Success) 24 | { 25 | throw new CudaException(result); 26 | } 27 | 28 | this._releasedCudaMemory = false; 29 | } 30 | 31 | public IntPtr Start => this._startPointer; 32 | 33 | public long ByteCount { get; } 34 | 35 | public void Dispose() 36 | { 37 | this.ReleaseUnmanagedResources(); 38 | GC.SuppressFinalize(this); 39 | } 40 | 41 | ~CudaHostMemoryRegion() 42 | { 43 | this.ReleaseUnmanagedResources(); 44 | } 45 | 46 | private void ReleaseUnmanagedResources() 47 | { 48 | if (this._releasedCudaMemory) 49 | { 50 | return; 51 | } 52 | 53 | var result = DriverAPINativeMethods.MemoryManagement.cuMemFreeHost(this._startPointer); 54 | if (result != CUResult.Success) 55 | { 56 | throw new CudaException(result); 57 | } 58 | 59 | this._releasedCudaMemory = true; 60 | } 61 | } 62 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume.GPU/DataLocation.cs: -------------------------------------------------------------------------------- 1 | namespace ConvNetSharp.Volume.GPU 2 | { 3 | public enum DataLocation 4 | { 5 | Host, 6 | Device 7 | } 8 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume.GPU/Double/BuilderInstance.cs: -------------------------------------------------------------------------------- 1 | namespace ConvNetSharp.Volume.GPU.Double 2 | { 3 | public class BuilderInstance : BuilderInstance 4 | { 5 | } 6 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume.GPU/Double/Kernels/concat.cu: -------------------------------------------------------------------------------- 1 | extern "C" { 2 | __global__ void Run(int n, double* __restrict left, double* __restrict right, double* __restrict output, int elementPerBatch, int threshold, int mode) { 3 | int i = blockIdx.x*blockDim.x + threadIdx.x; 4 | int batch = i / elementPerBatch; 5 | int rest = i % elementPerBatch; 6 | if (i < n) { 7 | if (rest < threshold) { 8 | if (mode == 1) { 9 | output[i] = left[0]; 10 | } else { 11 | output[i] = left[batch * threshold + rest]; 12 | } 13 | } else { 14 | if (mode == 2) { 15 | output[i] = right[0]; 16 | } else { 17 | output[i] = right[batch * (elementPerBatch - threshold) + rest - threshold]; 18 | } 19 | } 20 | } 21 | } 22 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume.GPU/Double/Kernels/div.cu: -------------------------------------------------------------------------------- 1 | extern "C" { 2 | __global__ void Run(int n, double* __restrict left, double* __restrict right, double* __restrict output, int rightIsScalar) { 3 | int i = blockIdx.x*blockDim.x + threadIdx.x; 4 | if (i < n) { 5 | if (rightIsScalar == 1) { 6 | output[i] = left[i] / right[0]; 7 | } else { 8 | output[i] = left[i] / right[i]; 9 | } 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume.GPU/Double/Kernels/exp.cu: -------------------------------------------------------------------------------- 1 | extern "C" { 2 | __global__ void Run(int n, double* __restrict input, double* __restrict output) { 3 | int i = blockIdx.x*blockDim.x + threadIdx.x; 4 | if (i < n) output[i] = exp(input[i]); 5 | } 6 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume.GPU/Double/Kernels/extract.cu: -------------------------------------------------------------------------------- 1 | extern "C" { 2 | __global__ void Run(int n, double* __restrict x, double* __restrict output, int length, int offset, int inputCount) { 3 | int i = blockIdx.x*blockDim.x + threadIdx.x; 4 | 5 | int batch = i / length; 6 | int rest = i % length; 7 | 8 | int batchCount = n / length; 9 | 10 | int inputWidth = inputCount / batchCount; 11 | 12 | if (i < n) { 13 | output[i] = x[batch * inputWidth + rest + offset]; 14 | } 15 | } 16 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume.GPU/Double/Kernels/leakyrelu.cu: -------------------------------------------------------------------------------- 1 | extern "C" { 2 | __global__ void Run(int n, double* __restrict input, double* __restrict output, double alpha) { 3 | int i = blockIdx.x*blockDim.x + threadIdx.x; 4 | if (i < n) output[i] = input[i] > 0 ? input[i] : input[i] * alpha; 5 | } 6 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume.GPU/Double/Kernels/leakyrelu_gradient.cu: -------------------------------------------------------------------------------- 1 | extern "C" { 2 | __global__ void Run(int n, double* __restrict input, double* __restrict gradient, double* __restrict output, double alpha) { 3 | int i = blockIdx.x*blockDim.x + threadIdx.x; 4 | if (i < n) output[i] = gradient[i] * ((input[i] >= 0) + (input[i] < 0) * alpha); 5 | } 6 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume.GPU/Double/Kernels/log.cu: -------------------------------------------------------------------------------- 1 | extern "C" { 2 | __global__ void Run(int n, double* __restrict input, double* __restrict output) { 3 | int i = blockIdx.x*blockDim.x + threadIdx.x; 4 | if (i < n) output[i] = log(input[i]); 5 | } 6 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume.GPU/Double/Kernels/power.cu: -------------------------------------------------------------------------------- 1 | extern "C" { 2 | __global__ void Run(int n, double* __restrict left, double* __restrict right, double* __restrict output, int rightIsScalar) { 3 | int i = blockIdx.x*blockDim.x + threadIdx.x; 4 | if (i < n) { 5 | if (rightIsScalar == 1){ 6 | output[i] = pow(left[i], right[0]); 7 | } else { 8 | output[i] = pow(left[i], right[i]); 9 | } 10 | } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume.GPU/Double/Kernels/tile.cu: -------------------------------------------------------------------------------- 1 | extern "C" { 2 | __global__ void Run(int n, double* __restrict input, double* __restrict output, 3 | int inputWidth, int inputHeight, int inputChannel, int inputBatchsize, 4 | int outputWidth, int outputHeight, int outputChannel, int outputBatchsize) { 5 | int i = blockIdx.x*blockDim.x + threadIdx.x; 6 | if (i < n) { 7 | // compute input coordinates 8 | int channelTotal = inputWidth * inputHeight; 9 | int batchTotal = channelTotal * inputChannel; 10 | 11 | int b = i / batchTotal; 12 | 13 | int c = (i % batchTotal ) / channelTotal; 14 | int y = ((i % batchTotal) % channelTotal) / inputWidth; 15 | int x = ((i % batchTotal) % channelTotal) % inputWidth; 16 | 17 | // use modulus to create tiles 18 | int inputX = x % inputWidth; 19 | int inputY = y % inputHeight; 20 | int inputC = c % inputChannel; 21 | int inputB = b % inputBatchsize; 22 | 23 | output[i] = input[inputX + inputY * inputWidth + inputC * channelTotal + inputB * batchTotal]; 24 | } 25 | } 26 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume.GPU/Double/VolumeBuilder.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace ConvNetSharp.Volume.GPU.Double 4 | { 5 | public class VolumeBuilder : VolumeBuilder 6 | { 7 | public GpuContext Context { get; set; } = GpuContext.Default; 8 | 9 | public override Volume Build(VolumeStorage storage, Shape shape) 10 | { 11 | if (storage is VolumeStorage gpuStorage) 12 | { 13 | return new Volume(new VolumeStorage(gpuStorage, shape)); 14 | } 15 | 16 | throw new NotImplementedException(); 17 | } 18 | 19 | public override Volume Random(Shape shape, double mu = 0, double std = 1.0) 20 | { 21 | return new Volume(new VolumeStorage(RandomUtilities.RandomDoubleArray(shape.TotalLength, mu, std), shape, this.Context)); 22 | } 23 | 24 | public override Volume SameAs(VolumeStorage example, Shape shape) 25 | { 26 | if (example is VolumeStorage gpuStorage) 27 | { 28 | return new Volume(new VolumeStorage(shape, gpuStorage.Context)); 29 | } 30 | 31 | throw new NotImplementedException(); 32 | } 33 | 34 | public override Volume SameAs(VolumeStorage example, double value, Shape shape) 35 | { 36 | if (example is VolumeStorage gpuStorage) 37 | { 38 | return new Volume(new VolumeStorage(new double[shape.TotalLength].Populate(value), shape, gpuStorage.Context)); 39 | } 40 | 41 | throw new NotImplementedException(); 42 | } 43 | 44 | public override Volume From(double[] value, Shape shape) 45 | { 46 | shape.GuessUnknownDimension(value.Length); 47 | 48 | if (shape.TotalLength != value.Length) 49 | { 50 | throw new ArgumentException($"Array size ({value.Length}) and shape ({shape}) are incompatible"); 51 | } 52 | 53 | return new Volume(new VolumeStorage(value, shape, this.Context)); 54 | } 55 | 56 | public override Volume SameAs(Shape shape) 57 | { 58 | return new Volume(new VolumeStorage(shape, this.Context)); 59 | } 60 | } 61 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume.GPU/IVolumeStorage.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using ManagedCuda; 3 | 4 | namespace ConvNetSharp.Volume.GPU 5 | { 6 | public interface IVolumeStorage where T : struct, IEquatable, IFormattable 7 | { 8 | CudaDeviceVariable DeviceBuffer { get; } 9 | 10 | void CopyToDevice(); 11 | } 12 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume.GPU/Single/BuilderInstance.cs: -------------------------------------------------------------------------------- 1 | namespace ConvNetSharp.Volume.GPU.Single 2 | { 3 | public class BuilderInstance : BuilderInstance 4 | { 5 | } 6 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume.GPU/Single/Kernels/concat.cu: -------------------------------------------------------------------------------- 1 | extern "C" { 2 | __global__ void Run(int n, float* __restrict left, float* __restrict right, float* __restrict output, int elementPerBatch, int threshold, int mode) { 3 | int i = blockIdx.x*blockDim.x + threadIdx.x; 4 | int batch = i / elementPerBatch; 5 | int rest = i % elementPerBatch; 6 | if (i < n) { 7 | if (rest < threshold) { 8 | if (mode == 1) { 9 | output[i] = left[0]; 10 | } else { 11 | output[i] = left[batch * threshold + rest]; 12 | } 13 | } else { 14 | if (mode == 2) { 15 | output[i] = right[0]; 16 | } else { 17 | output[i] = right[batch * (elementPerBatch - threshold) + rest - threshold]; 18 | } 19 | } 20 | } 21 | } 22 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume.GPU/Single/Kernels/div.cu: -------------------------------------------------------------------------------- 1 | extern "C" { 2 | __global__ void Run(int n, float* __restrict left, float* __restrict right, float* __restrict output, int rightIsScalar) { 3 | int i = blockIdx.x*blockDim.x + threadIdx.x; 4 | if (i < n) { 5 | if (rightIsScalar == 1) { 6 | output[i] = left[i] / right[0]; 7 | } else { 8 | output[i] = left[i] / right[i]; 9 | } 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume.GPU/Single/Kernels/exp.cu: -------------------------------------------------------------------------------- 1 | extern "C" { 2 | __global__ void Run(int n, float* __restrict input, float* __restrict output) { 3 | int i = blockIdx.x*blockDim.x + threadIdx.x; 4 | if (i < n) output[i] = exp(input[i]); 5 | } 6 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume.GPU/Single/Kernels/extract.cu: -------------------------------------------------------------------------------- 1 | extern "C" { 2 | __global__ void Run(int n, float* __restrict x, float* __restrict output, int length, int offset, int inputCount) { 3 | int i = blockIdx.x*blockDim.x + threadIdx.x; 4 | 5 | int batch = i / length; 6 | int rest = i % length; 7 | 8 | int batchCount = n / length; 9 | 10 | int inputWidth = inputCount / batchCount; 11 | 12 | if (i < n) { 13 | output[i] = x[batch * inputWidth + rest + offset]; 14 | } 15 | } 16 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume.GPU/Single/Kernels/leakyrelu.cu: -------------------------------------------------------------------------------- 1 | extern "C" { 2 | __global__ void Run(int n, float* __restrict input, float* __restrict output, float alpha) { 3 | int i = blockIdx.x*blockDim.x + threadIdx.x; 4 | if (i < n) output[i] = input[i] > 0 ? input[i] : input[i] * alpha; 5 | } 6 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume.GPU/Single/Kernels/leakyrelu_gradient.cu: -------------------------------------------------------------------------------- 1 | extern "C" { 2 | __global__ void Run(int n, float* __restrict input, float* __restrict gradient, float* __restrict output, float alpha) { 3 | int i = blockIdx.x*blockDim.x + threadIdx.x; 4 | if (i < n) output[i] = gradient[i] * ((input[i] >= 0) + (input[i] < 0) * alpha); 5 | } 6 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume.GPU/Single/Kernels/log.cu: -------------------------------------------------------------------------------- 1 | extern "C" { 2 | __global__ void Run(int n, float* __restrict input, float* __restrict output) { 3 | int i = blockIdx.x*blockDim.x + threadIdx.x; 4 | if (i < n) output[i] = log(input[i]); 5 | } 6 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume.GPU/Single/Kernels/power.cu: -------------------------------------------------------------------------------- 1 | extern "C" { 2 | __global__ void Run(int n, float* __restrict left, float* __restrict right, float* __restrict output, int rightIsScalar) { 3 | int i = blockIdx.x*blockDim.x + threadIdx.x; 4 | if (i < n) { 5 | if (rightIsScalar == 1){ 6 | output[i] = pow(left[i], right[0]); 7 | } else { 8 | output[i] = pow(left[i], right[i]); 9 | } 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume.GPU/Single/Kernels/tile.cu: -------------------------------------------------------------------------------- 1 | extern "C" { 2 | __global__ void Run(int n, float* __restrict input, float* __restrict output, 3 | int inputWidth, int inputHeight, int inputChannel, int inputBatchsize, 4 | int outputWidth, int outputHeight, int outputChannel, int outputBatchsize) { 5 | int i = blockIdx.x*blockDim.x + threadIdx.x; 6 | if (i < n) { 7 | // compute input coordinates 8 | int channelTotal = inputWidth * inputHeight; 9 | int batchTotal = channelTotal * inputChannel; 10 | 11 | int b = i / batchTotal; 12 | 13 | int c = (i % batchTotal ) / channelTotal; 14 | int y = ((i % batchTotal) % channelTotal) / inputWidth; 15 | int x = ((i % batchTotal) % channelTotal) % inputWidth; 16 | 17 | // use modulus to create tiles 18 | int inputX = x % inputWidth; 19 | int inputY = y % inputHeight; 20 | int inputC = c % inputChannel; 21 | int inputB = b % inputBatchsize; 22 | 23 | output[i] = input[inputX + inputY * inputWidth + inputC * channelTotal + inputB * batchTotal]; 24 | } 25 | } 26 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume.GPU/Single/VolumeBuilder.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace ConvNetSharp.Volume.GPU.Single 4 | { 5 | public class VolumeBuilder : VolumeBuilder 6 | { 7 | public GpuContext Context { get; set; } = GpuContext.Default; 8 | 9 | public override Volume Build(VolumeStorage storage, Shape shape) 10 | { 11 | if (storage is VolumeStorage gpuStorage) 12 | { 13 | return new Volume(new VolumeStorage(gpuStorage, shape)); 14 | } 15 | 16 | throw new NotImplementedException(); 17 | } 18 | 19 | public override Volume Random(Shape shape, double mu = 0, double std = 1.0) 20 | { 21 | return new Volume(new VolumeStorage(RandomUtilities.RandomSingleArray(shape.TotalLength, mu, std), shape, this.Context)); 22 | } 23 | 24 | public override Volume SameAs(VolumeStorage example, Shape shape) 25 | { 26 | if (example is VolumeStorage gpuStorage) 27 | { 28 | return new Volume(new VolumeStorage(shape, gpuStorage.Context)); 29 | } 30 | 31 | throw new NotImplementedException(); 32 | } 33 | 34 | public override Volume SameAs(VolumeStorage example, float value, Shape shape) 35 | { 36 | if (example is VolumeStorage gpuStorage) 37 | { 38 | return new Volume(new VolumeStorage(new float[shape.TotalLength].Populate(value), shape, gpuStorage.Context)); 39 | } 40 | 41 | throw new NotImplementedException(); 42 | } 43 | 44 | public override Volume From(float[] value, Shape shape) 45 | { 46 | shape.GuessUnknownDimension(value.Length); 47 | 48 | if (shape.TotalLength != value.Length) 49 | { 50 | throw new ArgumentException($"Array size ({value.Length}) and shape ({shape}) are incompatible"); 51 | } 52 | 53 | return new Volume(new VolumeStorage(value, shape, this.Context)); 54 | } 55 | 56 | public override Volume SameAs(Shape shape) 57 | { 58 | return new Volume(new VolumeStorage(shape, this.Context)); 59 | } 60 | } 61 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume.GPU/TensorReduceOpExtension.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using ManagedCuda.CudaDNN; 3 | 4 | namespace ConvNetSharp.Volume.GPU 5 | { 6 | internal static class TensorReduceOpExtension 7 | { 8 | public static cudnnReduceTensorOp ToCudnn(this TensorReduceOp op) 9 | { 10 | switch (op) 11 | { 12 | case TensorReduceOp.Add: 13 | return cudnnReduceTensorOp.Add; 14 | case TensorReduceOp.Mul: 15 | return cudnnReduceTensorOp.Mul; 16 | case TensorReduceOp.Min: 17 | return cudnnReduceTensorOp.Min; 18 | case TensorReduceOp.Max: 19 | return cudnnReduceTensorOp.Max; 20 | case TensorReduceOp.AMax: 21 | return cudnnReduceTensorOp.AMax; 22 | case TensorReduceOp.Avg: 23 | return cudnnReduceTensorOp.Avg; 24 | case TensorReduceOp.Norm1: 25 | return cudnnReduceTensorOp.Norm1; 26 | case TensorReduceOp.Norm2: 27 | return cudnnReduceTensorOp.Norm2; 28 | default: 29 | throw new ArgumentOutOfRangeException(nameof(op), op, null); 30 | } 31 | } 32 | } 33 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume.GPU/img/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbovar/ConvNetSharp/ccaf6f6959c12026c4e1a0e396a7711654c92551/src/ConvNetSharp.Volume.GPU/img/icon.png -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume.Tests/AssertNumber.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using NUnit.Framework; 4 | 5 | namespace ConvNetSharp.Volume.Tests 6 | { 7 | public static class AssertNumber 8 | { 9 | public static void AreEqual(double expected, T actual, double delta = 0) 10 | { 11 | var value = (double)Convert.ChangeType(actual, typeof(double)); 12 | Assert.AreEqual(expected, value, delta); 13 | } 14 | 15 | public static void AreEqual(T expected, T actual, double delta = 0) 16 | { 17 | var expval = (double)Convert.ChangeType(expected, typeof(double)); 18 | var value = (double)Convert.ChangeType(actual, typeof(double)); 19 | Assert.AreEqual(expval, value, delta); 20 | } 21 | 22 | public static void AreSequenceEqual(IEnumerable expected, IEnumerable actual, double delta = 0) 23 | { 24 | using (var enumerator1 = expected.GetEnumerator()) 25 | { 26 | using (var enumerator2 = actual.GetEnumerator()) 27 | { 28 | while (enumerator1.MoveNext()) 29 | { 30 | enumerator2.MoveNext(); 31 | 32 | AreEqual(enumerator1.Current, enumerator2.Current, delta); 33 | } 34 | } 35 | } 36 | } 37 | } 38 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume.Tests/ConvNetSharp.Volume.Tests.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netcoreapp3.1 5 | false 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | all 15 | runtime; build; native; contentfiles; analyzers; buildtransitive 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume.Tests/DoubleVolumeTests.cs: -------------------------------------------------------------------------------- 1 | using ConvNetSharp.Volume.Double; 2 | using NUnit.Framework; 3 | 4 | namespace ConvNetSharp.Volume.Tests 5 | { 6 | [TestFixture] 7 | public class DoubleVolumeTests : VolumeTests 8 | { 9 | protected override Volume NewVolume(double[] values, Shape shape) 10 | { 11 | return BuilderInstance.Volume.From(values, shape); 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume.Tests/GenericVolumeTests.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Linq; 3 | using ConvNetSharp.Volume.Double; 4 | using NUnit.Framework; 5 | 6 | namespace ConvNetSharp.Volume.Tests 7 | { 8 | [TestFixture] 9 | public class GenericVolumeTests 10 | { 11 | [Test] 12 | public void BuildVolumeFromStorageAndShape() 13 | { 14 | var shape = new Shape(2, 2); 15 | var storage = new NcwhVolumeStorage(new[] { 1.0, 2.0, 3.0, 4.0 }, shape); 16 | var volume = BuilderInstance.Volume.Build(storage, shape); 17 | 18 | Assert.IsTrue(storage.ToArray().SequenceEqual(volume.Storage.ToArray())); 19 | } 20 | 21 | [Test] 22 | public void ReShape_UnknownDimension() 23 | { 24 | var volume = BuilderInstance.Volume.From(new[] { 1.0, 2.0, 3.0 }, new Shape(3)); 25 | 26 | var reshaped = volume.ReShape(1, -1); 27 | Assert.AreEqual(reshaped.Shape.TotalLength, 3); 28 | } 29 | 30 | [Test] 31 | public void ReShape_WrongDimension() 32 | { 33 | var volume = BuilderInstance.Volume.From(new[] { 1.0, 2.0, 3.0 }, new Shape(3)); 34 | Assert.Throws(() => volume.ReShape(1, 4)); 35 | } 36 | 37 | [Test] 38 | public void ReShapeKeep() 39 | { 40 | var volume = BuilderInstance.Volume.From(new[] 41 | { 42 | 1.0, 2.0, 43 | 3.0, 4.0, 44 | 45 | 5.0, 6.0, 46 | 7.0, 8.0, 47 | 48 | 9.0, 10.0, 49 | 11.0, 12.0 50 | }, new Shape(2, 2, 1, 3)); 51 | 52 | var reshaped = volume.ReShape(1, 1, Shape.None, Shape.Keep); 53 | 54 | Assert.AreEqual(reshaped.Shape.TotalLength, 12); 55 | Assert.AreEqual(new Shape(1, 1, 4, 3), reshaped.Shape); 56 | } 57 | } 58 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume.Tests/ShapeTests.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using NUnit.Framework; 3 | 4 | namespace ConvNetSharp.Volume.Tests 5 | { 6 | [TestFixture] 7 | public class ShapeTests 8 | { 9 | [Test] 10 | public void GuessUnknownDimension() 11 | { 12 | var shape = new Shape(2, -1); 13 | shape.GuessUnknownDimension(10); 14 | 15 | Assert.AreEqual(5, shape.Dimensions[1]); 16 | } 17 | 18 | [Test] 19 | public void IncompatibleTotalLength() 20 | { 21 | var shape = new Shape(2, -1); 22 | Assert.Throws(() => shape.GuessUnknownDimension(9)); 23 | } 24 | 25 | [Test] 26 | public void NegativeTotalLength() 27 | { 28 | var shape = new Shape(2, -1); 29 | Assert.Throws(() => shape.GuessUnknownDimension(-1)); 30 | } 31 | 32 | [Test] 33 | public void SetDimension() 34 | { 35 | var shape = new Shape(2, 2); 36 | Assert.AreEqual(2, shape.Dimensions[0]); 37 | Assert.AreEqual(2, shape.Dimensions[1]); 38 | Assert.AreEqual(4, shape.TotalLength); 39 | 40 | shape.SetDimension(0, 1); 41 | Assert.AreEqual(1, shape.Dimensions[0]); 42 | Assert.AreEqual(2, shape.Dimensions[1]); 43 | Assert.AreEqual(2, shape.TotalLength); 44 | } 45 | 46 | [Test] 47 | public void ZeroDimension() 48 | { 49 | Assert.Throws(() => new Shape(2, 0)); 50 | } 51 | } 52 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume.Tests/SingleVolumeTests.cs: -------------------------------------------------------------------------------- 1 | using System.Linq; 2 | using ConvNetSharp.Volume.Single; 3 | using NUnit.Framework; 4 | 5 | namespace ConvNetSharp.Volume.Tests 6 | { 7 | [TestFixture] 8 | public class SingleVolumeTests : VolumeTests 9 | { 10 | protected override Volume NewVolume(double[] values, Shape shape) 11 | { 12 | var converted = values.Select(i => (float)i).ToArray(); 13 | return BuilderInstance.Volume.From(converted, shape); 14 | } 15 | } 16 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume/ActivationType.cs: -------------------------------------------------------------------------------- 1 | namespace ConvNetSharp.Volume 2 | { 3 | public enum ActivationType 4 | { 5 | Sigmoid, 6 | Relu, 7 | Tanh, 8 | ClippedRelu 9 | } 10 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume/ArrayExtensions.cs: -------------------------------------------------------------------------------- 1 | namespace ConvNetSharp.Volume 2 | { 3 | public static class ArrayExtensions 4 | { 5 | public static T[] Populate(this T[] arr, T value) 6 | { 7 | for (var i = 0; i < arr.Length; i++) 8 | { 9 | arr[i] = value; 10 | } 11 | 12 | return arr; 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume/BuilderInstance.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace ConvNetSharp.Volume 4 | { 5 | public class BuilderInstance where T : struct, IEquatable, IFormattable 6 | { 7 | private static readonly Lazy> Singleton = new Lazy>(Create); 8 | 9 | public static VolumeBuilder Volume { get; set; } = Singleton.Value; 10 | 11 | public static VolumeBuilder Create() 12 | { 13 | if (typeof(T) == typeof(double)) 14 | { 15 | return (VolumeBuilder)(object)new Double.VolumeBuilder(); 16 | } 17 | if (typeof(T) == typeof(float)) 18 | { 19 | return (VolumeBuilder)(object)new Single.VolumeBuilder(); 20 | } 21 | 22 | throw new NotSupportedException( 23 | $"Volumes of type '{typeof(T).Name}' are not supported. Only Double and Single are supported."); 24 | } 25 | } 26 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume/ConvNetSharp.Volume.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netstandard2.1 5 | 6 | 7 | 8 | 0.4.14 9 | 0.4.14.0 10 | Cedric Bovar 11 | Deep learning in C# 12 | Copyright (c) 2020 13 | 14 | 15 | 16 | Cognitio.ConvNetSharp.Volume 17 | 0.4.14 18 | false 19 | Deep Learning Convolution AI 20 | https://github.com/cbovar/ConvNetSharp/blob/master/LICENSE 21 | https://github.com/cbovar/ConvNetSharp 22 | icon.png 23 | True 24 | https://github.com/cbovar/ConvNetSharp/releases/tag/v0.4.14 25 | 0.4.14.0 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume/Double/BuilderInstance.cs: -------------------------------------------------------------------------------- 1 | namespace ConvNetSharp.Volume.Double 2 | { 3 | public class BuilderInstance : BuilderInstance 4 | { 5 | } 6 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume/Double/VolumeBuilder.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace ConvNetSharp.Volume.Double 4 | { 5 | public class VolumeBuilder : VolumeBuilder 6 | { 7 | public override Volume Build(VolumeStorage storage, Shape shape) 8 | { 9 | if (storage is NcwhVolumeStorage ncwh) 10 | { 11 | return new Volume(ncwh.ReShape(shape)); 12 | } 13 | 14 | throw new NotImplementedException(); 15 | } 16 | 17 | public override Volume From(double[] value, Shape shape) 18 | { 19 | shape.GuessUnknownDimension(value.Length); 20 | 21 | if (shape.TotalLength != value.Length) 22 | { 23 | throw new ArgumentException($"Array size ({value.Length}) and shape ({shape}) are incompatible"); 24 | } 25 | 26 | return new Volume(new NcwhVolumeStorage(value, shape)); 27 | } 28 | 29 | public override Volume Random(Shape shape, double mu = 0, double std = 1.0) 30 | { 31 | var vol = new Volume(new NcwhVolumeStorage(shape)); 32 | 33 | for (var n = 0; n < shape.Dimensions[3]; n++) 34 | { 35 | for (var c = 0; c < shape.Dimensions[2]; c++) 36 | { 37 | for (var y = 0; y < shape.Dimensions[1]; y++) 38 | { 39 | for (var x = 0; x < shape.Dimensions[0]; x++) 40 | { 41 | vol.Set(x, y, c, n, RandomUtilities.Randn(mu, std)); 42 | } 43 | } 44 | } 45 | } 46 | 47 | return vol; 48 | } 49 | 50 | public override Volume SameAs(VolumeStorage example, Shape shape) 51 | { 52 | if (example is NcwhVolumeStorage) 53 | { 54 | return new Volume(new NcwhVolumeStorage(shape)); 55 | } 56 | 57 | throw new NotImplementedException(); 58 | } 59 | 60 | public override Volume SameAs(VolumeStorage example, double value, Shape shape) 61 | { 62 | if (example is NcwhVolumeStorage) 63 | { 64 | return new Volume(new NcwhVolumeStorage(new double[shape.TotalLength].Populate(value), shape)); 65 | } 66 | 67 | throw new NotImplementedException(); 68 | } 69 | 70 | public override Volume SameAs(Shape shape) 71 | { 72 | return new Volume(new NcwhVolumeStorage(shape)); 73 | } 74 | } 75 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume/RandomUtilities.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace ConvNetSharp.Volume 4 | { 5 | public static class RandomUtilities 6 | { 7 | private static readonly Random Random = new Random(Seed); 8 | private static double val; 9 | private static bool returnVal; 10 | 11 | public static int Seed => 123; //DateTime.Now.Ticks; 12 | 13 | public static double GaussianRandom() 14 | { 15 | if (returnVal) 16 | { 17 | returnVal = false; 18 | return val; 19 | } 20 | 21 | double r = 0, u = 0, v = 0; 22 | 23 | //System.Random is not thread-safe 24 | lock (Random) 25 | { 26 | while (r == 0 || r > 1) 27 | { 28 | u = 2 * Random.NextDouble() - 1; 29 | v = 2 * Random.NextDouble() - 1; 30 | r = u * u + v * v; 31 | } 32 | } 33 | 34 | var c = Math.Sqrt(-2 * Math.Log(r) / r); 35 | val = v * c; // cache this 36 | returnVal = true; 37 | 38 | return u * c; 39 | } 40 | 41 | public static double NextDouble() 42 | { 43 | return Random.NextDouble(); 44 | } 45 | 46 | public static double Randn(double mu, double std) 47 | { 48 | return mu + GaussianRandom() * std; 49 | } 50 | 51 | public static double[] RandomDoubleArray(long length, double mu = 0.0, double std = 1.0, bool posisitveOnly = false) 52 | { 53 | var values = new double[length]; 54 | 55 | for (var i = 0; i < length; i++) 56 | { 57 | values[i] = Randn(mu, std); 58 | if (posisitveOnly) 59 | { 60 | values[i] = Math.Abs(values[i]); 61 | } 62 | } 63 | 64 | return values; 65 | } 66 | 67 | public static float[] RandomSingleArray(long length, double mu = 0.0, double std = 1.0, bool posisitveOnly = false) 68 | { 69 | var values = new float[length]; 70 | 71 | for (var i = 0; i < length; i++) 72 | { 73 | values[i] = (float)Randn(mu, std); 74 | if (posisitveOnly) 75 | { 76 | values[i] = Math.Abs(values[i]); 77 | } 78 | } 79 | 80 | return values; 81 | } 82 | } 83 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume/ShapeExtension.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace ConvNetSharp.Volume 4 | { 5 | public static class ShapeExtension 6 | { 7 | private static T ToT(object o) where T : struct, IEquatable, IFormattable 8 | { 9 | return (T)Convert.ChangeType(o, typeof(T)); 10 | } 11 | 12 | public static Volume ToVolume(this Shape shape) where T : struct, IEquatable, IFormattable 13 | { 14 | var builder = BuilderInstance.Create(); 15 | var vol = builder.From(new[] { ToT(shape.Dimensions[0]), ToT(shape.Dimensions[1]), ToT(shape.Dimensions[2]), ToT(shape.Dimensions[3]) }, 16 | new Shape(4)); 17 | 18 | return vol; 19 | } 20 | } 21 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume/Single/BuilderInstance.cs: -------------------------------------------------------------------------------- 1 | namespace ConvNetSharp.Volume.Single 2 | { 3 | public class BuilderInstance : BuilderInstance 4 | { 5 | } 6 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume/Single/VolumeBuilder.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace ConvNetSharp.Volume.Single 4 | { 5 | public class VolumeBuilder : VolumeBuilder 6 | { 7 | public override Volume Build(VolumeStorage storage, Shape shape) 8 | { 9 | if (storage is NcwhVolumeStorage ncwh) 10 | { 11 | return new Volume(ncwh.ReShape(shape)); 12 | } 13 | 14 | throw new NotImplementedException(); 15 | } 16 | 17 | public override Volume From(float[] value, Shape shape) 18 | { 19 | shape.GuessUnknownDimension(value.Length); 20 | 21 | if (shape.TotalLength != value.Length) 22 | { 23 | throw new ArgumentException($"Array size ({value.Length}) and shape ({shape}) are incompatible"); 24 | } 25 | 26 | return new Volume(new NcwhVolumeStorage(value, shape)); 27 | } 28 | 29 | public override Volume Random(Shape shape, double mu = 0, double std = 1.0) 30 | { 31 | return new Volume(new NcwhVolumeStorage(RandomUtilities.RandomSingleArray(shape.TotalLength, mu, std), shape)); 32 | } 33 | 34 | public override Volume SameAs(VolumeStorage example, Shape shape) 35 | { 36 | if (example is NcwhVolumeStorage) 37 | { 38 | return new Volume(new NcwhVolumeStorage(shape)); 39 | } 40 | 41 | throw new NotImplementedException(); 42 | } 43 | 44 | public override Volume SameAs(VolumeStorage example, float value, Shape shape) 45 | { 46 | if (example is NcwhVolumeStorage) 47 | { 48 | return new Volume(new NcwhVolumeStorage(new float[shape.TotalLength].Populate(value), shape)); 49 | } 50 | 51 | throw new NotImplementedException(); 52 | } 53 | 54 | public override Volume SameAs(Shape shape) 55 | { 56 | return new Volume(new NcwhVolumeStorage(shape)); 57 | } 58 | } 59 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume/TensorReduceOp.cs: -------------------------------------------------------------------------------- 1 | namespace ConvNetSharp.Volume 2 | { 3 | public enum TensorReduceOp 4 | { 5 | Add, 6 | Mul, 7 | Min, 8 | Max, 9 | AMax, 10 | Avg, 11 | Norm1, 12 | Norm2 13 | } 14 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume/VolumeBuilder.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace ConvNetSharp.Volume 4 | { 5 | /// 6 | /// TODO: simplify 7 | /// 8 | /// 9 | public abstract class VolumeBuilder where T : struct, IEquatable, IFormattable 10 | { 11 | public abstract Volume SameAs(Shape shape); 12 | 13 | public abstract Volume From(T[] value, Shape shape); 14 | 15 | public abstract Volume Random(Shape shape, double mu = 0, double std = 1.0); 16 | 17 | public abstract Volume SameAs(VolumeStorage example, Shape shape); 18 | 19 | /// 20 | /// Creates a volume with given shape, using provided storage as internal storage (no copy) 21 | /// 22 | /// 23 | /// 24 | /// 25 | public abstract Volume Build(VolumeStorage storage, Shape shape); 26 | 27 | /// 28 | /// Creates a volume with given shape, filled with the provided value and using the same storage type as provided 29 | /// example 30 | /// 31 | /// 32 | /// 33 | /// 34 | /// 35 | public abstract Volume SameAs(VolumeStorage example, T value, Shape shape); 36 | } 37 | } -------------------------------------------------------------------------------- /src/ConvNetSharp.Volume/img/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbovar/ConvNetSharp/ccaf6f6959c12026c4e1a0e396a7711654c92551/src/ConvNetSharp.Volume/img/icon.png -------------------------------------------------------------------------------- /src/external/ManagedCuda/CudaBlas.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbovar/ConvNetSharp/ccaf6f6959c12026c4e1a0e396a7711654c92551/src/external/ManagedCuda/CudaBlas.dll -------------------------------------------------------------------------------- /src/external/ManagedCuda/CudaDNN.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbovar/ConvNetSharp/ccaf6f6959c12026c4e1a0e396a7711654c92551/src/external/ManagedCuda/CudaDNN.dll -------------------------------------------------------------------------------- /src/external/ManagedCuda/ManagedCuda.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbovar/ConvNetSharp/ccaf6f6959c12026c4e1a0e396a7711654c92551/src/external/ManagedCuda/ManagedCuda.dll -------------------------------------------------------------------------------- /src/external/ManagedCuda/NVRTC.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbovar/ConvNetSharp/ccaf6f6959c12026c4e1a0e396a7711654c92551/src/external/ManagedCuda/NVRTC.dll --------------------------------------------------------------------------------