] InformationalVersion = "0.9.0.5"
15 |
--------------------------------------------------------------------------------
/build.bat:
--------------------------------------------------------------------------------
1 | @echo off
2 | cls
3 | .paket\paket.exe restore
4 | packages\FAKE\tools\FAKE.exe build.fsx %*
--------------------------------------------------------------------------------
/gallery.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | Alea TK
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 | Redirecting to latest version 0.9.0-beta3 ...
13 |
14 |
15 |
16 |
--------------------------------------------------------------------------------
/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | Alea TK
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 | Redirecting to latest version 0.9.0-beta3 ...
13 |
14 |
15 |
16 |
--------------------------------------------------------------------------------
/paket.dependencies:
--------------------------------------------------------------------------------
1 | content: none
2 | source https://www.nuget.org/api/v2
3 | source http://nuget.aleagpu.com/nuget
4 | source http://beta.aleagpu.com/nuget
5 | nuget Alea
6 | nuget CSMatIO
7 | nuget Fake
8 | nuget Nuget.CommandLine
9 | nuget NUnit 2.6.4
10 | nuget NUnit.Runners.Net4
11 | nuget SharpZipLib
12 | nuget Templatus
--------------------------------------------------------------------------------
/paket.lock:
--------------------------------------------------------------------------------
1 | CONTENT: NONE
2 | NUGET
3 | remote: https://www.nuget.org/api/v2
4 | Alea (3.0.2)
5 | CSMatIO (1.0.2)
6 | zlib.net (>= 1.0.4)
7 | FAKE (4.28)
8 | NuGet.CommandLine (3.3)
9 | NUnit (2.6.4)
10 | NUnit.Runners.Net4 (2.6.4)
11 | SharpZipLib (0.86)
12 | Templatus (0.3)
13 | zlib.net (1.0.4)
14 |
--------------------------------------------------------------------------------
/src/AleaTK/AleaTK.csproj:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Debug
6 | AnyCPU
7 | {EC47C435-0D7D-4819-BE05-11B9E57FB8F3}
8 | Library
9 | Properties
10 | AleaTK
11 | AleaTK
12 | v4.5.2
13 | 512
14 |
15 |
16 | true
17 | full
18 | false
19 | ..\..\debug\
20 | DEBUG;TRACE
21 | prompt
22 | 4
23 | true
24 | ..\..\debug\AleaTK.XML
25 | 1591
26 |
27 |
28 | pdbonly
29 | true
30 | ..\..\release\
31 | TRACE
32 | prompt
33 | 4
34 | true
35 | ..\..\release\AleaTK.XML
36 | 1591
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 | Properties\SolutionInfo.cs
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
70 |
71 |
72 |
73 |
74 |
75 |
76 |
77 |
78 |
79 |
80 |
81 |
82 |
83 |
84 |
85 |
86 |
87 |
88 |
89 |
90 |
91 |
92 |
99 |
100 |
101 |
102 |
103 |
104 | ..\..\packages\Alea\lib\net45\Alea.IL.dll
105 | True
106 | True
107 |
108 |
109 | ..\..\packages\Alea\lib\net45\Alea.Parallel.dll
110 | True
111 | True
112 |
113 |
114 | ..\..\packages\Alea\lib\net45\Alea.dll
115 | True
116 | True
117 |
118 |
119 |
120 |
121 |
122 |
--------------------------------------------------------------------------------
/src/AleaTK/ExprImpl/MapExpr.cs:
--------------------------------------------------------------------------------
1 | using System;
2 |
3 | namespace AleaTK.ExprImpl
4 | {
5 | public class Map1Expr : RExpr
6 | {
7 | public Map1Expr(Expr input, Func transform, string opCode = OpCodes.Map1)
8 | {
9 | Shape = input.Shape;
10 | OpCode = opCode;
11 | Input = input;
12 | Transform = transform;
13 | AddOperand(Input);
14 | }
15 |
16 | public Expr Input { get; }
17 |
18 | public Func Transform { get; }
19 |
20 | public override Shape Shape { get; }
21 |
22 | protected override IRValue GenerateRValue(Assignment assignment)
23 | {
24 | var device = assignment.Context.Device;
25 | var input = assignment.GetInput(Input).ToRValue();
26 | var transform = Transform;
27 | var layout = input.Layout;
28 | var inputRawReader = input.BufferReader.RawReader;
29 | Func rawReader = i => transform(inputRawReader(i));
30 | return new TensorReader(device, layout, rawReader);
31 | }
32 | }
33 |
34 | public class Map2Expr : RExpr
35 | {
36 | public Map2Expr(Expr input1, Expr input2, Func transform, string opCode = OpCodes.Map2)
37 | {
38 | Input1 = input1;
39 | Input2 = input2;
40 | Transform = transform;
41 | OpCode = opCode;
42 | Shape = Shape.Broadcast(input1.Shape, input2.Shape);
43 | AddOperand(Input1);
44 | AddOperand(Input2);
45 | }
46 |
47 | public Expr Input1 { get; }
48 |
49 | public Expr Input2 { get; }
50 |
51 | public Func Transform { get; }
52 |
53 | public override Shape Shape { get; }
54 |
55 | public override void Prepare(Assignment assignment)
56 | {
57 | assignment.RequireLayoutInnerChangeMost(Input1);
58 | assignment.RequireLayoutInnerChangeMost(Input2);
59 | assignment.RequireLayoutFullyUnitStride(Input1);
60 | assignment.RequireLayoutFullyUnitStride(Input2);
61 | }
62 |
63 | protected override IRValue GenerateRValue(Assignment assignment)
64 | {
65 | var device = assignment.Context.Device;
66 | var input1 = assignment.GetInput(Input1).ToRValue();
67 | var input2 = assignment.GetInput(Input2).ToRValue();
68 | var transform = Transform;
69 | var shape = Shape;
70 |
71 | if (Layout.CanFullyUnitStrideMapping(input1.Layout, input2.Layout))
72 | {
73 | var read1 = input1.BufferReader.GetFlatReader1(shape);
74 | var read2 = input2.BufferReader.GetFlatReader1(shape);
75 | var layout = new Layout(shape);
76 | Func rawReader = i => transform(read1(i), read2(i));
77 | return new TensorReader(device, layout, rawReader);
78 | }
79 |
80 | throw new NotImplementedException();
81 | }
82 | }
83 | }
--------------------------------------------------------------------------------
/src/AleaTK/ExprImpl/RandomExpr.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using Alea;
3 | using Alea.cuRAND;
4 |
5 | namespace AleaTK.ExprImpl
6 | {
7 | public class PseudoRandomExpr : LExpr
8 | {
9 | public PseudoRandomExpr(Shape shape, PseudoRandomType type, Distribution distribution, ulong seed, ulong offset, string opCode = OpCodes.Random)
10 | {
11 | OpCode = opCode;
12 | Shape = shape;
13 | Seed = seed;
14 | Distribution = distribution;
15 | Offset = offset;
16 | Type = type;
17 | }
18 |
19 | public PseudoRandomType Type { get; }
20 |
21 | public Distribution Distribution { get; }
22 |
23 | public ulong Seed { get; }
24 |
25 | public ulong Offset { get; }
26 |
27 | public override Shape Shape { get; }
28 |
29 | protected override bool Execute(Assignment assignment, ILValue output)
30 | {
31 | var ctx = assignment.Context;
32 | var rng = ctx.GetRandomGenerator(Type);
33 |
34 | lock (rng)
35 | {
36 | rng.SetPseudoRandomGeneratorSeed(Seed);
37 | rng.SetGeneratorOrdering(Ordering.PSEUDO_DEFAULT);
38 | rng.SetGeneratorOffset(Offset);
39 |
40 | if (typeof(T) == typeof(double))
41 | {
42 | var ptr = output.Buffer.Ptr.Reinterpret();
43 |
44 | if (Distribution is UniformDistribution)
45 | {
46 | rng.GenerateUniform(ptr, (ulong)output.Layout.Shape.Length);
47 | return true;
48 | }
49 |
50 | if (Distribution is NormalDistribution)
51 | {
52 | var dist = Distribution as NormalDistribution;
53 | rng.GenerateNormal(ptr, (ulong)output.Layout.Shape.Length, dist.Mean, dist.Stddev);
54 | return true;
55 | }
56 |
57 | throw new InvalidOperationException();
58 | }
59 |
60 | if (typeof(T) == typeof(float))
61 | {
62 | var ptr = output.Buffer.Ptr.Reinterpret();
63 |
64 | if (Distribution is UniformDistribution)
65 | {
66 | rng.GenerateUniform(ptr, (ulong)output.Layout.Shape.Length);
67 | return true;
68 | }
69 |
70 | if (Distribution is NormalDistribution)
71 | {
72 | var dist = Distribution as NormalDistribution;
73 | rng.GenerateNormal(ptr, (ulong)output.Layout.Shape.Length, (float)dist.Mean, (float)dist.Stddev);
74 | return true;
75 | }
76 |
77 | throw new InvalidOperationException();
78 | }
79 |
80 | if (typeof(T) == typeof(double2))
81 | {
82 | var ptr = output.Buffer.Ptr.Reinterpret();
83 |
84 | if (Distribution is UniformDistribution)
85 | {
86 | rng.GenerateUniform(ptr, (ulong)output.Layout.Shape.Length * 2UL);
87 | return true;
88 | }
89 |
90 | if (Distribution is NormalDistribution)
91 | {
92 | var dist = Distribution as NormalDistribution;
93 | rng.GenerateNormal(ptr, (ulong)output.Layout.Shape.Length * 2UL, dist.Mean, dist.Stddev);
94 | return true;
95 | }
96 |
97 | throw new InvalidOperationException();
98 | }
99 |
100 | if (typeof(T) == typeof(uint) || typeof(T) == typeof(int))
101 | {
102 | var ptr = output.Buffer.Ptr.Reinterpret();
103 |
104 | if (Distribution is UniformDistribution)
105 | {
106 | rng.Generate(ptr, (ulong)output.Layout.Shape.Length);
107 | return true;
108 | }
109 |
110 | throw new InvalidOperationException();
111 | }
112 | }
113 |
114 | return false;
115 | }
116 | }
117 | }
--------------------------------------------------------------------------------
/src/AleaTK/ExprImpl/ScalarExpr.cs:
--------------------------------------------------------------------------------
1 | using System;
2 |
3 | namespace AleaTK.ExprImpl
4 | {
5 | public class ScalarExpr : RExpr
6 | {
7 | public ScalarExpr(Shape shape, T value)
8 | {
9 | OpCode = OpCodes.Scalar;
10 | Shape = shape;
11 | Value = value;
12 | }
13 |
14 | public ScalarExpr(T value)
15 | {
16 | OpCode = OpCodes.Scalar;
17 | Shape = Shape.Scalar;
18 | Value = value;
19 | }
20 |
21 | public T Value { get; }
22 |
23 | public override Shape Shape { get; }
24 |
25 | protected override IRValue GenerateRValue(Assignment assignment)
26 | {
27 | var device = assignment.Context.Device;
28 | var layout = new Layout(Shape);
29 | var value = Value;
30 | Func rawReader = _ => value;
31 | return new TensorReader(device, layout, rawReader);
32 | }
33 | }
34 | }
--------------------------------------------------------------------------------
/src/AleaTK/ExprImpl/ShapeExpr.cs:
--------------------------------------------------------------------------------
1 | using System.Linq;
2 |
3 | namespace AleaTK.ExprImpl
4 | {
5 | public class ReShapeExpr : RExpr
6 | {
7 | public ReShapeExpr(Expr input, long[] dims, string opCode = OpCodes.ReShape)
8 | {
9 | // -1 means calc the shape, but only one -1 allowed.
10 | var numNegOne = dims.Select(x => x < 0 ? 1 : 0).Sum();
11 | Util.EnsureTrue(numNegOne == 0 || numNegOne == 1);
12 |
13 | if (numNegOne == 0)
14 | {
15 | var shape = new Shape(dims);
16 | // length must match old one
17 | Util.EnsureEqual(input.Shape.Length, shape.Length);
18 | Shape = shape;
19 | }
20 | else
21 | {
22 | var remainLength = dims.Select(x => x >= 0 ? x : 1L).Aggregate(ScalarOps.Mul);
23 | for (var i = 0; i < dims.Length; ++i)
24 | {
25 | if (dims[i] < 0)
26 | {
27 | dims[i] = input.Shape.Length/remainLength;
28 | break;
29 | }
30 | }
31 | // check if it is multiply correct
32 | var shape = new Shape(dims);
33 | Util.EnsureEqual(input.Shape.Length, shape.Length);
34 | Shape = shape;
35 | }
36 |
37 | Input = input;
38 | AddOperand(Input);
39 | }
40 |
41 | public Expr Input { get; }
42 |
43 | public override Shape Shape { get; }
44 |
45 | protected override IRValue GenerateRValue(Assignment assignment)
46 | {
47 | var device = assignment.Context.Device;
48 | var input = assignment.GetInput(Input).ToRValue();
49 | var shape = Shape;
50 |
51 | if (input.Layout.IsInnerChangeMost && input.Layout.IsFullyUnitStride)
52 | {
53 | var read = input.BufferReader.GetFlatReader1();
54 | var layout = new Layout(shape);
55 | return new TensorReader(device, layout, read);
56 | }
57 |
58 | return null;
59 | }
60 | }
61 | }
62 |
--------------------------------------------------------------------------------
/src/AleaTK/ExprImpl/TakeExpr.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Collections.Generic;
3 | using System.Linq;
4 | using System.Text;
5 | using System.Threading.Tasks;
6 |
7 | namespace AleaTK.ExprImpl
8 | {
9 | public class TakeExpr : RExpr
10 | {
11 | public TakeExpr(Expr indices, Expr source)
12 | {
13 | Indices = indices;
14 | Source = source;
15 | Util.EnsureTrue(Source.Shape.Rank == 2);
16 | Shape = Shape.Create(Indices.Shape.Concat(new[] {Source.Shape[1]}).ToArray());
17 | AddOperand(Indices);
18 | AddOperand(Source);
19 | }
20 |
21 | public Expr Indices { get; }
22 |
23 | public Expr Source { get; }
24 |
25 | public override Shape Shape { get; }
26 |
27 | protected override IRValue GenerateRValue(Assignment assignment)
28 | {
29 |
30 | var device = assignment.Context.Device;
31 | var indices = assignment.GetInput(Indices).ToRValue();
32 | var source = assignment.GetInput(Source).ToRValue();
33 | Util.EnsureTrue(indices.Layout.IsInnerChangeMostFullyPacked);
34 |
35 | var indicesReader = indices.BufferReader.RawReader;
36 | var sourceReader = source.BufferReader.GetReader2();
37 | var layout = new Layout(Shape);
38 | var dim = source.Layout.Shape[1];
39 | Func rawReader = i => sourceReader(indicesReader(i/dim), i%dim);
40 | return new TensorReader(device, layout, rawReader);
41 | }
42 | }
43 |
44 | public class TakeGradExpr : RExpr
45 | {
46 | public TakeGradExpr(Expr indices, Expr outputGradient, int sourceRows, T zero, Func add)
47 | {
48 | Indices = indices;
49 | OutputGradient = outputGradient;
50 | SourceRows = sourceRows;
51 | Zero = zero;
52 | Add = add;
53 | Shape = Shape.Create(SourceRows, OutputGradient.Shape[OutputGradient.Shape.Rank - 1]);
54 | AddOperand(Indices);
55 | AddOperand(OutputGradient);
56 | }
57 |
58 | public Expr Indices { get; }
59 |
60 | public Expr OutputGradient { get; }
61 |
62 | public int SourceRows { get; }
63 |
64 | public override Shape Shape { get; }
65 |
66 | public T Zero { get; }
67 |
68 | public Func Add { get; }
69 |
70 | protected override IRValue GenerateRValue(Assignment assignment)
71 | {
72 | var device = assignment.Context.Device;
73 | var indices = assignment.GetInput(Indices).ToRValue();
74 | var dOutput = assignment.GetInput(OutputGradient).ToRValue();
75 | Util.EnsureTrue(indices.Layout.IsInnerChangeMostFullyPacked);
76 | Util.EnsureTrue(dOutput.Layout.IsInnerChangeMostFullyPacked);
77 |
78 | var indicesReader = indices.BufferReader.RawReader;
79 | var dOutputReader = dOutput.BufferReader.RawReader;
80 | var layout = new Layout(Shape);
81 | var sourceRows = SourceRows;
82 | var batchSize = Indices.Shape.Length;
83 | var zero = Zero;
84 | var add = Add;
85 | var dims = Shape[1];
86 |
87 | Func rawReader = i =>
88 | {
89 | var row = i/ dims;
90 | var col = i% dims;
91 | var ret = zero;
92 | for (var j = 0; j < batchSize; ++j)
93 | {
94 | var idx = indicesReader(j);
95 | if (idx == row)
96 | {
97 | var value = dOutputReader(j*dims + col);
98 | ret = add(ret, value);
99 | }
100 | }
101 | return ret;
102 | };
103 |
104 | return new TensorReader(device, layout, rawReader);
105 | }
106 | }
107 | }
108 |
--------------------------------------------------------------------------------
/src/AleaTK/ML/Batcher.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Linq;
3 | using Alea;
4 | using Alea.Parallel.Device;
5 |
6 | namespace AleaTK.ML {
7 | public class Batcher: IDisposable {
8 | [GpuParam]
9 | private readonly int cols_, rows_, outputs_;
10 | private readonly bool doReset_;
11 |
12 | public long Index { get; private set; }
13 | public int Rows => rows_;
14 | public int Cols => cols_;
15 | public int Outputs => outputs_;
16 |
17 | public Batcher(Context context, float[,] data, float[,] labels, bool doReset=true) {
18 | doReset_ = doReset;
19 | Context = context;
20 | Random = new Random(0);
21 | rows_ = data.GetLength(0);
22 | cols_ = data.GetLength(1);
23 | outputs_ = labels.GetLength(1);
24 |
25 | Indices = Enumerable.Range(0, Rows).ToArray();
26 | Data = data;
27 | Labels = labels;
28 |
29 | IndicesTensor = context.Allocate(Indices);
30 | DataTensor1 = context.Allocate(data);
31 | LabelsTensor1 = context.Allocate(labels);
32 | DataTensor2 = context.Device.Allocate(Shape.Create(Rows, Cols));
33 | LabelsTensor2 = context.Device.Allocate(Shape.Create(Rows, Outputs));
34 | DataTensor = DataTensor1;
35 | LabelsTensor = LabelsTensor1;
36 |
37 | if (!doReset_) return;
38 | Index = -1;
39 | Reset();
40 | }
41 |
42 | #region props
43 |
44 | public Context Context { get; }
45 | public Random Random { get; }
46 | public int[] Indices { get; }
47 | public float[,] Data { get; }
48 | public float[,] Labels { get; }
49 | public Tensor IndicesTensor { get; }
50 | public Tensor DataTensor { get; private set; }
51 | public Tensor DataTensor1 { get; }
52 | public Tensor DataTensor2 { get; }
53 | public Tensor LabelsTensor { get; private set; }
54 | public Tensor LabelsTensor1 { get; }
55 | public Tensor LabelsTensor2 { get; }
56 |
57 | #endregion
58 |
59 | private void ShuffleIndices() {
60 | var rng = Random;
61 | var array = Indices;
62 | var n = array.Length;
63 | while (n > 1) {
64 | var k = rng.Next(n--);
65 | var temp = array[n];
66 | array[n] = array[k];
67 | array[k] = temp;
68 | }
69 | }
70 |
71 | public void Reset() {
72 | if (!doReset_ || Index == 0L || Context.Type != ContextType.Gpu) return;
73 |
74 | Index = 0L;
75 | ShuffleIndices();
76 | Context.Copy(IndicesTensor, Indices.AsTensor());
77 | var stream = Context.ToGpuContext().Stream;
78 | var srcData = DataTensor == DataTensor1 ? DataTensor1.Buffer.Ptr : DataTensor2.Buffer.Ptr;
79 | var dstData = DataTensor == DataTensor1 ? DataTensor2.Buffer.Ptr : DataTensor1.Buffer.Ptr;
80 | var srcLabels = LabelsTensor == LabelsTensor1 ? LabelsTensor1.Buffer.Ptr : LabelsTensor2.Buffer.Ptr;
81 | var dstLabels = LabelsTensor == LabelsTensor1 ? LabelsTensor2.Buffer.Ptr : LabelsTensor1.Buffer.Ptr;
82 | var idx = IndicesTensor.Buffer.Ptr;
83 | DeviceFor.For(stream, 0, Rows, i => {
84 | var j = idx[i];
85 | var srcDataOffset = srcData + i*cols_;
86 | var dstDataOffset = dstData + j* cols_;
87 | for (var k = 0; k < cols_; ++k) dstDataOffset[k] = srcDataOffset[k];
88 | var srcLabelsOffset = srcLabels + i*outputs_;
89 | var dstLabelsOffset = dstLabels + j* outputs_;
90 | for (var k = 0; k < outputs_; ++k) dstLabelsOffset[k] = srcLabelsOffset[k];
91 | });
92 | DataTensor = DataTensor == DataTensor1 ? DataTensor2 : DataTensor1;
93 | LabelsTensor = LabelsTensor == LabelsTensor1 ? LabelsTensor2 : LabelsTensor1;
94 | }
95 |
96 | public static Buffer CreateBuffer(Tensor t, long rows, int cols, long idx) {
97 | return new Buffer(t.Device, t.Memory, new Layout(Shape.Create(rows, cols)), t.Buffer.Ptr.LongPtr(idx * cols));
98 | }
99 |
100 | public bool Next(long batchSize, Executor executor, Variable dataVar, Variable labelsVar) {
101 | if (Index >= Rows) {
102 | Reset();
103 | return false;
104 | }
105 | var size = Index + batchSize >= Rows ? Rows - Index : batchSize;
106 | var dataBuffer = CreateBuffer(DataTensor, size, Cols, Index);
107 | var labelsBuffer = CreateBuffer(LabelsTensor, size, Outputs, Index);
108 | Index += batchSize;
109 |
110 | executor.SetTensor(dataVar, new Tensor(dataBuffer));
111 | executor.SetTensor(labelsVar, new Tensor(labelsBuffer));
112 | return true;
113 | }
114 |
115 | public void Dispose() {
116 | foreach (var t in new[] {DataTensor, DataTensor1, DataTensor2, LabelsTensor1, LabelsTensor2}) t.Buffer.Dispose();
117 | IndicesTensor.Buffer.Dispose();
118 | }
119 | }
120 | }
--------------------------------------------------------------------------------
/src/AleaTK/ML/Differentiable.cs:
--------------------------------------------------------------------------------
1 | using System.Collections.Generic;
2 |
3 | namespace AleaTK.ML
4 | {
5 | public abstract class Differentiable : Symbol
6 | {
7 | private readonly List _inputs = new List();
8 | private readonly List _outputs = new List();
9 | private readonly List _auxvars = new List();
10 |
11 | public IEnumerable Inputs => _inputs;
12 |
13 | public IEnumerable Outputs => _outputs;
14 |
15 | public IEnumerable AuxVars => _auxvars;
16 |
17 | protected void AddInput(Variable input)
18 | {
19 | _inputs.Add(input);
20 | }
21 |
22 | protected void AddOutput(Variable output)
23 | {
24 | output.Owner = this;
25 | _outputs.Add(output);
26 | }
27 |
28 | protected void AddAuxVar(Variable auxvar)
29 | {
30 | _auxvars.Add(auxvar);
31 | }
32 |
33 | public abstract void Forward(Executor executor);
34 |
35 | public abstract void Backward(Executor executor);
36 |
37 | public virtual void Initialize(Executor executor)
38 | {
39 | }
40 | }
41 |
42 | public interface ILayer {
43 | Variable Output { get; }
44 | }
45 | }
--------------------------------------------------------------------------------
/src/AleaTK/ML/Library.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Collections.Generic;
3 | using System.Linq;
4 | using System.Text;
5 | using System.Threading.Tasks;
6 | using AleaTK.ML.Operator;
7 |
8 | namespace AleaTK.ML
9 | {
10 | public static class Library
11 | {
12 | public static Variable Variable()
13 | {
14 | return new Variable(VariableType.Common);
15 | }
16 |
17 | public static Variable Variable(PartialShape shape)
18 | {
19 | return new Variable(VariableType.Common, shape);
20 | }
21 |
22 | public static Variable Parameter(Expr initializer)
23 | {
24 | return new Variable(VariableType.Parameter, initializer);
25 | }
26 |
27 | public static Variable Parameter()
28 | {
29 | return new Variable(VariableType.Parameter);
30 | }
31 |
32 | public static Variable AuxVariable()
33 | {
34 | return new Variable(VariableType.Auxilliary);
35 | }
36 |
37 | public static Variable Dot(Variable a, Variable b)
38 | {
39 | var op = new Dot(a, b);
40 | return op.C;
41 | }
42 |
43 | public static Variable L2Loss(Variable pred, Variable label)
44 | {
45 | var op = new L2Loss(pred, label);
46 | return op.Loss;
47 | }
48 |
49 | public static Variable Reshape(this Variable input, params long[] shape)
50 | {
51 | return new Reshape(input, PartialShape.Create(shape)).Output;
52 | }
53 | }
54 | }
55 |
--------------------------------------------------------------------------------
/src/AleaTK/ML/Operator/Activation.cs:
--------------------------------------------------------------------------------
1 | using static AleaTK.Library;
2 | using static AleaTK.ML.Library;
3 |
4 | namespace AleaTK.ML.Operator
5 | {
6 | public abstract class Activation : Differentiable, ILayer {
7 | protected Activation(Variable input)
8 | {
9 | Input = input;
10 | Output = Variable(input.Shape);
11 | AddInput(Input);
12 | AddOutput(Output);
13 | }
14 |
15 | public Variable Input { get; }
16 |
17 | public Variable Output { get; }
18 |
19 | public override void Forward(Executor executor)
20 | {
21 | var input = executor.GetTensor(Input);
22 | executor.AssignTensor(Output, ForwardExpr(input));
23 | }
24 |
25 | protected abstract Expr ForwardExpr(Tensor input);
26 |
27 | public override void Backward(Executor executor)
28 | {
29 | var output = executor.GetTensor(Output);
30 | var dOutput = executor.GetGradient(Output);
31 | executor.AssignGradient(Input, BackwardExpr(output)*dOutput);
32 | }
33 |
34 | protected abstract Expr BackwardExpr(Tensor output);
35 | }
36 |
37 | public class ActivationReLU : Activation {
38 | public ActivationReLU(Variable input) : base(input) { }
39 |
40 | protected override Expr ForwardExpr(Tensor input) { return Max(input, 0.0.AsScalar()); }
41 |
42 | protected override Expr BackwardExpr(Tensor output) { return ReLUGrad(output); }
43 | }
44 |
45 | public class ActivationSigmoid : Activation
46 | {
47 | public ActivationSigmoid(Variable input) : base(input) { }
48 |
49 | protected override Expr ForwardExpr(Tensor input)
50 | {
51 | return 1.0.AsScalar()/(1.0.AsScalar() + Exp(-input));
52 | }
53 |
54 | protected override Expr BackwardExpr(Tensor output) { return output*(1.0.AsScalar() - output); }
55 | }
56 |
57 | public class ActivationTanh : Activation
58 | {
59 | public ActivationTanh(Variable input) : base(input) { }
60 |
61 | protected override Expr ForwardExpr(Tensor input) { return Tanh(input); }
62 |
63 | protected override Expr BackwardExpr(Tensor output) { return 1.0.AsScalar() - output*output; }
64 | }
65 | }
66 |
--------------------------------------------------------------------------------
/src/AleaTK/ML/Operator/Add.cs:
--------------------------------------------------------------------------------
1 | using System.Linq;
2 | using static AleaTK.Library;
3 | using static AleaTK.ML.Library;
4 |
5 | namespace AleaTK.ML.Operator
6 | {
7 | public class Add : Differentiable
8 | {
9 | public Add(Variable a, Variable b)
10 | {
11 | A = a;
12 | B = b;
13 | C = Variable();
14 | AddInput(a);
15 | AddInput(b);
16 | AddOutput(C);
17 | }
18 |
19 | public Variable A { get; }
20 |
21 | public Variable B { get; }
22 |
23 | public Variable C { get; }
24 |
25 | public override void Forward(Executor executor)
26 | {
27 | var a = executor.GetTensor(A);
28 | var b = executor.GetTensor(B);
29 | executor.AssignTensor(C, a + b);
30 | }
31 |
32 | public override void Backward(Executor executor)
33 | {
34 | var a = executor.GetTensor(A);
35 | var b = executor.GetTensor(B);
36 | var dC = executor.GetGradient(C);
37 |
38 | var dA = a.Shape.Rank < dC.Shape.Rank
39 | ? ReduceSum(dC, Enumerable.Range(0, dC.Shape.Rank - a.Shape.Rank).ToArray())
40 | : dC;
41 |
42 | var dB = b.Shape.Rank < dC.Shape.Rank
43 | ? ReduceSum(dC, Enumerable.Range(0, dC.Shape.Rank - b.Shape.Rank).ToArray())
44 | : dC;
45 |
46 | executor.AssignGradient(A, dA);
47 | executor.AssignGradient(B, dB);
48 | }
49 | }
50 | }
51 |
--------------------------------------------------------------------------------
/src/AleaTK/ML/Operator/Dot.cs:
--------------------------------------------------------------------------------
1 | using static AleaTK.Library;
2 | using static AleaTK.ML.Library;
3 |
4 | namespace AleaTK.ML.Operator
5 | {
6 | public class Dot : Differentiable
7 | {
8 | public Dot(Variable a, Variable b)
9 | {
10 | A = a;
11 | B = b;
12 | C = Variable();
13 | AddInput(a);
14 | AddInput(b);
15 | AddOutput(C);
16 | }
17 |
18 | public Variable A { get; }
19 |
20 | public Variable B { get; }
21 |
22 | public Variable C { get; }
23 |
24 | public override void Forward(Executor executor)
25 | {
26 | var a = executor.GetTensor(A);
27 | var b = executor.GetTensor(B);
28 | executor.AssignTensor(C, Dot(a, b));
29 | }
30 |
31 | public override void Backward(Executor executor)
32 | {
33 | var a = executor.GetTensor(A);
34 | var b = executor.GetTensor(B);
35 | var dC = executor.GetGradient(C);
36 | executor.AssignGradient(A, Dot(dC, b.T));
37 | executor.AssignGradient(B, Dot(a.T, dC));
38 | }
39 | }
40 | }
--------------------------------------------------------------------------------
/src/AleaTK/ML/Operator/Dropout.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Linq;
3 | using Alea;
4 | using static AleaTK.Library;
5 | using static AleaTK.ML.Library;
6 |
7 | namespace AleaTK.ML.Operator
8 | {
9 | ///
10 | /// With probability (1 - dropoutProb) outputs the input element scaled up by 1 / (1 - dropoutProb),
11 | /// otherwise outputs 0. The scaling is so that the expected sum is unchanged.
12 | ///
13 | public class Dropout : Differentiable
14 | {
15 | public Dropout(Variable input, double dropoutProb = 0.5)
16 | {
17 | Util.EnsureTrue(dropoutProb > 0.0);
18 | Util.EnsureTrue(dropoutProb < 1.0);
19 |
20 | Input = input;
21 | Output = Variable(input.Shape);
22 |
23 | Scale = 1.0 / (1.0 - dropoutProb);
24 |
25 | Threshold = (uint) ((double) UInt32.MaxValue*dropoutProb);
26 |
27 | Mask = AuxVariable();
28 |
29 | AddInput(input);
30 | AddAuxVar(Mask);
31 | AddOutput(Output);
32 | }
33 |
34 | public Variable Input { get; }
35 |
36 | public Variable Output { get; }
37 |
38 | public Variable Mask { get; }
39 |
40 | public uint Threshold { get; }
41 |
42 | public double Scale { get; }
43 |
44 | public override void Forward(Executor executor)
45 | {
46 | var ctx = executor.Context;
47 | var input = executor.GetTensor(Input);
48 | // TODO: make sure the offset is correct in one training.
49 | executor.AssignTensor(Mask, RandomUniform(input.Shape));
50 | var mask = executor.GetTensor(Mask);
51 | executor.AssignTensor(Output, Dropout(input, mask, Threshold, Scale));
52 | }
53 |
54 | public override void Backward(Executor executor)
55 | {
56 | var dOutput = executor.GetGradient(Output);
57 | var mask = executor.GetTensor(Mask);
58 | executor.AssignGradient(Input, Dropout(dOutput, mask, Threshold, Scale));
59 | }
60 | }
61 | }
--------------------------------------------------------------------------------
/src/AleaTK/ML/Operator/Embedding.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Linq;
3 | using Alea;
4 | using Alea.CSharp;
5 | using static AleaTK.Library;
6 |
7 | namespace AleaTK.ML.Operator
8 | {
9 | public class Embedding : Differentiable, ILayer {
10 | public Embedding(Variable indices, int embedSize, int embedDim, double initScale = 0.5)
11 | {
12 | Indices = indices;
13 | Weights = Library.Parameter((initScale * 2.0).AsScalar()*RandomUniform(Shape.Create(embedSize, embedDim)) - initScale.AsScalar());
14 | Output = Library.Variable(PartialShape.Create(Indices.Shape.Concat(new long[] { embedDim }).ToArray()));
15 | EmbedSize = embedSize;
16 | EmbedDim = embedDim;
17 |
18 | AddInput(Indices);
19 | AddInput(Weights);
20 | AddOutput(Output);
21 | }
22 |
23 | public Variable Indices { get; }
24 |
25 | public Variable Weights { get; }
26 |
27 | public Variable Output { get; }
28 |
29 | public int EmbedSize { get; }
30 |
31 | public int EmbedDim { get; }
32 |
33 | public override void Forward(Executor executor)
34 | {
35 | var indices = executor.GetTensor(Indices);
36 | var weights = executor.GetTensor(Weights);
37 | executor.AssignTensor(Output, Take(indices, weights));
38 | }
39 |
40 | public override void Backward(Executor executor)
41 | {
42 | var ctx = executor.Context;
43 | var indices = executor.GetTensor(Indices);
44 | var gradout = executor.GetGradient(Output);
45 |
46 | // for performance fix.
47 | if (ctx.Type == ContextType.Gpu && gradout.Layout.IsInnerChangeMostFullyPacked && indices.Layout.IsInnerChangeMostFullyPacked)
48 | {
49 | var embedDim = EmbedDim;
50 | var batchSize = (int)indices.Shape.Length;
51 | var threadSize = 256;
52 |
53 | // first set all to 0
54 | executor.AssignGradient(Weights, Fill(executor.GetTensor(Weights).Shape, ScalarOps.Conv(0.0)));
55 | var dW = executor.GetGradient(Weights);
56 |
57 | // then use a 1 block kernel to update it, cause usually the batch size is not huge, but the embedsize is huge!
58 | var stream = ctx.ToGpuContext().Stream;
59 | var iPtr = indices.Buffer.Ptr;
60 |
61 | // the following kernel is for 1 block, so there is no need for synchornization,
62 | // there could be further optimized.
63 |
64 | if (typeof(T) == typeof(float))
65 | {
66 | var dOPtr = gradout.Buffer.Ptr.Reinterpret();
67 | var dWPtr = dW.Buffer.Ptr.Reinterpret();
68 | var lp = new LaunchParam(1, threadSize);
69 | //Console.WriteLine($"{indices.Shape} {gradout.Shape} {dW.Shape}");
70 | stream.Launch(() =>
71 | {
72 | for (var i = 0; i < batchSize; ++i)
73 | {
74 | var row = iPtr[i];
75 |
76 | for (var k = threadIdx.x; k < embedDim; k += blockDim.x)
77 | {
78 | dWPtr[row * embedDim + k] += dOPtr[i * embedDim + k];
79 | }
80 | }
81 | }, lp);
82 |
83 | return;
84 | }
85 |
86 | throw new NotImplementedException();
87 | }
88 | else
89 | {
90 | executor.AssignGradient(Weights, TakeGrad(indices, gradout, EmbedSize));
91 | }
92 | }
93 | }
94 | }
--------------------------------------------------------------------------------
/src/AleaTK/ML/Operator/FullyConnected.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Linq;
3 | using static AleaTK.Library;
4 | using static AleaTK.ML.Library;
5 |
6 | namespace AleaTK.ML.Operator
7 | {
8 | public class FullyConnected : Differentiable, ILayer {
9 | public FullyConnected(Variable data, long numHidden)
10 | {
11 | Util.EnsureTrue(data.HasShape);
12 | Util.EnsureEqual(2, data.Shape.Rank, "Input must be matrix.");
13 | Util.EnsureTrue(data.Shape[1] > 0L);
14 |
15 | Data = data;
16 |
17 | var numInput = data.Shape[1];
18 | var scale = Sqrt(12.0.AsScalar() / ((double)(numInput + numHidden)).AsScalar());
19 | Weights = Parameter(scale * (RandomUniform(Shape.Create(numInput, numHidden), 0UL, 0UL) - 0.5.AsScalar()));
20 |
21 | Bias = Parameter(Fill(Shape.Create(numHidden), ScalarOps.Conv(0.0)));
22 | Output = Variable(PartialShape.Create(data.Shape[0], numHidden));
23 |
24 | AddInput(Data);
25 | AddInput(Weights);
26 | AddInput(Bias);
27 | AddOutput(Output);
28 | }
29 |
30 | public Variable Data { get; }
31 |
32 | public Variable Weights { get; }
33 |
34 | public Variable Bias { get; }
35 |
36 | public Variable Output { get; }
37 |
38 | public override void Forward(Executor executor)
39 | {
40 | var data = executor.GetTensor(Data);
41 | var weights = executor.GetTensor(Weights);
42 | var bias = executor.GetTensor(Bias);
43 | executor.AssignTensor(Output, Dot(data.Reshape(data.Shape[0], -1), weights) + bias);
44 | }
45 |
46 | public override void Backward(Executor executor)
47 | {
48 | var data = executor.GetTensor(Data);
49 | var weights = executor.GetTensor(Weights);
50 | var dOutput = executor.GetGradient(Output);
51 | executor.AssignGradient(Data, Dot(dOutput, weights.T).Reshape(data.Shape.AsArray));
52 | executor.AssignGradient(Weights, Dot(data.Reshape(data.Shape[0], -1).T, dOutput));
53 | executor.AssignGradient(Bias, ReduceSum(dOutput, 0));
54 | }
55 | }
56 | }
57 |
--------------------------------------------------------------------------------
/src/AleaTK/ML/Operator/L2Loss.cs:
--------------------------------------------------------------------------------
1 | using static AleaTK.Library;
2 | using static AleaTK.ML.Library;
3 |
4 | namespace AleaTK.ML.Operator
5 | {
6 | public class L2Loss : Differentiable
7 | {
8 | public L2Loss(Variable pred, Variable label)
9 | {
10 | Pred = pred;
11 | Label = label;
12 | Loss = Variable();
13 | AddInput(pred);
14 | AddInput(label);
15 | AddOutput(Loss);
16 | }
17 |
18 | public Variable Pred { get; }
19 |
20 | public Variable Label { get; }
21 |
22 | public Variable Loss { get; }
23 |
24 | public override void Forward(Executor executor)
25 | {
26 | var pred = executor.GetTensor(Pred);
27 | var label = executor.GetTensor(Label);
28 | executor.AssignTensor(Loss, ReduceSum((pred - label) * (pred - label)));
29 | }
30 |
31 | public override void Backward(Executor executor)
32 | {
33 | var pred = executor.GetTensor(Pred);
34 | var label = executor.GetTensor(Label);
35 | executor.AssignGradient(Pred, 2.0.AsScalar() * (pred - label));
36 | }
37 | }
38 | }
--------------------------------------------------------------------------------
/src/AleaTK/ML/Operator/Pooling.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using Alea.cuDNN;
3 | using static AleaTK.ML.Library;
4 |
5 | namespace AleaTK.ML.Operator
6 | {
7 | public class Pooling2D : Differentiable, ILayer {
8 | public Pooling2D(Variable data, PoolingMode mode, int kernelH, int kernelW, int strideH, int strideW)
9 | {
10 | Descriptor = new PoolingDescriptor();
11 | Descriptor.Set2D(mode, NanPropagation.NOT_PROPAGATE_NAN, kernelH, kernelW, 0, 0, strideH, strideW);
12 |
13 | var dataType = Dnn.DataTypeOf();
14 | var dataDesc = new TensorDescriptor();
15 | dataDesc.Set4D(dataType, TensorFormat.CUDNN_TENSOR_NCHW, 10, (int)data.Shape[1], (int)data.Shape[2], (int)data.Shape[3]);
16 |
17 | int n, c, h, w;
18 | Descriptor.Get2dForwardOutputDim(dataDesc, out n, out c, out h, out w);
19 |
20 | Data = data;
21 | Output = Variable(PartialShape.Create(-1, c, h, w));
22 |
23 | AddInput(Data);
24 | AddOutput(Output);
25 |
26 | dataDesc.Dispose();
27 | }
28 |
29 | public PoolingDescriptor Descriptor { get; }
30 |
31 | public Variable Data { get; }
32 |
33 | public Variable Output { get; }
34 |
35 | public override void Forward(Executor executor)
36 | {
37 | var data = executor.GetTensor(Data);
38 | var output = executor.GetTensor(Output, Shape.Create(data.Shape[0], Output.Shape[1], Output.Shape[2], Output.Shape[3]));
39 |
40 | if (executor.Context.Type == ContextType.Gpu)
41 | {
42 | var dnn = executor.Context.ToGpuContext().Dnn;
43 |
44 | using (var dataDescRcpt = executor.TensorDescRepo.Acquire())
45 | using (var outputDescRcpt = executor.TensorDescRepo.Acquire())
46 | {
47 | var dataDesc = dataDescRcpt.Value;
48 | var outputDesc = outputDescRcpt.Value;
49 | var dataType = Dnn.DataTypeOf();
50 |
51 | dataDesc.Set4D(dataType, TensorFormat.CUDNN_TENSOR_NCHW, (int)data.Shape[0], (int)data.Shape[1], (int)data.Shape[2], (int)data.Shape[3]);
52 | outputDesc.Set4D(dataType, TensorFormat.CUDNN_TENSOR_NCHW, (int)data.Shape[0], (int)Output.Shape[1], (int)Output.Shape[2], (int)Output.Shape[3]);
53 |
54 | dnn.PoolingForward(Descriptor, ScalarOps.Conv(1.0), dataDesc, data.Buffer.Ptr, ScalarOps.Conv(0.0), outputDesc, output.Buffer.Ptr);
55 |
56 | return;
57 | }
58 | }
59 |
60 | throw new NotImplementedException();
61 | }
62 |
63 | public override void Backward(Executor executor)
64 | {
65 | var data = executor.GetTensor(Data);
66 | var output = executor.GetTensor(Output);
67 | var dOutput = executor.GetGradient(Output);
68 | var dData = executor.GetGradient(Data, Shape.Create(data.Shape.AsArray));
69 |
70 | if (executor.Context.Type == ContextType.Gpu)
71 | {
72 | var dnn = executor.Context.ToGpuContext().Dnn;
73 |
74 | using (var dataDescRcpt = executor.TensorDescRepo.Acquire())
75 | using (var outputDescRcpt = executor.TensorDescRepo.Acquire())
76 | using (var dDataDescRcpt = executor.TensorDescRepo.Acquire())
77 | using (var dOutputDescRcpt = executor.TensorDescRepo.Acquire())
78 | {
79 | var dataDesc = dataDescRcpt.Value;
80 | var outputDesc = outputDescRcpt.Value;
81 | var dDataDesc = dDataDescRcpt.Value;
82 | var dOutputDesc = dOutputDescRcpt.Value;
83 | var dataType = Dnn.DataTypeOf();
84 |
85 | dataDesc.Set4D(dataType, TensorFormat.CUDNN_TENSOR_NCHW, (int)data.Shape[0], (int)data.Shape[1], (int)data.Shape[2], (int)data.Shape[3]);
86 | outputDesc.Set4D(dataType, TensorFormat.CUDNN_TENSOR_NCHW, (int)data.Shape[0], (int)Output.Shape[1], (int)Output.Shape[2], (int)Output.Shape[3]);
87 | dDataDesc.Set4D(dataType, TensorFormat.CUDNN_TENSOR_NCHW, (int)data.Shape[0], (int)data.Shape[1], (int)data.Shape[2], (int)data.Shape[3]);
88 | dOutputDesc.Set4D(dataType, TensorFormat.CUDNN_TENSOR_NCHW, (int)data.Shape[0], (int)Output.Shape[1], (int)Output.Shape[2], (int)Output.Shape[3]);
89 |
90 | dnn.PoolingBackward(Descriptor, ScalarOps.Conv(1.0), outputDesc, output.Buffer.Ptr, dOutputDesc,
91 | dOutput.Buffer.Ptr, dataDesc, data.Buffer.Ptr, ScalarOps.Conv(0.0), dDataDesc, dData.Buffer.Ptr);
92 |
93 | return;
94 | }
95 | }
96 |
97 | throw new NotImplementedException();
98 | }
99 | }
100 | }
--------------------------------------------------------------------------------
/src/AleaTK/ML/Operator/Reshape.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Collections.Generic;
3 | using System.Linq;
4 | using System.Text;
5 | using System.Threading.Tasks;
6 | using static AleaTK.Library;
7 | using static AleaTK.ML.Library;
8 |
9 | namespace AleaTK.ML.Operator
10 | {
11 | public class Reshape : Differentiable, ILayer {
12 | public Reshape(Variable input, PartialShape shape)
13 | {
14 | Util.EnsureTrue(input.Type != VariableType.Parameter);
15 | Shape = input.HasShape ? PartialShape.Reshape(input.Shape, shape) : shape;
16 | Input = input;
17 | Output = Variable(Shape);
18 | AddInput(Input);
19 | AddOutput(Output);
20 | }
21 |
22 | public Variable Input { get; }
23 |
24 | public Variable Output { get; }
25 |
26 | public PartialShape Shape { get; }
27 |
28 | public override void Forward(Executor executor)
29 | {
30 | if (executor.GetData(Input).Tensor.Layout.IsInnerChangeMostFullyPacked)
31 | {
32 | var tensor = executor.GetData(Input).Tensor;
33 | var shape = tensor.Layout.Shape.Reshape(Shape.AsArray);
34 | var layout = new Layout(shape);
35 | var newTensor = new Tensor(tensor.Device, tensor.Memory, layout, tensor.Ptr);
36 | executor.GetData(Output).SetTensor(newTensor);
37 | }
38 | else
39 | {
40 | throw new NotImplementedException("TODO: Assign a new tensor for different layout.");
41 | }
42 | }
43 |
44 | public override void Backward(Executor executor)
45 | {
46 | if (executor.GetData(Output).Gradient == null) return;
47 |
48 | if (executor.GetData(Output).Gradient.Layout.IsInnerChangeMostFullyPacked)
49 | {
50 | var inputData = executor.GetData(Input);
51 | var outputData = executor.GetData(Output);
52 | var inputCounter = inputData.GradientAggregationCounter;
53 | var outputCounter = outputData.GradientAggregationCounter;
54 | var outputGradient = outputData.Gradient;
55 | var inputShape = inputData.Tensor.Layout.Shape;
56 |
57 | if (inputCounter == 0)
58 | {
59 | var inputLayout = new Layout(inputShape);
60 | var inputGradient = new Tensor(outputGradient.Device, outputGradient.Memory, inputLayout, outputGradient.Ptr);
61 | inputData.SetGradient(inputGradient);
62 | inputData.GradientAggregationCounter = outputCounter;
63 | }
64 | else
65 | {
66 | var inputGradient = inputData.Gradient.Cast();
67 | var layout = new Layout(inputShape);
68 | var gradient = (new Tensor(outputGradient.Device, outputGradient.Memory, layout, outputGradient.Ptr)).Cast();
69 | var ctx = executor.Context;
70 | ctx.Assign(inputGradient, inputGradient + gradient);
71 | inputData.GradientAggregationCounter = inputCounter + outputCounter;
72 | }
73 | }
74 | else
75 | {
76 | throw new NotImplementedException("TODO: Assign a new tensor for different layout.");
77 | }
78 | }
79 | }
80 | }
81 |
--------------------------------------------------------------------------------
/src/AleaTK/ML/Symbol.cs:
--------------------------------------------------------------------------------
1 | using AleaTK.ML.Operator;
2 |
3 | namespace AleaTK.ML
4 | {
5 | public class Symbol
6 | {
7 | }
8 | }
9 |
--------------------------------------------------------------------------------
/src/AleaTK/ML/Variable.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Threading.Tasks;
3 | using AleaTK.ML.Operator;
4 |
5 | namespace AleaTK.ML
6 | {
7 | public enum VariableType
8 | {
9 | Common = 0,
10 | Parameter,
11 | Auxilliary
12 | }
13 |
14 | public abstract class Variable : Symbol
15 | {
16 | private readonly PartialShape _shape;
17 | private readonly Expr _initializer;
18 | private Differentiable _owner;
19 |
20 | protected Variable(Type dataType, VariableType type)
21 | {
22 | DataType = dataType;
23 | Type = type;
24 | _shape = null;
25 | _initializer = null;
26 | _owner = null;
27 | }
28 |
29 | protected Variable(Type dataType, VariableType type, PartialShape shape)
30 | {
31 | DataType = dataType;
32 | Type = type;
33 | _shape = shape;
34 | _initializer = null;
35 | _owner = null;
36 | }
37 |
38 | protected Variable(Type dataType, VariableType type, Expr initializer)
39 | {
40 | DataType = dataType;
41 | Type = type;
42 | _shape = initializer != null ? new PartialShape(initializer.Shape.AsArray) : null;
43 | _initializer = initializer;
44 | _owner = null;
45 | }
46 |
47 | public Type DataType { get; }
48 |
49 | public VariableType Type { get; }
50 |
51 | public PartialShape Shape
52 | {
53 | get
54 | {
55 | Util.EnsureTrue(_shape != null, "This variable doesn't have shape.");
56 | return _shape;
57 | }
58 | }
59 |
60 | public Expr UntypedInitializer
61 | {
62 | get
63 | {
64 | Util.EnsureTrue(_initializer != null, "This variable doesn't have initializer.");
65 | return _initializer;
66 | }
67 | }
68 |
69 | public Differentiable Owner
70 | {
71 | get { return _owner; }
72 | set
73 | {
74 | Util.EnsureTrue(_owner == null, "Owner is already set.");
75 | _owner = value;
76 | }
77 | }
78 |
79 | public bool HasShape => _shape != null;
80 |
81 | public bool HasInitializer => _initializer != null;
82 |
83 | public bool HasOwner => _owner != null;
84 |
85 | public abstract IValue TensorToValue(Tensor blob);
86 |
87 | public abstract Expr TensorToExpr(Tensor blob);
88 |
89 | public abstract void GetOrAllocate(Device device, Layout layout, long length, ref Tensor blob);
90 |
91 | public abstract Task Initialize(Context ctx, ref Tensor blob);
92 | }
93 |
94 | public class Variable : Variable
95 | {
96 | public Variable(VariableType type) : base(typeof (T), type)
97 | {
98 | }
99 |
100 | public Variable(VariableType type, PartialShape shape) : base(typeof (T), type, shape)
101 | {
102 | }
103 |
104 | public Variable(VariableType type, Expr initalizer) : base(typeof (T), type, initalizer)
105 | {
106 | }
107 |
108 | public Expr Initalizer => UntypedInitializer.CastExpr();
109 |
110 | public override IValue TensorToValue(Tensor blob)
111 | {
112 | return blob.Cast();
113 | }
114 |
115 | public override Expr TensorToExpr(Tensor blob)
116 | {
117 | return blob.Cast();
118 | }
119 |
120 | private void VerifyShape(Shape targetShape)
121 | {
122 | if (!HasShape) return;
123 |
124 | var myShape = Shape;
125 | Util.EnsureEqual(myShape.Rank, targetShape.Rank, "Tensor shape doesn't match variable shape.");
126 | for (var i = 0; i < myShape.Rank; ++i)
127 | {
128 | if (myShape[i] >= 0)
129 | {
130 | Util.EnsureEqual(myShape[i], targetShape[i], "Shape must match.");
131 | }
132 | }
133 | }
134 |
135 | public override void GetOrAllocate(Device device, Layout layout, long length, ref Tensor blob)
136 | {
137 | VerifyShape(layout.Shape);
138 |
139 | Tensor tensor;
140 |
141 | if (blob == null)
142 | {
143 | tensor = device.Allocate(layout, length);
144 | blob = tensor.ToTensor();
145 | return;
146 | }
147 |
148 | if (Layout.Match(blob.Layout, layout))
149 | {
150 | return;
151 | }
152 |
153 | if (blob.Memory.Memory.Length >= length)
154 | {
155 | var memory = new BufferMemory(blob.Memory.Memory, 0L, length);
156 | blob = new Tensor(device, memory, layout, memory.Memory.Handle);
157 | return;
158 | }
159 |
160 | tensor = device.Allocate(layout, length);
161 | blob = tensor.ToTensor();
162 | }
163 |
164 | public override Task Initialize(Context ctx, ref Tensor blob)
165 | {
166 | if (!HasInitializer) return Task.Run(() => { });
167 | var shape = Initalizer.Shape;
168 | var layout = new Layout(shape);
169 | var length = layout.Shape.Length;
170 | GetOrAllocate(ctx.Device, layout, length, ref blob);
171 | var tensor = blob.Cast();
172 | return ctx.Assign(tensor, Initalizer);
173 | }
174 |
175 | public static Variable operator +(Variable lhs, Variable rhs)
176 | {
177 | var op = new Add(lhs, rhs);
178 | return op.C;
179 | }
180 | }
181 | }
--------------------------------------------------------------------------------
/src/AleaTK/Properties/AssemblyInfo.cs:
--------------------------------------------------------------------------------
1 | using System.Reflection;
2 | using System.Runtime.CompilerServices;
3 | using System.Runtime.InteropServices;
4 |
5 | [assembly: AssemblyTitle("AleaTK")]
6 | [assembly: AssemblyDescription("")]
7 | [assembly: ComVisible(false)]
8 |
--------------------------------------------------------------------------------
/src/AleaTK/paket.references:
--------------------------------------------------------------------------------
1 | Alea
--------------------------------------------------------------------------------
/src/AleaTKUtil/AleaTKUtil.csproj:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Debug
6 | AnyCPU
7 | {19810D0A-BD63-4360-ACA3-1DB47C91E0A3}
8 | Library
9 | Properties
10 | AleaTKUtil
11 | AleaTKUtil
12 | v4.5.2
13 | 512
14 |
15 |
16 | true
17 | full
18 | false
19 | ..\..\debug\
20 | DEBUG;TRACE
21 | prompt
22 | 4
23 |
24 |
25 | pdbonly
26 | true
27 | ..\..\release\
28 | TRACE
29 | prompt
30 | 4
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
56 |
57 |
58 |
59 |