")
14 | os.Exit(1)
15 | }
16 |
17 | pageCount, err := strconv.Atoi(os.Args[2])
18 | if err != nil {
19 | fmt.Fprintln(os.Stderr, "Invalid page count:", os.Args[3])
20 | os.Exit(1)
21 | }
22 |
23 | res, err := IndexWikipedia(pageCount, os.Args[1])
24 | if err != nil {
25 | fmt.Fprintln(os.Stderr, err)
26 | os.Exit(1)
27 | }
28 |
29 | encoded, _ := json.Marshal(res)
30 | if err := ioutil.WriteFile(os.Args[3], encoded, 0755); err != nil {
31 | fmt.Fprintln(os.Stderr, err)
32 | os.Exit(1)
33 | }
34 | }
35 |
--------------------------------------------------------------------------------
/svm/problem.go:
--------------------------------------------------------------------------------
1 | // Package svm implements Support Vector Machines.
2 | package svm
3 |
4 | // A Sample represents an arbitrary piece of information.
5 | // All samples in a given sample space must have the same number of components.
6 | type Sample struct {
7 | V []float64
8 |
9 | // UserInfo can be used by a Kernel to uniquely identify a given Sample.
10 | // If a solver generates its own Samples, said Samples will have UserInfo set to 0.
11 | UserInfo int
12 | }
13 |
14 | // A Kernel takes two samples from a sample space and computes an inner product between them.
15 | type Kernel func(s1, s2 Sample) float64
16 |
17 | // A Problem defines everything needed to build a support vector machine that classifies samples in
18 | // a sample space.
19 | type Problem struct {
20 | Positives []Sample
21 | Negatives []Sample
22 | Kernel Kernel
23 | }
24 |
--------------------------------------------------------------------------------
/rnn/seqtoseq/sample.go:
--------------------------------------------------------------------------------
1 | // Package seqtoseq implements gradient-based training
2 | // for models which take an input sequence and produce
3 | // an output sequence of the same length.
4 | package seqtoseq
5 |
6 | import (
7 | "github.com/unixpickle/num-analysis/linalg"
8 | "github.com/unixpickle/sgd"
9 | )
10 |
11 | // Sample is a training sample containing an input
12 | // sequence and its corresponding output sequence.
13 | // All gradienters in this package take sample sets
14 | // with Sample elements.
15 | type Sample struct {
16 | Inputs []linalg.Vector
17 | Outputs []linalg.Vector
18 | }
19 |
20 | // Hash returns a randomly-distributed hash of the sample.
21 | func (s Sample) Hash() []byte {
22 | allVecs := make([]linalg.Vector, len(s.Inputs)+len(s.Outputs))
23 | copy(allVecs, s.Inputs)
24 | copy(allVecs[len(s.Inputs):], s.Outputs)
25 | return sgd.HashVectors(allVecs...)
26 | }
27 |
--------------------------------------------------------------------------------
/demos/minimax/images/player1_king.svg:
--------------------------------------------------------------------------------
1 |
2 |
22 |
--------------------------------------------------------------------------------
/demos/minimax/images/player2_king.svg:
--------------------------------------------------------------------------------
1 |
2 |
22 |
--------------------------------------------------------------------------------
/demos/objectrecog/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Object Recognition
6 |
7 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
22 |
23 |
26 |
27 |
28 |
--------------------------------------------------------------------------------
/demos/hopfield/scripts/main.js:
--------------------------------------------------------------------------------
1 | (function() {
2 |
3 | var DRAWING_COUNT = 3;
4 |
5 | function App() {
6 | this._drawingsElement = document.getElementById('drawings');
7 | this._drawings = [];
8 | for (var i = 0; i < DRAWING_COUNT; ++i) {
9 | var drawing = new window.app.Canvas();
10 | this._drawings[i] = drawing;
11 | this._drawingsElement.appendChild(drawing.element());
12 | }
13 |
14 | this._mainDrawing = new window.app.Canvas();
15 | this._mainDrawingElement = document.getElementById('main-drawing');
16 | this._mainDrawingElement.appendChild(this._mainDrawing.element());
17 |
18 | var vectorSize = this._mainDrawing.dimension() * this._mainDrawing.dimension();
19 | var network = new window.app.HopfieldNet(vectorSize);
20 | this._controls = new window.app.Controls(network, this._drawings, this._mainDrawing);
21 | }
22 |
23 | window.addEventListener('load', function() {
24 | new App();
25 | });
26 |
27 | })();
28 |
--------------------------------------------------------------------------------
/rnn/rnntest/state_out_block_test.go:
--------------------------------------------------------------------------------
1 | package rnntest
2 |
3 | import (
4 | "testing"
5 |
6 | "github.com/unixpickle/autofunc"
7 | "github.com/unixpickle/weakai/neuralnet"
8 | "github.com/unixpickle/weakai/rnn"
9 | )
10 |
11 | func TestStateOutBlock(t *testing.T) {
12 | net := neuralnet.Network{
13 | &neuralnet.DenseLayer{
14 | InputCount: 8,
15 | OutputCount: 4,
16 | },
17 | &neuralnet.HyperbolicTangent{},
18 | }
19 | net.Randomize()
20 | startVar := &autofunc.Variable{Vector: []float64{0.3, -0.3, 0.2, 0.5}}
21 | block := &rnn.StateOutBlock{
22 | Block: &rnn.BatcherBlock{
23 | B: net.BatchLearner(),
24 | StateSize: 4,
25 | Start: startVar,
26 | },
27 | }
28 | learner := append(stateOutBlockLearner{startVar}, net.Parameters()...)
29 | NewChecker4In(block, learner).FullCheck(t)
30 | }
31 |
32 | type stateOutBlockLearner []*autofunc.Variable
33 |
34 | func (s stateOutBlockLearner) Parameters() []*autofunc.Variable {
35 | return s
36 | }
37 |
--------------------------------------------------------------------------------
/demos/boosting_demo/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "math/rand"
6 | "time"
7 |
8 | "github.com/unixpickle/num-analysis/linalg"
9 | "github.com/unixpickle/weakai/boosting"
10 | )
11 |
12 | func main() {
13 | rand.Seed(time.Now().UnixNano())
14 |
15 | data := RandomSampleList(80)
16 | classes := RandomClassVector(data.Len())
17 |
18 | grad := boosting.Gradient{
19 | Loss: boosting.ExpLoss{},
20 | Desired: classes,
21 | List: data,
22 | Pool: StumpPool(data),
23 | }
24 | var i int
25 | for {
26 | loss := grad.Step()
27 | errs := errorCount(&grad.Sum, data, classes)
28 | fmt.Println("Epoch:", i, "loss:", loss, "errors:", errs)
29 | if errs == 0 {
30 | break
31 | }
32 | i++
33 | }
34 | }
35 |
36 | func errorCount(c boosting.Classifier, l SampleList, classes linalg.Vector) int {
37 | var count int
38 | for i, a := range c.Classify(l) {
39 | if (a < 0) != (classes[i] < 0) {
40 | count++
41 | }
42 | }
43 | return count
44 | }
45 |
--------------------------------------------------------------------------------
/evolution/entity.go:
--------------------------------------------------------------------------------
1 | package evolution
2 |
3 | // An Entity is a potential solution that evolution can manipulate and cause to reproduce.
4 | // Entities are immutable, so mutations and cross-over generate new Entities.
5 | type Entity interface {
6 | // Fitness an arbitrary measure of how good a given Entity is.
7 | // The higher this value, the "better" the Entity.
8 | Fitness() float64
9 |
10 | // Similarity returns a measurement of how similar this entity is to a bunch of other entities.
11 | // The higher the value, the more similar this entity is.
12 | Similarity(e []Entity) float64
13 |
14 | // Mutate returns an entity which is a mutated form of the current one.
15 | // The stepSize argument is a number indicating how drastic the mutation should be.
16 | // The higher the stepSize, the bigger the mutation.
17 | Mutate(stepSize float64) Entity
18 |
19 | // CrossOver returns an entity which is a random-ish combination of this one and another one.
20 | CrossOver(e Entity) Entity
21 | }
22 |
--------------------------------------------------------------------------------
/demos/nearestneighbors/README.md:
--------------------------------------------------------------------------------
1 | # Abstract
2 |
3 | This is probably the simplest, worst search engine ever. First, it indexes some Wikipedia pages. This process could be greatly improved, since it currently treats every word on a page equally when it should prioritize words near the beginnings of articles. After indexing, it lets you search a query against the index, using Nearest Neighbors to find the pages whose keyword vectors match your query the best.
4 |
5 | # Usage
6 |
7 | First, you must generate an index. Here is an example of how to do that:
8 |
9 | $ cd nearestneighbors
10 | $ go run indexer/*.go https://en.wikipedia.org/wiki/Monkey 1000 monkey1000.json
11 | $ go run prune_index/*.go monkey1000.json monkey1000_pruned.json 0.2
12 |
13 | This will generate a file called `monkey1000_pruned.json` with an index of 1000 pages which are somehow relevant to monkeys.
14 |
15 | After generating an index, you can search it like so:
16 |
17 | $ go run search/*.go monkey1000_pruned.json "your query here"
18 |
--------------------------------------------------------------------------------
/demos/svm/math_prototypes/nonmin_grad.m:
--------------------------------------------------------------------------------
1 | % Generate a KKT situation wherein the
2 | % active inequality constraints are
3 | % linearly dependent but still are not
4 | % at a minimum.
5 |
6 | function [gradient, basis] = nonmin_grad(dim)
7 | basis = constraint_vectors(dim);
8 | while true
9 | gradient = rand(dim, 1)*2 - repmat([1], dim, 1);
10 | gradient = basis * gradient;
11 | isMinimum = false;
12 | for i = 1:dim
13 | subBasis = [basis(:, 1:i-1) basis(:, i+1:dim)];
14 | solution = subBasis\gradient;
15 | allNeg = true;
16 | for j = 1:dim-1
17 | if solution(j) >= 0
18 | allNeg = false;
19 | end
20 | end
21 | if allNeg
22 | isMinimum = true;
23 | end
24 | end
25 | if !isMinimum
26 | break
27 | end
28 | end
29 | end
30 |
31 | function [vecs] = constraint_vectors(dim)
32 | equalityConstraint = repmat([1; -1], 1, dim)(1:dim);
33 | projSpace = null(equalityConstraint);
34 | vecs = projSpace*inv(projSpace'*projSpace)*projSpace';
35 | end
36 |
--------------------------------------------------------------------------------
/rnn/seqtoseq/cost_test.go:
--------------------------------------------------------------------------------
1 | package seqtoseq
2 |
3 | import (
4 | "math"
5 | "math/rand"
6 | "testing"
7 |
8 | "github.com/unixpickle/num-analysis/linalg"
9 | "github.com/unixpickle/sgd"
10 | "github.com/unixpickle/weakai/neuralnet"
11 | "github.com/unixpickle/weakai/rnn"
12 | )
13 |
14 | func TestTotalCostBlock(t *testing.T) {
15 | var samples sgd.SliceSampleSet
16 | for i := 0; i < 100; i++ {
17 | size := rand.Intn(10)
18 | inSeq := make([]linalg.Vector, size)
19 | outSeq := make([]linalg.Vector, size)
20 | for i := range inSeq {
21 | inSeq[i] = linalg.RandVector(3)
22 | outSeq[i] = linalg.RandVector(4)
23 | }
24 | samples = append(samples, Sample{Inputs: inSeq, Outputs: outSeq})
25 | }
26 | block := rnn.NewLSTM(3, 4)
27 | actual := TotalCostBlock(block, 7, samples, neuralnet.MeanSquaredCost{})
28 | expected := TotalCostSeqFunc(&rnn.BlockSeqFunc{B: block},
29 | 7, samples, neuralnet.MeanSquaredCost{})
30 | if math.Abs(actual-expected) > 1e-5 {
31 | t.Errorf("expected %v but got %v", expected, actual)
32 | }
33 | }
34 |
--------------------------------------------------------------------------------
/rbm/train_test.go:
--------------------------------------------------------------------------------
1 | package rbm
2 |
3 | import (
4 | "runtime"
5 | "testing"
6 | )
7 |
8 | const benchmarkLayerSize = 50
9 | const benchmarkSampleCount = 10
10 |
11 | func BenchmarkTrain(b *testing.B) {
12 | benchmarkTrain(b, 1)
13 | }
14 |
15 | func BenchmarkTrainConcurrent(b *testing.B) {
16 | n := runtime.GOMAXPROCS(0)
17 | runtime.GOMAXPROCS(10)
18 | benchmarkTrain(b, 10)
19 | runtime.GOMAXPROCS(n)
20 | }
21 |
22 | func benchmarkTrain(b *testing.B, batchSize int) {
23 | samples := make([][]bool, benchmarkSampleCount)
24 | for i := range samples {
25 | s := make([]bool, benchmarkLayerSize)
26 | for j := range s {
27 | // Some kind of psuedo-random nonsense,
28 | // just to enforce determinism.
29 | if (i*j*3+17)%19 < 7 {
30 | s[j] = true
31 | }
32 | }
33 | samples[i] = s
34 | }
35 | trainer := Trainer{
36 | GibbsSteps: 10,
37 | StepSize: 0.01,
38 | Epochs: 5,
39 | BatchSize: batchSize,
40 | }
41 | b.ResetTimer()
42 | for i := 0; i < b.N; i++ {
43 | r := NewRBM(50, 50)
44 | trainer.Train(r, samples)
45 | }
46 | }
47 |
--------------------------------------------------------------------------------
/demos/svm/math_prototypes/prove2.m:
--------------------------------------------------------------------------------
1 | % "Prove" that, if g is a linear combination
2 | % of independent vectors V1 through Vn, then
3 | % sign(dot(Xi, g))==sign(Ai) where Xi is Vi
4 | % with all the other V's projected out and Ai
5 | % is the coefficient of Vi in the linear
6 | % combination for g.
7 |
8 | function [rightCount, totalCount] = prove2(dim)
9 | rightCount = 0;
10 | totalCount = 0;
11 | basis = rand(dim, dim)*2 - repmat([1], dim, dim);
12 | for i = 1:1000
13 | g = rand(dim, 1);
14 | combination = basis\g;
15 | broken = false;
16 | for j = 1:dim
17 | num = combination(j);
18 | projOutBasis = [basis(:, 1:j-1) basis(:, j+1:dim)];
19 | projMat = projOutBasis*inv(projOutBasis'*projOutBasis)*projOutBasis';
20 | basisVec = basis(:, j);
21 | basisVec = basisVec - projMat*basisVec;
22 | d = dot(basisVec, g);
23 | if sign(d) != sign(num)
24 | broken = true;
25 | break
26 | end
27 | end
28 | if !broken
29 | rightCount += 1;
30 | end
31 | totalCount += 1;
32 | end
33 | end
34 |
--------------------------------------------------------------------------------
/neuralnet/network_test.go:
--------------------------------------------------------------------------------
1 | package neuralnet
2 |
3 | import (
4 | "testing"
5 |
6 | "github.com/unixpickle/serializer"
7 | )
8 |
9 | func TestNetworkSerialize(t *testing.T) {
10 | network := Network{
11 | &DenseLayer{InputCount: 3, OutputCount: 2},
12 | &Sigmoid{},
13 | }
14 | network.Randomize()
15 |
16 | encoded, err := network.Serialize()
17 | if err != nil {
18 | t.Fatal(err)
19 | }
20 | layerType := network.SerializerType()
21 |
22 | decoded, err := serializer.GetDeserializer(layerType)(encoded)
23 | if err != nil {
24 | t.Fatal(err)
25 | }
26 |
27 | decodedNet, ok := decoded.(Network)
28 | if !ok {
29 | t.Fatalf("expected *Network but got %T", decoded)
30 | }
31 |
32 | if len(network) != len(decodedNet) {
33 | t.Fatalf("expected %d layers but got %d", len(network), len(decodedNet))
34 | }
35 |
36 | _, ok = decodedNet[0].(*DenseLayer)
37 | if !ok {
38 | t.Fatalf("expected *DenseLayer but got %T", decodedNet[0])
39 | }
40 |
41 | _, ok = decodedNet[1].(*Sigmoid)
42 | if !ok {
43 | t.Fatalf("expected Sigmoid but got %T", decodedNet[1])
44 | }
45 | }
46 |
--------------------------------------------------------------------------------
/demos/idtrees/solve_csv/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "log"
6 | "os"
7 |
8 | "github.com/unixpickle/weakai/idtrees"
9 | )
10 |
11 | func main() {
12 | if len(os.Args) != 2 {
13 | fmt.Fprintln(os.Stderr, "Usage: idtrees ")
14 | fmt.Fprintln(os.Stderr, "")
15 | fmt.Fprintln(os.Stderr, " The first row of the input CSV file specifies field names.")
16 | fmt.Fprintln(os.Stderr, " Fields with names starting with _ are ignored.")
17 | fmt.Fprintln(os.Stderr, " The field whose name begins with * is identified by the tree.")
18 | fmt.Fprintln(os.Stderr, "")
19 | os.Exit(1)
20 | }
21 |
22 | log.Println("Reading CSV file...")
23 |
24 | f, err := os.Open(os.Args[1])
25 | if err != nil {
26 | fmt.Fprintln(os.Stderr, "Error opening file:", err)
27 | os.Exit(1)
28 | }
29 | defer f.Close()
30 |
31 | samples, keys, err := ReadCSV(f)
32 | if err != nil {
33 | fmt.Fprintln(os.Stderr, err)
34 | os.Exit(1)
35 | }
36 |
37 | log.Println("Generating tree...")
38 | tree := idtrees.ID3(samples, keys, 0)
39 | log.Println("Printing tree...")
40 |
41 | fmt.Println(tree)
42 | }
43 |
--------------------------------------------------------------------------------
/demos/svm/math_prototypes/disprove1.m:
--------------------------------------------------------------------------------
1 | % This disproves my first theory for the annoying
2 | % active set case described in nonmin_grad.m.
3 | % My theory was that, if v1, v2, v3, ..., vn are
4 | % dependent inequality gradients, and the point is
5 | % not a local constrained minimum, then excluding
6 | % vn and stepping in the direction of the first v
7 | % with a positive coefficient (with all the other
8 | % vs besides vn projected out) will result in a
9 | % feasible direction that increases the gradient
10 | % while also increasing the constraint corresponding
11 | % to vn.
12 |
13 | function [grad, basis] = disprove1(dim)
14 | while true
15 | [grad, basis] = nonmin_grad(dim);
16 | solution = basis(:, 1:dim-1)\grad;
17 | for i = 1:dim-1
18 | projOut = [basis(:, 1:i-1) basis(:, i+1:dim-1)];
19 | projMatrix = projOut*inv(projOut'*projOut)*projOut';
20 | if solution(i) >= 0
21 | projVec1 = projMatrix * basis(:, i);
22 | projVec2 = projMatrix * basis(:, dim);
23 | if dot(projVec1, projVec2) < 0
24 | return
25 | end
26 | end
27 | end
28 | end
29 | end
30 |
--------------------------------------------------------------------------------
/demos/mapcolor/types.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | type Value interface{}
4 |
5 | type Constraint interface {
6 | // Vairables returns an ordered pair of Variables.
7 | Variables() []Variable
8 |
9 | // RestrictedDomain computes the domain for one of the
10 | // two variables (specified by idx) given the domain of
11 | // the other variable.
12 | RestrictedDomain(idx int, d Domain) Domain
13 | }
14 |
15 | type Variable interface {
16 | Constraints() []Constraint
17 | InitialDomain() Domain
18 | }
19 |
20 | type Solution map[Variable]Value
21 |
22 | type Domain []Value
23 |
24 | func DomainIntersection(d1, d2 Domain) Domain {
25 | res := Domain{}
26 | for _, x := range d1 {
27 | if d2.Contains(x) {
28 | res = append(res, x)
29 | }
30 | }
31 | return res
32 | }
33 |
34 | func DomainUnion(d1, d2 Domain) Domain {
35 | res := make(Domain, len(d1))
36 | copy(res, d1)
37 | for _, x := range d2 {
38 | if !res.Contains(x) {
39 | res = append(res, x)
40 | }
41 | }
42 | return res
43 | }
44 |
45 | func (d Domain) Contains(v Value) bool {
46 | for _, x := range d {
47 | if x == v {
48 | return true
49 | }
50 | }
51 | return false
52 | }
53 |
--------------------------------------------------------------------------------
/neuralnet/sample_set.go:
--------------------------------------------------------------------------------
1 | package neuralnet
2 |
3 | import (
4 | "github.com/unixpickle/num-analysis/linalg"
5 | "github.com/unixpickle/sgd"
6 | )
7 |
8 | // VectorSample represents a supervised training sample
9 | // for a classifier which takes input vectors and makes
10 | // output vectors.
11 | type VectorSample struct {
12 | // Input is the input given to the classifier.
13 | Input linalg.Vector
14 |
15 | // Output is the desired output from the classifier.
16 | Output linalg.Vector
17 | }
18 |
19 | // Hash generates a randomly-distributed hash based on
20 | // the vector data.
21 | func (v VectorSample) Hash() []byte {
22 | return sgd.HashVectors(v.Input, v.Output)
23 | }
24 |
25 | // VectorSampleSet creates an sgd.SampleSet of
26 | // VectorSamples given a slice of inputs and a
27 | // slice of corresponding outputs.
28 | func VectorSampleSet(inputs []linalg.Vector, outputs []linalg.Vector) sgd.SampleSet {
29 | if len(inputs) != len(outputs) {
30 | panic("input and output counts do not match")
31 | }
32 | res := make(sgd.SliceSampleSet, len(inputs))
33 | for i, in := range inputs {
34 | res[i] = VectorSample{Input: in, Output: outputs[i]}
35 | }
36 | return res
37 | }
38 |
--------------------------------------------------------------------------------
/rbf/least_squares_test.go:
--------------------------------------------------------------------------------
1 | package rbf
2 |
3 | import (
4 | "testing"
5 |
6 | "github.com/unixpickle/autofunc"
7 | "github.com/unixpickle/num-analysis/linalg"
8 | "github.com/unixpickle/sgd"
9 | "github.com/unixpickle/weakai/neuralnet"
10 | )
11 |
12 | func TestLeastSquares(t *testing.T) {
13 | var samples sgd.SliceSampleSet
14 | for i := 0; i < 100; i++ {
15 | sample := neuralnet.VectorSample{
16 | Input: linalg.RandVector(5),
17 | }
18 | samples = append(samples, sample)
19 | }
20 | net := &Network{
21 | DistLayer: NewDistLayerSamples(5, 10, samples),
22 | ScaleLayer: NewScaleLayerShared(1),
23 | ExpLayer: &ExpLayer{Normalize: true},
24 | OutLayer: neuralnet.NewDenseLayer(10, 4),
25 | }
26 | net.OutLayer.Biases.Var.Vector.Scale(0)
27 | for i, x := range samples {
28 | sample := x.(neuralnet.VectorSample)
29 | sample.Output = net.Apply(&autofunc.Variable{Vector: sample.Input}).Output()
30 | samples[i] = sample
31 | }
32 | expected := net.OutLayer.Weights.Data.Vector
33 | net.OutLayer = nil
34 | actual := LeastSquares(net, samples, 3).Weights.Data.Vector
35 | if actual.Copy().Scale(-1).Add(expected).MaxAbs() > 1e-5 {
36 | t.Errorf("expected %v but got %v", expected, actual)
37 | }
38 | }
39 |
--------------------------------------------------------------------------------
/demos/neuralnet/image_demos/imgclass/classify.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "io/ioutil"
6 | "math"
7 | "os"
8 |
9 | "github.com/unixpickle/autofunc"
10 | "github.com/unixpickle/weakai/neuralnet"
11 | )
12 |
13 | func ClassifyCmd(netPath, imgPath string) {
14 | networkData, err := ioutil.ReadFile(netPath)
15 | if err != nil {
16 | fmt.Fprintln(os.Stderr, "Error reading network:", err)
17 | os.Exit(1)
18 | }
19 | network, err := neuralnet.DeserializeNetwork(networkData)
20 | if err != nil {
21 | fmt.Fprintln(os.Stderr, "Error deserializing network:", err)
22 | os.Exit(1)
23 | }
24 |
25 | img, width, height, err := ReadImageFile(imgPath)
26 | if err != nil {
27 | fmt.Fprintln(os.Stderr, "Error reading image:", err)
28 | os.Exit(1)
29 | }
30 |
31 | firstLayer := network[1].(*neuralnet.ConvLayer)
32 | if width != firstLayer.InputWidth || height != firstLayer.InputHeight {
33 | fmt.Fprintf(os.Stderr, "Expected dimensions %dx%d but got %dx%d\n",
34 | firstLayer.InputWidth, firstLayer.InputHeight, width, height)
35 | }
36 |
37 | output := network.Apply(&autofunc.Variable{Vector: img}).Output()
38 |
39 | for i, x := range output {
40 | fmt.Printf("Class %d: probability %f\n", i, math.Exp(x))
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/demos/idtrees/celebrity_lookup/list.txt:
--------------------------------------------------------------------------------
1 | https://en.wikipedia.org/wiki/Leonardo_DiCaprio
2 | https://en.wikipedia.org/wiki/Johnny_Depp
3 | https://en.wikipedia.org/wiki/Brad_Pitt
4 | https://en.wikipedia.org/wiki/Tom_Cruise
5 | https://en.wikipedia.org/wiki/Tom_Hanks
6 | https://en.wikipedia.org/wiki/Christian_Bale
7 | https://en.wikipedia.org/wiki/Will_Smith
8 | https://en.wikipedia.org/wiki/Denzel_Washington
9 | https://en.wikipedia.org/wiki/Robert_Downey_Jr.
10 | https://en.wikipedia.org/wiki/Justin_Bieber
11 | https://en.wikipedia.org/wiki/Kanye_West
12 | https://en.wikipedia.org/wiki/Eminem
13 | https://en.wikipedia.org/wiki/Chris_Martin
14 | https://en.wikipedia.org/wiki/Angelina_Jolie
15 | https://en.wikipedia.org/wiki/Jennifer_Lawrence
16 | https://en.wikipedia.org/wiki/Natalie_Portman
17 | https://en.wikipedia.org/wiki/Scarlett_Johansson
18 | https://en.wikipedia.org/wiki/Jennifer_Aniston
19 | https://en.wikipedia.org/wiki/Emma_Stone
20 | https://en.wikipedia.org/wiki/Mila_Kunis
21 | https://en.wikipedia.org/wiki/Sandra_Bullock
22 | https://en.wikipedia.org/wiki/Kristen_Stewart
23 | https://en.wikipedia.org/wiki/Kendall_Jenner
24 | https://en.wikipedia.org/wiki/Chloë_Grace_Moretz
25 | https://en.wikipedia.org/wiki/Snooki
26 | https://en.wikipedia.org/wiki/Selena_Gomez
27 |
--------------------------------------------------------------------------------
/demos/mapcolor/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "io/ioutil"
6 | "os"
7 | "strings"
8 | )
9 |
10 | func main() {
11 | if len(os.Args) != 3 {
12 | fmt.Fprintln(os.Stderr, "Usage: constraints USA.svg ")
13 | os.Exit(1)
14 | }
15 |
16 | variables := VariablesForStates()
17 | cancelChan := make(chan struct{})
18 | solutions := SolveConstraints(variables, cancelChan)
19 |
20 | solution := <-solutions
21 | close(cancelChan)
22 |
23 | cssCode := ""
24 | for state, color := range solution {
25 | stateAbbrev := state.(*StateVariable).Name
26 | cssCode += "#" + stateAbbrev + "{fill:" + color.(string) + ";}"
27 | }
28 |
29 | outputCode, err := setupOutputMap(cssCode)
30 | if err != nil {
31 | fmt.Fprintln(os.Stderr, "Could not use input file:", err)
32 | os.Exit(1)
33 | }
34 |
35 | if err := ioutil.WriteFile(os.Args[2], []byte(outputCode), 0755); err != nil {
36 | fmt.Fprintln(os.Stderr, err)
37 | os.Exit(1)
38 | }
39 | }
40 |
41 | func setupOutputMap(cssCode string) (string, error) {
42 | plainMap, err := ioutil.ReadFile(os.Args[1])
43 | if err != nil {
44 | return "", err
45 | }
46 | mapString := string(plainMap)
47 | return strings.Replace(mapString, "/* Insert CSS rules here */", cssCode, 1), nil
48 | }
49 |
--------------------------------------------------------------------------------
/demos/minimax/styles/board.css:
--------------------------------------------------------------------------------
1 | @media (max-height: 600px), (max-width: 600px) {
2 | #board {
3 | position: absolute;
4 | background-image: url('../images/board.svg');
5 | background-size: 64px 64px;
6 | width: 256px;
7 | height: 256px;
8 | left: calc(50% - 128px);
9 | top: calc(50% - 128px);
10 | }
11 | }
12 |
13 | @media (min-height: 600px) and (min-width: 600px) {
14 | #board {
15 | position: absolute;
16 | background-image: url('../images/board.svg');
17 | background-size: 140px 140px;
18 | width: 560px;
19 | height: 560px;
20 | left: calc(50% - 280px);
21 | top: calc(50% - 280px);
22 | }
23 | }
24 |
25 | .board-ai-turn {
26 | opacity: 0.5;
27 | pointer-events: none;
28 | }
29 |
30 | .board-piece {
31 | position: absolute;
32 | width: 12.5%;
33 | height: 12.5%;
34 | background-size: 100% 100%;
35 | }
36 |
37 | .board-piece-player1 {
38 | background-image: url('../images/player1_piece.svg');
39 | }
40 |
41 | .board-piece-player2 {
42 | background-image: url('../images/player2_piece.svg');
43 | }
44 |
45 | .board-piece-player1-king {
46 | background-image: url('../images/player1_king.svg');
47 | }
48 |
49 | .board-piece-player2-king {
50 | background-image: url('../images/player2_king.svg');
51 | }
52 |
--------------------------------------------------------------------------------
/rbf/exp_layer_test.go:
--------------------------------------------------------------------------------
1 | package rbf
2 |
3 | import (
4 | "testing"
5 |
6 | "github.com/unixpickle/autofunc"
7 | "github.com/unixpickle/autofunc/functest"
8 | "github.com/unixpickle/num-analysis/linalg"
9 | )
10 |
11 | func TestExpLayerOutput(t *testing.T) {
12 | l := &ExpLayer{}
13 | in := &autofunc.Variable{Vector: []float64{-2, -3, 1, -1}}
14 | actual := l.Apply(in).Output()
15 | expected := autofunc.Exp{}.Apply(in).Output()
16 | if actual.Copy().Scale(-1).Add(expected).MaxAbs() > 1e-5 {
17 | t.Errorf("expected %v but got %v", expected, actual)
18 | }
19 |
20 | l.Normalize = true
21 | actual = l.Apply(in).Output()
22 | expOut := autofunc.Exp{}.Apply(in)
23 | expSum := autofunc.SumAll(expOut)
24 | expected = autofunc.ScaleFirst(expOut, autofunc.Inverse(expSum)).Output()
25 | if actual.Copy().Scale(-1).Add(expected).MaxAbs() > 1e-5 {
26 | t.Errorf("expected %v but got %v", expected, actual)
27 | }
28 | }
29 |
30 | func TestExpLayerProp(t *testing.T) {
31 | l := &ExpLayer{Normalize: true}
32 | in := &autofunc.Variable{Vector: []float64{-2, -3, 1, -1}}
33 | rv := autofunc.RVector{in: linalg.RandVector(4)}
34 | ch := functest.RFuncChecker{
35 | F: l,
36 | Input: in,
37 | Vars: []*autofunc.Variable{in},
38 | RV: rv,
39 | }
40 | ch.FullCheck(t)
41 | }
42 |
--------------------------------------------------------------------------------
/demos/svm/basic_demo/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "math/rand"
6 | "time"
7 |
8 | "github.com/unixpickle/weakai/svm"
9 | )
10 |
11 | func main() {
12 | rand.Seed(time.Now().UnixNano())
13 |
14 | positives := []svm.Sample{
15 | svm.Sample{V: []float64{0, 0}},
16 | svm.Sample{V: []float64{0, 0.3}},
17 | svm.Sample{V: []float64{0.3, 0}},
18 | svm.Sample{V: []float64{0.1, 0.1}},
19 | }
20 |
21 | negatives := []svm.Sample{
22 | svm.Sample{V: []float64{1, 1}},
23 | svm.Sample{V: []float64{1, 0.7}},
24 | svm.Sample{V: []float64{0.7, 1}},
25 | svm.Sample{V: []float64{0.9, 0.9}},
26 | }
27 |
28 | problem := &svm.Problem{
29 | Positives: positives,
30 | Negatives: negatives,
31 | Kernel: svm.LinearKernel,
32 | }
33 |
34 | solution := svm.RandomlySolveLinear(problem, 100000, 5)
35 | fmt.Println("Solution from random solver:", solution)
36 |
37 | subgradientSolver := &svm.SubgradientSolver{
38 | Tradeoff: 0.001,
39 | Steps: 10000,
40 | StepSize: 0.001,
41 | }
42 | solution = subgradientSolver.Solve(problem)
43 |
44 | fmt.Println("Solution from subgradient solver:", solution)
45 |
46 | gradientSolver := &svm.GradientDescentSolver{
47 | Tradeoff: 0.0001,
48 | Timeout: time.Minute,
49 | }
50 | solution = gradientSolver.Solve(problem).Linearize()
51 |
52 | fmt.Println("Solution from gradient descent solver:", solution)
53 | }
54 |
--------------------------------------------------------------------------------
/demos/svm/math_prototypes/prove1.m:
--------------------------------------------------------------------------------
1 | % "Prove" that, if a set of dependent KKT vectors
2 | % do not represent a local minimum, then there is
3 | % a pair of vectors which, after projecting out the
4 | % other constraint gradients, have a positive dot
5 | % product with each other and with the overall gradient.
6 |
7 | function [rightCount, totalCount] = prove1(dim)
8 | rightCount = 0;
9 | totalCount = 0;
10 | for i = 1:1000
11 | totalCount += 1;
12 | [grad, basis] = nonmin_grad(dim);
13 | gotSolution = false;
14 | for j = 1:dim
15 | % Delete vector j; treat it as the dependent residue.
16 | delVec = basis(:, j);
17 | delBasis = [basis(:, 1:j-1) basis(:, j+1:dim)];
18 | for k = 1:dim-1
19 | % Step in the direction of k with the rest of the
20 | % vectors (besides vector j) projected out.
21 | others = [delBasis(:, 1:k-1) delBasis(:, k+1:dim-1)];
22 | othersProj = others * inv(others'*others) * others';
23 | directionVec = delBasis(:, k);
24 | directionVec = directionVec - othersProj*directionVec;
25 | if dot(directionVec,delVec) >= 0 && dot(directionVec,grad) >= 0
26 | gotSolution = true;
27 | break;
28 | end
29 | end
30 | if gotSolution
31 | break;
32 | end
33 | end
34 | if gotSolution
35 | rightCount += 1;
36 | end
37 | end
38 | end
39 |
--------------------------------------------------------------------------------
/demos/idtrees/sample_data/celebs.csv:
--------------------------------------------------------------------------------
1 | _Name,Age,Children,Sings,Acts,Active,*HasBeenMarried,Female
2 | Leonardo DiCaprio,41,0,false,false,true,false,false
3 | Johnny Depp,52,2,false,false,true,true,false
4 | Brad Pitt,52,6,false,false,true,true,false
5 | Tom Cruise,53,3,false,false,true,true,false
6 | Tom Hanks,59,4,false,false,true,true,false
7 | Christian Bale,41,2,false,false,true,true,false
8 | Will Smith,47,0,false,false,true,true,false
9 | Denzel Washington,61,4,false,false,true,true,false
10 | Robert Downey Jr.,50,3,false,false,true,true,false
11 | Justin Bieber,21,0,false,false,true,false,false
12 | Kanye West,38,2,false,false,true,true,false
13 | Eminem,43,3,false,false,true,true,false
14 | Chris Martin,38,0,false,false,false,true,false
15 | Angelina Jolie,40,6,false,false,true,true,true
16 | Jennifer Lawrence,25,0,false,false,true,false,true
17 | Natalie Portman,34,1,false,false,true,true,true
18 | Scarlett Johansson,31,1,false,false,true,true,true
19 | Jennifer Aniston,46,0,false,false,true,true,true
20 | Emma Stone,27,0,false,false,true,false,true
21 | Mila Kunis,32,1,false,false,true,true,true
22 | Sandra Bullock,51,2,false,false,true,true,true
23 | Kristen Stewart,25,0,false,false,true,false,true
24 | Kendall Jenner,20,0,false,false,true,false,true
25 | Chloë Grace Moretz,18,0,false,false,true,false,true
26 | Snooki,28,2,false,false,true,true,true
27 | Selena Gomez,23,0,false,false,true,false,true
28 |
--------------------------------------------------------------------------------
/neuralnet/interfaces.go:
--------------------------------------------------------------------------------
1 | package neuralnet
2 |
3 | import (
4 | "errors"
5 |
6 | "github.com/unixpickle/autofunc"
7 | "github.com/unixpickle/serializer"
8 | "github.com/unixpickle/sgd"
9 | )
10 |
11 | var uninitPanicMessage = errors.New("layer has uninitialized parameters")
12 |
13 | // A Layer represents any differentiable function
14 | // that can be used as a layer in a neural net.
15 | //
16 | // Layers must be serializable, evaluatable, and
17 | // differentiable.
18 | //
19 | // The autofunc.Results and autofunc.RResults from
20 | // a Layer are only valid so long as the Layer is
21 | // not modified (e.g. by SetCache, by changing
22 | // a struct field, etc.).
23 | //
24 | // Layers must support concurrent calls to their
25 | // autofunc.RFunc methods.
26 | // However, serialization methods needn't be safe
27 | // for concurrency.
28 | type Layer interface {
29 | serializer.Serializer
30 | autofunc.RFunc
31 | }
32 |
33 | // A Randomizer is anything which can be reset to
34 | // a random state.
35 | // For instance, some layers of a neural network
36 | // should be initially randomized to break symmetry.
37 | type Randomizer interface {
38 | Randomize()
39 | }
40 |
41 | // A LearnBatcher is a Learner that can be evaluated
42 | // in batch.
43 | type BatchLearner interface {
44 | sgd.Learner
45 | autofunc.RBatcher
46 | }
47 |
48 | // A SingleLearner is a Learner that can evaluate a
49 | // single input at once.
50 | type SingleLearner interface {
51 | sgd.Learner
52 | autofunc.RFunc
53 | }
54 |
--------------------------------------------------------------------------------
/idtrees/stringer.go:
--------------------------------------------------------------------------------
1 | package idtrees
2 |
3 | import (
4 | "bytes"
5 | "fmt"
6 | "strings"
7 | )
8 |
9 | // String returns a human-readable representation of
10 | // the tree, using indentation to signify depth.
11 | func (t *Tree) String() string {
12 | if t.Classification != nil {
13 | return classificationString(t.Classification)
14 | }
15 |
16 | var buf bytes.Buffer
17 |
18 | buf.WriteString(fmt.Sprintf("%v", t.Attr))
19 | buf.WriteRune('\n')
20 |
21 | split := t.ValSplit
22 |
23 | if t.NumSplit != nil {
24 | lessKey := fmt.Sprintf("<= %v", t.NumSplit.Threshold)
25 | greaterKey := fmt.Sprintf("> %v", t.NumSplit.Threshold)
26 | split = ValSplit{
27 | lessKey: t.NumSplit.LessEqual,
28 | greaterKey: t.NumSplit.Greater,
29 | }
30 | }
31 |
32 | isFirst := true
33 | for value, subtree := range split {
34 | if !isFirst {
35 | buf.WriteRune('\n')
36 | }
37 | isFirst = false
38 | var subBuf bytes.Buffer
39 | subBuf.WriteRune(' ')
40 | subBuf.WriteString(fmt.Sprintf("%v", value))
41 | subBuf.WriteString(" -> ")
42 | subBuf.WriteString(subtree.String())
43 | buf.WriteString(strings.Replace(subBuf.String(), "\n", "\n ", -1))
44 | }
45 |
46 | return buf.String()
47 | }
48 |
49 | func classificationString(m map[Class]float64) string {
50 | if len(m) == 0 {
51 | return "Unreachable"
52 | }
53 | var parts []string
54 | for key, frac := range m {
55 | parts = append(parts, fmt.Sprintf("%v=%.02f%%", key, frac*100))
56 | }
57 | return strings.Join(parts, " ")
58 | }
59 |
--------------------------------------------------------------------------------
/demos/neuralnet/image_demos/autoencoder/run.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "image"
6 | "image/png"
7 | "io/ioutil"
8 | "os"
9 |
10 | "github.com/unixpickle/autofunc"
11 | "github.com/unixpickle/tensor"
12 | "github.com/unixpickle/weakai/neuralnet"
13 | )
14 |
15 | func Run() {
16 | encoderPath := os.Args[2]
17 |
18 | encoderData, err := ioutil.ReadFile(encoderPath)
19 | if err != nil {
20 | fmt.Fprintln(os.Stderr, err)
21 | os.Exit(1)
22 | }
23 |
24 | network, err := neuralnet.DeserializeNetwork(encoderData)
25 | if err != nil {
26 | fmt.Fprintln(os.Stderr, err)
27 | os.Exit(1)
28 | }
29 |
30 | inputPath := os.Args[3]
31 | outputPath := os.Args[4]
32 |
33 | f, err := os.Open(inputPath)
34 | if err != nil {
35 | fmt.Fprintln(os.Stderr, err)
36 | os.Exit(1)
37 | }
38 | defer f.Close()
39 | inputImage, _, err := image.Decode(f)
40 | if err != nil {
41 | fmt.Fprintln(os.Stderr, err)
42 | os.Exit(1)
43 | }
44 |
45 | res := network.Apply(&autofunc.Variable{Vector: ImageTensor(inputImage).Data})
46 |
47 | tensor := &tensor.Float64{
48 | Width: inputImage.Bounds().Dx(),
49 | Height: inputImage.Bounds().Dy(),
50 | Depth: 3,
51 | Data: res.Output(),
52 | }
53 |
54 | image := ImageFromTensor(tensor)
55 | outFile, err := os.Create(outputPath)
56 | if err != nil {
57 | fmt.Fprintln(os.Stderr, err)
58 | os.Exit(1)
59 | }
60 | defer outFile.Close()
61 | if err := png.Encode(outFile, image); err != nil {
62 | fmt.Fprintln(os.Stderr, err)
63 | os.Exit(1)
64 | }
65 | }
66 |
--------------------------------------------------------------------------------
/rbf/doc.go:
--------------------------------------------------------------------------------
1 | // Package rbf implements Radial Basis Function networks.
2 | //
3 | // A network consists of four independent components:
4 | // distance computation, scaling, exponentiation, and
5 | // linear classification.
6 | //
7 | // A network can be pre-trained using random sampling and
8 | // least squares.
9 | // Doing so is extremely easy if you already have an
10 | // sgd.SampleSet that produces neuralnet.VectorSamples:
11 | //
12 | // import "github.com/unixpickle/sgd"
13 | //
14 | // ...
15 | //
16 | // var samples sgd.SampleSet
17 | // var inDims, centerCount int
18 | // // Set samples and dimensions here.
19 | // net := &rbf.Network{
20 | // DistLayer: rbf.NewDistLayerSamples(inDims, centerCount, samples),
21 | // ScaleLayer: rbf.NewScaleLayerShared(0.05),
22 | // ExpLayer: &rbf.ExpLayer{Normalize: true},
23 | // }
24 | // net.OutLayer = rbf.LeastSquares(n.Net, samples, 20)
25 | //
26 | // You can also use stochastic gradient descent to train a
27 | // network, or to fine-tune it after pre-training:
28 | //
29 | // import "github.com/unixpickle/sgd"
30 | //
31 | // ...
32 | //
33 | // gradienter := &neuralnet.BatchRGradienter{
34 | // Learner: net,
35 | // CostFunc: neuralnet.MeanSquaredCost{},
36 | // }
37 | // adam := &sgd.Adam{Gradienter: gradienter}
38 | // sgd.SGDInteractive(adam, samples, 0.001, 50, func() bool {
39 | // log.Println("Next training epoch...")
40 | // return true
41 | // })
42 | //
43 | package rbf
44 |
--------------------------------------------------------------------------------
/demos/boosting_demo/stumps.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "sort"
5 |
6 | "github.com/unixpickle/num-analysis/linalg"
7 | "github.com/unixpickle/weakai/boosting"
8 | )
9 |
10 | // A TreeStump makes classifications based on a vector
11 | // component being greater than a certain threshold.
12 | type TreeStump struct {
13 | FieldIndex int
14 | Threshold float64
15 | }
16 |
17 | func (t TreeStump) Classify(s boosting.SampleList) linalg.Vector {
18 | l := s.(SampleList)
19 | res := make(linalg.Vector, s.Len())
20 | for i, sample := range l {
21 | if sample[t.FieldIndex] >= t.Threshold {
22 | res[i] = 1
23 | } else {
24 | res[i] = -1
25 | }
26 | }
27 | return res
28 | }
29 |
30 | func StumpPool(samples SampleList) boosting.Pool {
31 | dims := len(samples[0])
32 | res := make([]boosting.Classifier, 0, len(samples)*dims)
33 | for d := 0; d < dims; d++ {
34 | values := make([]float64, 0, len(samples))
35 | seenValues := map[float64]bool{}
36 | for _, s := range samples {
37 | val := s[d]
38 | if !seenValues[val] {
39 | seenValues[val] = true
40 | values = append(values, val)
41 | }
42 | }
43 | sort.Float64s(values)
44 | for i, val := range values {
45 | var t TreeStump
46 | if i == 0 {
47 | t = TreeStump{FieldIndex: d, Threshold: val - 1}
48 | } else {
49 | lastVal := values[i-1]
50 | average := (lastVal + val) / 2
51 | t = TreeStump{FieldIndex: d, Threshold: average}
52 | }
53 | res = append(res, t)
54 | }
55 | }
56 | return boosting.NewStaticPool(res, samples)
57 | }
58 |
--------------------------------------------------------------------------------
/rnn/rnntest/parallel_block_test.go:
--------------------------------------------------------------------------------
1 | package rnntest
2 |
3 | import (
4 | "testing"
5 |
6 | "github.com/unixpickle/autofunc"
7 | "github.com/unixpickle/weakai/rnn"
8 | )
9 |
10 | func TestParallelBlockOutput(t *testing.T) {
11 | block1 := rnn.NewLSTM(3, 3)
12 | block2 := rnn.NewLSTM(3, 4)
13 | input := []autofunc.Result{
14 | &autofunc.Variable{Vector: []float64{1, -1, 0.5}},
15 | &autofunc.Variable{Vector: []float64{-0.5, 1, 0.5}},
16 | }
17 | o1 := block1.ApplyBlock([]rnn.State{block1.StartState(), block1.StartState()}, input)
18 | o2 := block2.ApplyBlock([]rnn.State{block2.StartState(), block2.StartState()}, input)
19 | expected1 := append(o1.Outputs()[0], o2.Outputs()[0]...)
20 | expected2 := append(o1.Outputs()[1], o2.Outputs()[1]...)
21 |
22 | pb := rnn.ParallelBlock{block1, block2}
23 | pbOut := pb.ApplyBlock([]rnn.State{pb.StartState(), pb.StartState()}, input)
24 | actual1 := pbOut.Outputs()[0]
25 | actual2 := pbOut.Outputs()[1]
26 |
27 | if actual1.Copy().Scale(-1).Add(expected1).MaxAbs() > 1e-5 {
28 | t.Errorf("expected output1 %v but got %v", expected1, actual1)
29 | }
30 | if actual2.Copy().Scale(-1).Add(expected2).MaxAbs() > 1e-5 {
31 | t.Errorf("expected output2 %v but got %v", expected2, actual2)
32 | }
33 | }
34 |
35 | func TestParallelBlock(t *testing.T) {
36 | if testing.Short() {
37 | t.SkipNow()
38 | }
39 | block1 := rnn.NewLSTM(4, 3)
40 | block2 := rnn.NewLSTM(4, 1)
41 | block3 := rnn.NewLSTM(4, 2)
42 | pb := rnn.ParallelBlock{block1, block2, block3}
43 | ch := NewChecker4In(pb, pb)
44 | ch.FullCheck(t)
45 | }
46 |
--------------------------------------------------------------------------------
/demos/hopfield/scripts/controls.js:
--------------------------------------------------------------------------------
1 | (function() {
2 |
3 | var INTERVAL = 100;
4 |
5 | function Controls(network, trainingSamples, drawing) {
6 | this._network = network;
7 | this._trainingSamples = trainingSamples;
8 | this._drawing = drawing;
9 |
10 | this._timer = null;
11 | this._toggleButton = document.getElementById('toggle-clock');
12 | this._trainButton = document.getElementById('train-network');
13 | this._registerButtons();
14 | }
15 |
16 | Controls.prototype._timerTick = function() {
17 | this._network.setVector(this._drawing.vector());
18 | this._network.convergeStep();
19 | this._drawing.setVector(this._network.vector());
20 | };
21 |
22 | Controls.prototype._train = function() {
23 | var samples = [];
24 | for (var i = 0, len = this._trainingSamples.length; i < len; ++i) {
25 | samples[i] = this._trainingSamples[i].vector();
26 | }
27 | this._network.train(samples);
28 | };
29 |
30 | Controls.prototype._registerButtons = function() {
31 | this._toggleButton.addEventListener('click', function() {
32 | if (this._timer === null) {
33 | this._toggleButton.innerText = 'Stop Running';
34 | this._timer = setInterval(this._timerTick.bind(this), INTERVAL);
35 | } else {
36 | this._toggleButton.innerText = 'Start Running';
37 | clearInterval(this._timer);
38 | this._timer = null;
39 | }
40 | }.bind(this));
41 | this._trainButton.addEventListener('click', this._train.bind(this));
42 | };
43 |
44 | window.app.Controls = Controls;
45 |
46 | })();
47 |
--------------------------------------------------------------------------------
/demos/objectrecog/scripts/recognition.js:
--------------------------------------------------------------------------------
1 | (function() {
2 |
3 | function findSubImage(largeImage, subImage) {
4 | var bigW = largeImage.width;
5 | var bigH = largeImage.height;
6 | var bigData = largeImage.getContext('2d').getImageData(0, 0, bigW, bigH).data;
7 |
8 | var subW = subImage.width;
9 | var subH = subImage.height;
10 | var subData = subImage.getContext('2d').getImageData(0, 0, subW, subH).data;
11 |
12 | var maxCorrelation = -Infinity;
13 | var matchX = 0;
14 | var matchY = 0;
15 |
16 | var maxX = bigW - subW;
17 | var maxY = bigH - subH;
18 | for (var y = 0; y < maxY; ++y) {
19 | for (var x = 0; x < maxX; ++x) {
20 | var correlation = 0;
21 | var magnitude1 = 0;
22 | var magnitude2 = 0;
23 | for (var subY = 0; subY < subH; ++subY) {
24 | for (var subX = 0; subX < subW; ++subX) {
25 | var subPixel = getImagePixel(subData, subX, subY, subW);
26 | var bigPixel = getImagePixel(bigData, x+subX, y+subY, bigW);
27 | correlation += subPixel * bigPixel;
28 | magnitude1 += subPixel * subPixel;
29 | magnitude2 += bigPixel * bigPixel;
30 | }
31 | }
32 | correlation /= Math.sqrt(magnitude1);
33 | correlation /= Math.sqrt(magnitude2);
34 | if (correlation > maxCorrelation) {
35 | maxCorrelation = correlation;
36 | matchX = x;
37 | matchY = y;
38 | }
39 | }
40 | }
41 |
42 | return {x: matchX, y: matchY};
43 | }
44 |
45 | function getImagePixel(data, x, y, w) {
46 | return data[4*(x+y*w)] + data[4*(x+y*w)+1] + data[4*(x+y*w)+2];
47 | }
48 |
49 | window.app.findSubImage = findSubImage;
50 |
51 | })();
52 |
--------------------------------------------------------------------------------
/demos/evolution_demo/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "math"
6 | "math/rand"
7 | "time"
8 |
9 | "github.com/unixpickle/weakai/evolution"
10 | )
11 |
12 | func main() {
13 | rand.Seed(time.Now().UnixNano())
14 | solver := evolution.Solver{
15 | StepCount: 1000,
16 | StepSizeInitial: 0.1,
17 | StepSizeFinal: 0,
18 | MaxPopulation: 30,
19 | MutateProbability: 0.3,
20 | CrossOverProbability: 0.3,
21 | SelectionProbability: 0.5,
22 | DFTradeoff: evolution.LinearDFTradeoff(0.1, 0.9),
23 | }
24 | solutions := solver.Solve([]evolution.Entity{Point{0, 0}})
25 | fmt.Println("Solution:", solutions[0])
26 | }
27 |
28 | func maximizeMe(x, y float64) float64 {
29 | return -(math.Pow(x-1, 2) + math.Pow(y, 2)) * math.Pow(math.Sin(x)*math.Cos(x)-1, 2)
30 | }
31 |
32 | type Point struct {
33 | X float64
34 | Y float64
35 | }
36 |
37 | func (p Point) Fitness() float64 {
38 | return maximizeMe(p.X, p.Y)
39 | }
40 |
41 | func (p Point) Similarity(e []evolution.Entity) float64 {
42 | var distanceSum float64
43 | for _, ent := range e {
44 | p1 := ent.(Point)
45 | distanceSum += math.Sqrt(math.Pow(p1.X-p.X, 2) + math.Pow(p1.Y-p.Y, 2))
46 | }
47 | if distanceSum == 0 {
48 | distanceSum = 0.00001
49 | }
50 | return 1 / distanceSum
51 | }
52 |
53 | func (p Point) Mutate(stepSize float64) evolution.Entity {
54 | diffX := (rand.Float64() * stepSize * 2) - stepSize
55 | diffY := (rand.Float64() * stepSize * 2) - stepSize
56 | return Point{p.X + diffX, p.Y + diffY}
57 | }
58 |
59 | func (p Point) CrossOver(e evolution.Entity) evolution.Entity {
60 | p1 := e.(Point)
61 | if rand.Intn(2) == 0 {
62 | return Point{X: p.X, Y: p1.Y}
63 | } else {
64 | return Point{X: p1.X, Y: p.Y}
65 | }
66 | }
67 |
--------------------------------------------------------------------------------
/svm/kernels.go:
--------------------------------------------------------------------------------
1 | package svm
2 |
3 | import "math"
4 |
5 | // LinearKernel is a Kernel that returns the straight dot product of the two input samples.
6 | func LinearKernel(s1, s2 Sample) float64 {
7 | if len(s1.V) != len(s2.V) {
8 | panic("samples must be of the sample dimension")
9 | }
10 | var sum float64
11 | for i, x := range s1.V {
12 | sum += x * s2.V[i]
13 | }
14 | return sum
15 | }
16 |
17 | // PolynomialKernel generates a Kernel that plugs vectors x and y into the formula (x*y + b)^n.
18 | func PolynomialKernel(b, n float64) Kernel {
19 | return func(x, y Sample) float64 {
20 | return math.Pow(LinearKernel(x, y)+b, n)
21 | }
22 | }
23 |
24 | // RadialBasisKernel generates a Kernel that plugs the vectors into exp(-c*||x-y||^2).
25 | func RadialBasisKernel(coeff float64) Kernel {
26 | return func(x, y Sample) float64 {
27 | var diffSquared float64
28 | for i, v := range x.V {
29 | diffSquared += math.Pow(v-y.V[i], 2)
30 | }
31 | return math.Exp(-coeff * diffSquared)
32 | }
33 | }
34 |
35 | // CachedKernel generates a Kernel which caches results from a different kernel.
36 | // This requires that each Sample has a unique UserInfo, excepting ones with UserInfo == 0.
37 | // The caching Kernel will not use the cache for any samples that have UserInfo values of 0.
38 | func CachedKernel(k Kernel) Kernel {
39 | cache := map[int]map[int]float64{}
40 | return func(s1, s2 Sample) float64 {
41 | if s1.UserInfo == 0 || s2.UserInfo == 0 {
42 | return k(s1, s2)
43 | }
44 | s1Cache := cache[s1.UserInfo]
45 | if s1Cache == nil {
46 | s1Cache = map[int]float64{}
47 | cache[s1.UserInfo] = s1Cache
48 | }
49 | if val, ok := s1Cache[s2.UserInfo]; ok {
50 | return val
51 | } else {
52 | res := k(s1, s2)
53 | s1Cache[s2.UserInfo] = res
54 | return res
55 | }
56 | }
57 | }
58 |
--------------------------------------------------------------------------------
/rbm/dbn.go:
--------------------------------------------------------------------------------
1 | package rbm
2 |
3 | import (
4 | "math/rand"
5 |
6 | "github.com/unixpickle/weakai/neuralnet"
7 | )
8 |
9 | // A DBN is a Deep Belief Network, which is
10 | // nothing more than stacked RBMs.
11 | type DBN []*RBM
12 |
13 | // Sample samples an output vector given an
14 | // input vector.
15 | // If r is nil, this uses the rand package's
16 | // default generator.
17 | func (d DBN) Sample(r *rand.Rand, input []bool) []bool {
18 | currentInput := input
19 | for _, layer := range d {
20 | output := make([]bool, len(layer.HiddenBiases))
21 | layer.SampleHidden(r, output, currentInput)
22 | currentInput = output
23 | }
24 | return currentInput
25 | }
26 |
27 | // SampleInput samples an input vector given
28 | // an output vector.
29 | // If r is nil, this uses the rand package's
30 | // default generator.
31 | func (d DBN) SampleInput(r *rand.Rand, output []bool) []bool {
32 | currentOutput := output
33 | for i := len(d) - 1; i >= 0; i-- {
34 | layer := d[i]
35 | input := make([]bool, len(layer.VisibleBiases))
36 | layer.SampleVisible(r, input, currentOutput)
37 | currentOutput = input
38 | }
39 | return currentOutput
40 | }
41 |
42 | // BuildANN builds a feed-forward neural network
43 | // that is based off of the weights and biases in
44 | // this DBN.
45 | func (d DBN) BuildANN() neuralnet.Network {
46 | network := neuralnet.Network{}
47 | for _, x := range d {
48 | inputSize := len(x.VisibleBiases)
49 | outputSize := len(x.HiddenBiases)
50 | layer := &neuralnet.DenseLayer{
51 | InputCount: inputSize,
52 | OutputCount: outputSize,
53 | }
54 | layer.Randomize()
55 | copy(layer.Weights.Data.Vector, x.Weights.Data)
56 | copy(layer.Biases.Var.Vector, x.HiddenBiases)
57 | network = append(network, layer)
58 | network = append(network, neuralnet.Sigmoid{})
59 | }
60 | return network
61 | }
62 |
--------------------------------------------------------------------------------
/idtrees/tree_test.go:
--------------------------------------------------------------------------------
1 | package idtrees
2 |
3 | import "testing"
4 |
5 | type treeTestSample map[Attr]Val
6 |
7 | func (t treeTestSample) Attr(n Attr) Val {
8 | return t[n]
9 | }
10 |
11 | func (t treeTestSample) Class() Class {
12 | return t["class"]
13 | }
14 |
15 | type treeTest struct {
16 | Samples []Sample
17 | Attrs []Attr
18 | Expected *Tree
19 | }
20 |
21 | func (t *treeTest) Run(test *testing.T, prefix string) {
22 | for maxGos := 0; maxGos < 4; maxGos++ {
23 | actual := t.actual(maxGos)
24 | if !treesEqual(t.Expected, actual) {
25 | test.Errorf(prefix+": bad tree with %d Gos:\n%s", maxGos, actual.String())
26 | break
27 | }
28 | }
29 | }
30 |
31 | func (t *treeTest) actual(maxGos int) *Tree {
32 | return ID3(t.Samples, t.Attrs, maxGos)
33 | }
34 |
35 | func treesEqual(t1, t2 *Tree) bool {
36 | if t1.Classification != nil {
37 | if t2.Classification == nil {
38 | return false
39 | }
40 | if len(t1.Classification) != len(t2.Classification) {
41 | return false
42 | }
43 | for k, v := range t1.Classification {
44 | if t2.Classification[k] != v {
45 | return false
46 | }
47 | }
48 | return true
49 | }
50 |
51 | if t1.Attr != t2.Attr {
52 | return false
53 | }
54 |
55 | if t1.NumSplit != nil {
56 | if t2.NumSplit == nil {
57 | return false
58 | }
59 | if t1.NumSplit.Threshold != t2.NumSplit.Threshold {
60 | return false
61 | }
62 | return treesEqual(t1.NumSplit.Greater, t2.NumSplit.Greater) &&
63 | treesEqual(t1.NumSplit.LessEqual, t2.NumSplit.LessEqual)
64 | }
65 |
66 | if len(t1.ValSplit) != len(t2.ValSplit) {
67 | return false
68 | }
69 | for val, tree := range t1.ValSplit {
70 | tree2 := t2.ValSplit[val]
71 | if tree2 == nil {
72 | return false
73 | }
74 | if !treesEqual(tree, tree2) {
75 | return false
76 | }
77 | }
78 |
79 | return true
80 | }
81 |
--------------------------------------------------------------------------------
/demos/neuralnet/image_demos/autoencoder/images.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "image"
5 | "image/color"
6 | "math"
7 | "os"
8 | "path/filepath"
9 |
10 | "github.com/unixpickle/tensor"
11 | )
12 |
13 | func ImageTensor(img image.Image) *tensor.Float64 {
14 | res := tensor.NewFloat64(img.Bounds().Dx(), img.Bounds().Dy(), 3)
15 | for y := 0; y < res.Height; y++ {
16 | for x := 0; x < res.Width; x++ {
17 | r, g, b, _ := img.At(x+img.Bounds().Min.X, y+img.Bounds().Min.Y).RGBA()
18 | res.Set(x, y, 0, float64(r)/65535.0)
19 | res.Set(x, y, 1, float64(g)/65535.0)
20 | res.Set(x, y, 2, float64(b)/65535.0)
21 | }
22 | }
23 | return res
24 | }
25 |
26 | func ImageFromTensor(t *tensor.Float64) image.Image {
27 | res := image.NewRGBA(image.Rect(0, 0, t.Width, t.Height))
28 |
29 | for y := 0; y < t.Height; y++ {
30 | for x := 0; x < t.Width; x++ {
31 | r := math.Min(math.Max(t.Get(x, y, 0), 0), 1)
32 | g := math.Min(math.Max(t.Get(x, y, 1), 0), 1)
33 | b := math.Min(math.Max(t.Get(x, y, 2), 0), 1)
34 | color := color.RGBA{
35 | A: 0xff,
36 | R: uint8(r * 0xff),
37 | G: uint8(g * 0xff),
38 | B: uint8(b * 0xff),
39 | }
40 | res.Set(x, y, color)
41 | }
42 | }
43 |
44 | return res
45 | }
46 |
47 | func ReadImages(path string) (<-chan image.Image, error) {
48 | dir, err := os.Open(path)
49 | if err != nil {
50 | return nil, err
51 | }
52 | defer dir.Close()
53 | names, err := dir.Readdirnames(0)
54 | if err != nil {
55 | return nil, err
56 | }
57 |
58 | resChan := make(chan image.Image)
59 | go func() {
60 | for _, name := range names {
61 | imagePath := filepath.Join(path, name)
62 | f, err := os.Open(imagePath)
63 | if err != nil {
64 | continue
65 | }
66 | image, _, _ := image.Decode(f)
67 | f.Close()
68 | if image != nil {
69 | resChan <- image
70 | }
71 | }
72 | close(resChan)
73 | }()
74 | return resChan, nil
75 | }
76 |
--------------------------------------------------------------------------------
/rnn/irnn.go:
--------------------------------------------------------------------------------
1 | package rnn
2 |
3 | import (
4 | "math"
5 | "math/rand"
6 |
7 | "github.com/unixpickle/autofunc"
8 | "github.com/unixpickle/num-analysis/linalg"
9 | "github.com/unixpickle/serializer"
10 | "github.com/unixpickle/sgd"
11 | "github.com/unixpickle/weakai/neuralnet"
12 | )
13 |
14 | // IRNN is an RNN that implements the identity RNN
15 | // described in https://arxiv.org/pdf/1504.00941v2.pdf.
16 | // An IRNN outputs its current state, so that Blocks
17 | // may be stacked on top of it to use this state.
18 | type IRNN interface {
19 | Block
20 | serializer.Serializer
21 | sgd.Learner
22 | }
23 |
24 | // NewIRNN creates a block for an identity RNN with
25 | // the given number of hidden units.
26 | // The initial state connections are set to an identity
27 | // matrix scaled by identityScale.
28 | func NewIRNN(inputSize, hiddenSize int, identityScale float64) IRNN {
29 | matrix := &neuralnet.DenseLayer{
30 | InputCount: inputSize + hiddenSize,
31 | OutputCount: hiddenSize,
32 | Biases: &autofunc.LinAdd{
33 | Var: &autofunc.Variable{
34 | Vector: make(linalg.Vector, hiddenSize),
35 | },
36 | },
37 | Weights: &autofunc.LinTran{
38 | Rows: hiddenSize,
39 | Cols: inputSize + hiddenSize,
40 | Data: &autofunc.Variable{
41 | Vector: make(linalg.Vector, hiddenSize*(inputSize+hiddenSize)),
42 | },
43 | },
44 | }
45 | weights := matrix.Weights.Data.Vector
46 | for i := 0; i < hiddenSize; i++ {
47 | weights[i+inputSize+i*(inputSize+hiddenSize)] = identityScale
48 | }
49 |
50 | weightScale := 1 / math.Sqrt(float64(inputSize))
51 | for j := 0; j < hiddenSize; j++ {
52 | for i := 0; i < inputSize; i++ {
53 | weights[i+j*(inputSize+hiddenSize)] = (rand.Float64()*2 - 1) * weightScale
54 | }
55 | }
56 |
57 | network := neuralnet.Network{
58 | matrix,
59 | &neuralnet.ReLU{},
60 | }
61 |
62 | return &StateOutBlock{
63 | Block: NewNetworkBlock(network, hiddenSize),
64 | }
65 | }
66 |
--------------------------------------------------------------------------------
/rnn/rnntest/stacked_block_test.go:
--------------------------------------------------------------------------------
1 | package rnntest
2 |
3 | import (
4 | "testing"
5 |
6 | "github.com/unixpickle/autofunc"
7 | "github.com/unixpickle/weakai/neuralnet"
8 | "github.com/unixpickle/weakai/rnn"
9 | )
10 |
11 | func TestStackedBlock(t *testing.T) {
12 | testVars := []*autofunc.Variable{
13 | {Vector: []float64{0.098591, -0.595453, -0.751214, 0.266051}},
14 | {Vector: []float64{0.988517, 0.107284, -0.331529, 0.028565}},
15 | {Vector: []float64{-0.150604, 0.889039, 0.120916, 0.240999}},
16 | {Vector: []float64{0.961058, 0.878608, 0.052284, -0.635746}},
17 | {Vector: []float64{0.31415, -0.2718}},
18 | {Vector: []float64{-0.6}},
19 | }
20 | testSeqs := [][]*autofunc.Variable{
21 | {testVars[0], testVars[2]},
22 | {testVars[1]},
23 | {testVars[2], testVars[1], testVars[3]},
24 | }
25 | testRV := autofunc.RVector{
26 | testVars[0]: []float64{0.62524, 0.52979, 0.33020, 0.54462},
27 | testVars[1]: []float64{0.13498, 0.12607, 0.35989, 0.23255},
28 | testVars[2]: []float64{0.85996, 0.68435, 0.68506, 0.96907},
29 | testVars[3]: []float64{0.79095, 0.33867, 0.86759, 0.16159},
30 | testVars[4]: []float64{-0.79095, 0.33867},
31 | testVars[5]: []float64{0.33867},
32 | }
33 | net1 := neuralnet.Network{
34 | &neuralnet.DenseLayer{
35 | InputCount: 6,
36 | OutputCount: 6,
37 | },
38 | &neuralnet.HyperbolicTangent{},
39 | }
40 | net1.Randomize()
41 | net2 := neuralnet.Network{
42 | &neuralnet.DenseLayer{
43 | InputCount: 5,
44 | OutputCount: 5,
45 | },
46 | &neuralnet.HyperbolicTangent{},
47 | }
48 | net2.Randomize()
49 | block := &rnn.StackedBlock{
50 | &rnn.BatcherBlock{B: net1.BatchLearner(), StateSize: 2, Start: testVars[4]},
51 | &rnn.BatcherBlock{B: net2.BatchLearner(), StateSize: 1, Start: testVars[5]},
52 | }
53 | checker := &BlockChecker{
54 | B: block,
55 | Input: testSeqs,
56 | Vars: testVars,
57 | RV: testRV,
58 | }
59 | checker.FullCheck(t)
60 | }
61 |
--------------------------------------------------------------------------------
/rbf/dist_layer_test.go:
--------------------------------------------------------------------------------
1 | package rbf
2 |
3 | import (
4 | "math/rand"
5 | "testing"
6 |
7 | "github.com/unixpickle/autofunc"
8 | "github.com/unixpickle/autofunc/functest"
9 | "github.com/unixpickle/num-analysis/linalg"
10 | )
11 |
12 | func TestDistLayerOutput(t *testing.T) {
13 | net := NewDistLayer(10, 3, 0)
14 | net.SetCenters([]linalg.Vector{
15 | {1, 2, 1, 2, 1, 2, 1, 2, 2, 1},
16 | {1, 0, 0, 0, 2, 1, 0, 0, 1, 0},
17 | {0, 0, 0, 0, 1, 0, 0, 0, 1, 1},
18 | })
19 | in := &autofunc.Variable{Vector: []float64{1, 1, 1, 1, 2, 1, 1, 0, 1, 2}}
20 | actual := net.Apply(in).Output()
21 | expected := []float64{10, 8, 8}
22 | if actual.Copy().Scale(-1).Add(expected).MaxAbs() > 1e-6 {
23 | t.Errorf("expected %v but got %v", expected, actual)
24 | }
25 | }
26 |
27 | func TestDistLayerProp(t *testing.T) {
28 | net := NewDistLayer(10, 3, 0)
29 | net.SetCenters([]linalg.Vector{
30 | {1, 2, 1, 2, 1, 2, 1, 2, 2, 1},
31 | {1, 0, 0, 0, 2, 1, 0, 0, 1, 0},
32 | {0, 0, 0, 0, 1, 0, 0, 0, 1, 1},
33 | })
34 | in := &autofunc.Variable{Vector: []float64{1, 1, 1, 1, 2, 1, 1, 0, 1, 2}}
35 | vars := append(net.Parameters(), in)
36 | rv := autofunc.RVector{}
37 | for _, v := range vars {
38 | rv[v] = make(linalg.Vector, len(v.Vector))
39 | for i := range rv[v] {
40 | rv[v][i] = rand.NormFloat64()
41 | }
42 | }
43 | checker := functest.RFuncChecker{
44 | F: net,
45 | Vars: vars,
46 | Input: in,
47 | RV: rv,
48 | }
49 | checker.FullCheck(t)
50 | }
51 |
52 | func BenchmarkDistLayer(b *testing.B) {
53 | f := NewDistLayer(300, 300, 1)
54 | in := &autofunc.Variable{Vector: make(linalg.Vector, 300)}
55 | for i := range in.Vector {
56 | in.Vector[i] = rand.NormFloat64()
57 | }
58 | g := autofunc.NewGradient(append(f.Parameters(), in))
59 | up := append(linalg.Vector{}, in.Vector...)
60 | b.ResetTimer()
61 | for i := 0; i < b.N; i++ {
62 | f.Apply(in).PropagateGradient(append(linalg.Vector{}, up...), g)
63 | }
64 | }
65 |
--------------------------------------------------------------------------------
/idtrees/forest.go:
--------------------------------------------------------------------------------
1 | package idtrees
2 |
3 | import (
4 | "math"
5 | "math/rand"
6 | )
7 |
8 | // A TreeGen generates decision trees which classify
9 | // a set of samples using a set of attributes.
10 | type TreeGen func(s []Sample, attrs []Attr) *Tree
11 |
12 | // A Forest is a list of bagged trees that are used
13 | // to classify samples.
14 | type Forest []*Tree
15 |
16 | // BuildForest creates a random forest with n trees,
17 | // where each tree was trained on nSamples samples
18 | // and nAttrs attributes.
19 | //
20 | // If nAttrs is 0, the rounded square root of the
21 | // number of attributes is used.
22 | func BuildForest(n int, samples []Sample, attrs []Attr,
23 | nSamples, nAttrs int, g TreeGen) Forest {
24 | if nAttrs == 0 {
25 | nAttrs = int(math.Sqrt(float64(len(attrs))) + 0.5)
26 | }
27 | sampleCopy := make([]Sample, len(samples))
28 | attrCopy := make([]Attr, len(attrs))
29 |
30 | copy(sampleCopy, samples)
31 | copy(attrCopy, attrs)
32 |
33 | res := make(Forest, n)
34 | for i := 0; i < n; i++ {
35 | randomizeSamples(sampleCopy, nSamples)
36 | randomizeAttrs(attrCopy, nAttrs)
37 | res[i] = g(sampleCopy[:nSamples], attrCopy[:nAttrs])
38 | }
39 | return res
40 | }
41 |
42 | // Classify uses f to compute the class probabilities
43 | // of the given sample.
44 | func (f Forest) Classify(s AttrMap) map[Class]float64 {
45 | res := map[Class]float64{}
46 | for _, t := range f {
47 | x := t.Classify(s)
48 | for k, v := range x {
49 | res[k] += v
50 | }
51 | }
52 | scaler := 1 / float64(len(f))
53 | for k, v := range res {
54 | res[k] = v * scaler
55 | }
56 | return res
57 | }
58 |
59 | func randomizeSamples(s []Sample, n int) {
60 | for i := 0; i < n; i++ {
61 | idx := rand.Intn(len(s)-i) + i
62 | s[i], s[idx] = s[idx], s[i]
63 | }
64 | }
65 |
66 | func randomizeAttrs(a []Attr, n int) {
67 | for i := 0; i < n; i++ {
68 | idx := rand.Intn(len(a)-i) + i
69 | a[i], a[idx] = a[idx], a[i]
70 | }
71 | }
72 |
--------------------------------------------------------------------------------
/rnn/network_seq_func.go:
--------------------------------------------------------------------------------
1 | package rnn
2 |
3 | import (
4 | "github.com/unixpickle/autofunc"
5 | "github.com/unixpickle/autofunc/seqfunc"
6 | "github.com/unixpickle/serializer"
7 | "github.com/unixpickle/weakai/neuralnet"
8 | )
9 |
10 | func init() {
11 | var n NetworkSeqFunc
12 | serializer.RegisterTypedDeserializer(n.SerializerType(), DeserializeNetworkSeqFunc)
13 | }
14 |
15 | // A NetworkSeqFunc is a seqfunc.RFunc which applies a
16 | // neuralnet.Network to each input to generate an output.
17 | type NetworkSeqFunc struct {
18 | Network neuralnet.Network
19 | }
20 |
21 | // DeserializeNetworkSeqFunc deserializes a NetworkSeqFunc
22 | // that was previously serialized.
23 | func DeserializeNetworkSeqFunc(d []byte) (*NetworkSeqFunc, error) {
24 | net, err := neuralnet.DeserializeNetwork(d)
25 | if err != nil {
26 | return nil, err
27 | }
28 | return &NetworkSeqFunc{Network: net}, nil
29 | }
30 |
31 | // ApplySeqs applies the network to the sequences.
32 | func (n *NetworkSeqFunc) ApplySeqs(in seqfunc.Result) seqfunc.Result {
33 | mb := &seqfunc.MapBatcher{B: n.Network.BatchLearner()}
34 | return mb.ApplySeqs(in)
35 | }
36 |
37 | // ApplySeqsR applies the network to the sequences.
38 | func (n *NetworkSeqFunc) ApplySeqsR(rv autofunc.RVector, in seqfunc.RResult) seqfunc.RResult {
39 | mb := &seqfunc.MapRBatcher{B: n.Network.BatchLearner()}
40 | return mb.ApplySeqsR(rv, in)
41 | }
42 |
43 | // Parameters returns the network's parameters.
44 | func (n *NetworkSeqFunc) Parameters() []*autofunc.Variable {
45 | return n.Network.Parameters()
46 | }
47 |
48 | // SerializerType returns the unique ID used to serialize
49 | // a NetworkSeqFunc with the serializer package.
50 | func (n *NetworkSeqFunc) SerializerType() string {
51 | return "github.com/unixpickle/weakai/rnn.NetworkSeqFunc"
52 | }
53 |
54 | // Serialize serializes the NetworkSeqFunc.
55 | func (n *NetworkSeqFunc) Serialize() ([]byte, error) {
56 | return n.Network.Serialize()
57 | }
58 |
--------------------------------------------------------------------------------
/boosting/gradient.go:
--------------------------------------------------------------------------------
1 | package boosting
2 |
3 | import (
4 | "github.com/unixpickle/autofunc"
5 | "github.com/unixpickle/num-analysis/linalg"
6 | )
7 |
8 | // Gradient performs gradient boosting.
9 | type Gradient struct {
10 | // Loss is the loss function to use for training.
11 | Loss LossFunc
12 |
13 | // Desired is the vector of desired classifications.
14 | Desired linalg.Vector
15 |
16 | // List is the list of samples to boost on.
17 | List SampleList
18 |
19 | // Pool is used to create weak learners for training.
20 | Pool Pool
21 |
22 | // Sum is the current ensemble classifier, which
23 | // is added to during each step of boosting.
24 | Sum SumClassifier
25 |
26 | // OutCache is the current output of the ensemble
27 | // classifier.
28 | // This is used to avoid recomputing the outputs
29 | // of all the classifiers at each iteration.
30 | //
31 | // If you modify Pool, List, Desired, or Sum during
32 | // training, you should nil out or recompute OutCache
33 | // to reflect the new situation.
34 | OutCache linalg.Vector
35 | }
36 |
37 | // Step performs a step of gradient boosting and
38 | // returns the loss before the step was performed.
39 | func (g *Gradient) Step() float64 {
40 | if g.OutCache == nil {
41 | g.OutCache = g.Sum.Classify(g.List)
42 | }
43 | curOutput := &autofunc.Variable{
44 | Vector: g.OutCache,
45 | }
46 | curLoss := g.Loss.Loss(curOutput, g.Desired)
47 |
48 | grad := autofunc.NewGradient([]*autofunc.Variable{curOutput})
49 | curLoss.PropagateGradient([]float64{1}, grad)
50 |
51 | classifier := g.Pool.BestClassifier(g.List, grad[curOutput])
52 | classOutput := classifier.Classify(g.List)
53 | stepAmount := g.Loss.OptimalStep(curOutput.Vector, classOutput, g.Desired)
54 |
55 | g.Sum.Weights = append(g.Sum.Weights, stepAmount)
56 | g.Sum.Classifiers = append(g.Sum.Classifiers, classifier)
57 |
58 | g.OutCache.Add(classOutput.Scale(stepAmount))
59 |
60 | return curLoss.Output()[0]
61 | }
62 |
--------------------------------------------------------------------------------
/demos/idtrees/mnist/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "log"
5 |
6 | "github.com/unixpickle/mnist"
7 | "github.com/unixpickle/weakai/idtrees"
8 | )
9 |
10 | const (
11 | ForestSize = 100
12 | TrainingSize = 4000
13 | )
14 |
15 | func main() {
16 | log.Println("Creating training samples...")
17 | samples := trainingSamples()
18 | attrs := trainingAttrs()
19 | log.Println("Training forest...")
20 | forest := idtrees.BuildForest(ForestSize, samples, attrs, TrainingSize, 75,
21 | func(s []idtrees.Sample, a []idtrees.Attr) *idtrees.Tree {
22 | return idtrees.ID3(s, a, 0)
23 | })
24 | log.Println("Running classifications...")
25 | hist := mnist.LoadTestingDataSet().CorrectnessHistogram(func(data []float64) int {
26 | sample := newImageSample(mnist.Sample{Intensities: data})
27 | res := forest.Classify(sample)
28 | var maxVal float64
29 | var maxClass int
30 | for class, x := range res {
31 | if x > maxVal {
32 | maxVal = x
33 | maxClass = class.(int)
34 | }
35 | }
36 | return maxClass
37 | })
38 | log.Println("Results:", hist)
39 | }
40 |
41 | func trainingSamples() []idtrees.Sample {
42 | set := mnist.LoadTrainingDataSet()
43 | res := make([]idtrees.Sample, len(set.Samples))
44 | for i, x := range set.Samples {
45 | res[i] = newImageSample(x)
46 | }
47 | return res
48 | }
49 |
50 | func trainingAttrs() []idtrees.Attr {
51 | attrs := make([]idtrees.Attr, 28*28)
52 | for i := 0; i < 28*28; i++ {
53 | attrs[i] = i
54 | }
55 | return attrs
56 | }
57 |
58 | type imageSample struct {
59 | Intensities []float64
60 | Label int
61 | }
62 |
63 | func newImageSample(s mnist.Sample) *imageSample {
64 | return &imageSample{
65 | Intensities: s.Intensities,
66 | Label: s.Label,
67 | }
68 | }
69 |
70 | func (i *imageSample) Attr(idx idtrees.Attr) idtrees.Val {
71 | return i.Intensities[idx.(int)]
72 | }
73 |
74 | func (i *imageSample) Class() idtrees.Class {
75 | return i.Label
76 | }
77 |
--------------------------------------------------------------------------------
/rbf/exp_layer.go:
--------------------------------------------------------------------------------
1 | package rbf
2 |
3 | import (
4 | "errors"
5 |
6 | "github.com/unixpickle/autofunc"
7 | "github.com/unixpickle/serializer"
8 | )
9 |
10 | func init() {
11 | var e ExpLayer
12 | serializer.RegisterTypedDeserializer(e.SerializerType(), DeserializeExpLayer)
13 | }
14 |
15 | // ExpLayer exponentiates its inputs, with the option to
16 | // normalize the results by dividing by the sum.
17 | type ExpLayer struct {
18 | Normalize bool
19 | }
20 |
21 | // DeserializeExpLayer deserializes an ExpLayer.
22 | func DeserializeExpLayer(d []byte) (*ExpLayer, error) {
23 | if len(d) != 1 {
24 | return nil, errors.New("expected 1 byte for ExpLayer")
25 | }
26 | return &ExpLayer{Normalize: d[0] != 0}, nil
27 | }
28 |
29 | // Apply applies the layer to an input.
30 | func (e *ExpLayer) Apply(in autofunc.Result) autofunc.Result {
31 | if !e.Normalize {
32 | return autofunc.Exp{}.Apply(in)
33 | }
34 | return autofunc.Pool(in, func(in autofunc.Result) autofunc.Result {
35 | scale := autofunc.Scale(autofunc.SumAllLogDomain(in), -1)
36 | return autofunc.Exp{}.Apply(autofunc.AddFirst(in, scale))
37 | })
38 | }
39 |
40 | // ApplyR applies the layer to an input.
41 | func (e *ExpLayer) ApplyR(rv autofunc.RVector, in autofunc.RResult) autofunc.RResult {
42 | if !e.Normalize {
43 | return autofunc.Exp{}.ApplyR(rv, in)
44 | }
45 | return autofunc.PoolR(in, func(in autofunc.RResult) autofunc.RResult {
46 | scale := autofunc.ScaleR(autofunc.SumAllLogDomainR(in), -1)
47 | return autofunc.Exp{}.ApplyR(rv, autofunc.AddFirstR(in, scale))
48 | })
49 | }
50 |
51 | // SerializerType returns the unique ID used to serialize
52 | // an ExpLayer with the serializer package.
53 | func (e *ExpLayer) SerializerType() string {
54 | return "github.com/unixpickle/weakai/rbf.ExpLayer"
55 | }
56 |
57 | // Serialize serializes the layer.
58 | func (e *ExpLayer) Serialize() ([]byte, error) {
59 | if e.Normalize {
60 | return []byte{1}, nil
61 | }
62 | return []byte{0}, nil
63 | }
64 |
--------------------------------------------------------------------------------
/boosting/static_pool.go:
--------------------------------------------------------------------------------
1 | package boosting
2 |
3 | import (
4 | "github.com/gonum/blas"
5 | "github.com/gonum/blas/blas64"
6 | "github.com/unixpickle/num-analysis/linalg"
7 | )
8 |
9 | // A StaticPool stores an unchanging list of classifiers
10 | // and caches their outputs on an unchanging sample set.
11 | type StaticPool struct {
12 | classifiers []Classifier
13 | outputMatrix blas64.General
14 | }
15 |
16 | // NewStaticPool creates a static pool for the given
17 | // classifiers and samples.
18 | // In the process of creating the pool, every classifier
19 | // is run on every sample.
20 | // As a result, NewStaticPool may take some time to run.
21 | func NewStaticPool(c []Classifier, s SampleList) *StaticPool {
22 | res := &StaticPool{
23 | classifiers: make([]Classifier, len(c)),
24 | outputMatrix: blas64.General{
25 | Rows: len(c),
26 | Cols: s.Len(),
27 | Stride: s.Len(),
28 | Data: make([]float64, len(c)*s.Len()),
29 | },
30 | }
31 | copy(res.classifiers, c)
32 |
33 | var rowIdx int
34 | for _, classifier := range c {
35 | output := classifier.Classify(s)
36 | copy(res.outputMatrix.Data[rowIdx:], output)
37 | rowIdx += s.Len()
38 | }
39 |
40 | return res
41 | }
42 |
43 | // BestClassifier returns the classifier in the static
44 | // pool whose output correlates the most highly with
45 | // the given weight vector, as measured by absolute
46 | // cosine distance.
47 | //
48 | // The list argument is ignored, since a StaticPool
49 | // always uses the set of samples it was given when
50 | // it was initialized.
51 | func (s *StaticPool) BestClassifier(list SampleList, weights linalg.Vector) Classifier {
52 | vec := blas64.Vector{
53 | Inc: 1,
54 | Data: weights,
55 | }
56 | output := blas64.Vector{
57 | Inc: 1,
58 | Data: make([]float64, len(s.classifiers)),
59 | }
60 | blas64.Gemv(blas.NoTrans, 1, s.outputMatrix, vec, 0, output)
61 | largest := blas64.Iamax(len(s.classifiers), output)
62 | return s.classifiers[largest]
63 | }
64 |
--------------------------------------------------------------------------------
/demos/minimax/scripts/main.js:
--------------------------------------------------------------------------------
1 | (function() {
2 |
3 | var gameState = new window.app.GameState();
4 | var boardView = null;
5 |
6 | function initialize() {
7 | boardView = new window.app.BoardView();
8 | boardView.on('move', function(id, position) {
9 | var available = gameState.availableMoves();
10 | for (var i = 0, len = available.length; i < len; ++i) {
11 | var move = available[i];
12 | if (move.getPiece().getId() === id) {
13 | var dest = move.getDestination();
14 | if (dest.x === position.x && dest.y === position.y) {
15 | gameState = gameState.stateAfterMove(move);
16 | break;
17 | }
18 | }
19 | }
20 | boardView.updateWithState(gameState);
21 | if (gameState.playerTurn() === 2) {
22 | playAITurn();
23 | }
24 | });
25 | }
26 |
27 | function playAITurn() {
28 | boardView.element().className = 'board-ai-turn';
29 | window.app.optimalTurn(gameState, function(moves) {
30 | if (moves.length === 0) {
31 | handleAILoss();
32 | return;
33 | }
34 | var idx = 0;
35 | var triggerNext;
36 | triggerNext = function() {
37 | gameState = gameState.stateAfterMove(moves[idx++]);
38 | boardView.updateWithState(gameState);
39 | if (idx === moves.length) {
40 | boardView.element().className = '';
41 | if (gameState.availableMoves().length === 0) {
42 | handleHumanLoss();
43 | }
44 | } else {
45 | setTimeout(triggerNext, 500);
46 | }
47 | };
48 | triggerNext();
49 | });
50 | }
51 |
52 | function handleAILoss() {
53 | setTimeout(function() {
54 | alert('The AI has lost!');
55 | }, 300);
56 | }
57 |
58 | function handleHumanLoss() {
59 | setTimeout(function() {
60 | alert('You have lost!');
61 | }, 300);
62 | }
63 |
64 | window.addEventListener('load', initialize);
65 |
66 | })();
67 |
--------------------------------------------------------------------------------
/demos/minimax/scripts/eventemitter.js:
--------------------------------------------------------------------------------
1 | // eventemitter.js version 0.1.1.
2 | (function() {
3 | // An EventEmitter makes it easy to emit and subscribe to events.
4 | function EventEmitter() {
5 | this._listeners = {};
6 | }
7 |
8 | EventEmitter.makeSubclass = function(p) {
9 | var keys = Object.keys(EventEmitter.prototype);
10 | for (var i = 0, len = keys.length; i < len; ++i) {
11 | var key = keys[i];
12 | p[key] = EventEmitter.prototype[key];
13 | }
14 | };
15 |
16 | EventEmitter.prototype.listeners = function(name) {
17 | return (this._listeners[name] || []).slice();
18 | };
19 |
20 | EventEmitter.prototype.addListener = function(name, listener) {
21 | var listeners = this._listeners[name];
22 | if (!listeners) {
23 | this._listeners[name] = [listener];
24 | } else {
25 | listeners.push(listener);
26 | }
27 | };
28 |
29 | EventEmitter.prototype.removeListener = function(name, listener) {
30 | var listeners = this._listeners[name];
31 | if (!listeners) {
32 | return;
33 | }
34 | var idx = listeners.indexOf(listener);
35 | if (idx < 0) {
36 | return;
37 | }
38 | listeners.splice(idx, 1);
39 | };
40 |
41 | EventEmitter.prototype.on = EventEmitter.prototype.addListener;
42 |
43 | EventEmitter.prototype.once = function(name, listener) {
44 | var f;
45 | f = function() {
46 | listener.apply(this, arguments);
47 | this.removeListener(name, f);
48 | }.bind(this);
49 | this.addListener(name, f);
50 | };
51 |
52 | EventEmitter.prototype.removeAllListeners = function(name) {
53 | if ('undefined' === typeof name) {
54 | this._listeners = {};
55 | } else {
56 | this._listeners[name] = [];
57 | }
58 | };
59 |
60 | EventEmitter.prototype.emit = function(name) {
61 | var listeners = this.listeners(name);
62 | var eventArgs = Array.prototype.slice.call(arguments, 1);
63 | for (var i = 0, len = listeners.length; i < len; ++i) {
64 | listeners[i].apply(null, eventArgs);
65 | }
66 | };
67 |
68 | window.EventEmitter = EventEmitter;
69 |
70 | })();
71 |
--------------------------------------------------------------------------------
/demos/neuralnet/image_demos/imgclass/images.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "image"
6 | "io/ioutil"
7 | "os"
8 | "path/filepath"
9 | "strings"
10 |
11 | "github.com/unixpickle/num-analysis/linalg"
12 | )
13 |
14 | const (
15 | ImageDepth = 3
16 | )
17 |
18 | func LoadTrainingImages(dir string) (imgs map[string][]linalg.Vector, width, height int,
19 | err error) {
20 | dirContents, err := ioutil.ReadDir(dir)
21 | if err != nil {
22 | return
23 | }
24 | imgs = map[string][]linalg.Vector{}
25 | for _, item := range dirContents {
26 | if !item.IsDir() {
27 | continue
28 | }
29 | category := item.Name()
30 | subPath := filepath.Join(dir, item.Name())
31 | subContents, err := ioutil.ReadDir(subPath)
32 | if err != nil {
33 | return nil, 0, 0, err
34 | }
35 | for _, subItem := range subContents {
36 | if strings.HasPrefix(subItem.Name(), ".") {
37 | continue
38 | }
39 | imgPath := filepath.Join(subPath, subItem.Name())
40 | img, w, h, err := ReadImageFile(imgPath)
41 | if err != nil {
42 | return nil, 0, 0, fmt.Errorf("failed to read image %s: %s", imgPath,
43 | err.Error())
44 | } else if width == 0 && height == 0 {
45 | width = w
46 | height = h
47 | } else if w != width || h != height {
48 | return nil, 0, 0, fmt.Errorf("expected dimensions %dx%d but got %dx%d: %s",
49 | width, height, w, h, imgPath)
50 | }
51 | imgs[category] = append(imgs[category], img)
52 | }
53 | }
54 | return
55 | }
56 |
57 | func ReadImageFile(path string) (data linalg.Vector, width, height int, err error) {
58 | f, err := os.Open(path)
59 | if err != nil {
60 | return
61 | }
62 | defer f.Close()
63 | img, _, err := image.Decode(f)
64 | if err != nil {
65 | return
66 | }
67 | data = make(linalg.Vector, 0, img.Bounds().Dx()*img.Bounds().Dy()*ImageDepth)
68 | width = img.Bounds().Dx()
69 | height = img.Bounds().Dy()
70 | for y := 0; y < height; y++ {
71 | for x := 0; x < width; x++ {
72 | r, g, b, _ := img.At(x, y).RGBA()
73 | data = append(data, float64(r)/0xffff, float64(g)/0xffff, float64(b)/0xffff)
74 | }
75 | }
76 | return
77 | }
78 |
--------------------------------------------------------------------------------
/demos/objectrecog/scripts/eventemitter.js:
--------------------------------------------------------------------------------
1 | // eventemitter.js version 0.1.1.
2 | (function() {
3 | // An EventEmitter makes it easy to emit and subscribe to events.
4 | function EventEmitter() {
5 | this._listeners = {};
6 | }
7 |
8 | EventEmitter.makeSubclass = function(p) {
9 | var keys = Object.keys(EventEmitter.prototype);
10 | for (var i = 0, len = keys.length; i < len; ++i) {
11 | var key = keys[i];
12 | p[key] = EventEmitter.prototype[key];
13 | }
14 | };
15 |
16 | EventEmitter.prototype.listeners = function(name) {
17 | return (this._listeners[name] || []).slice();
18 | };
19 |
20 | EventEmitter.prototype.addListener = function(name, listener) {
21 | var listeners = this._listeners[name];
22 | if (!listeners) {
23 | this._listeners[name] = [listener];
24 | } else {
25 | listeners.push(listener);
26 | }
27 | };
28 |
29 | EventEmitter.prototype.removeListener = function(name, listener) {
30 | var listeners = this._listeners[name];
31 | if (!listeners) {
32 | return;
33 | }
34 | var idx = listeners.indexOf(listener);
35 | if (idx < 0) {
36 | return;
37 | }
38 | listeners.splice(idx, 1);
39 | };
40 |
41 | EventEmitter.prototype.on = EventEmitter.prototype.addListener;
42 |
43 | EventEmitter.prototype.once = function(name, listener) {
44 | var f;
45 | f = function() {
46 | listener.apply(this, arguments);
47 | this.removeListener(name, f);
48 | }.bind(this);
49 | this.addListener(name, f);
50 | };
51 |
52 | EventEmitter.prototype.removeAllListeners = function(name) {
53 | if ('undefined' === typeof name) {
54 | this._listeners = {};
55 | } else {
56 | this._listeners[name] = [];
57 | }
58 | };
59 |
60 | EventEmitter.prototype.emit = function(name) {
61 | var listeners = this.listeners(name);
62 | var eventArgs = Array.prototype.slice.call(arguments, 1);
63 | for (var i = 0, len = listeners.length; i < len; ++i) {
64 | listeners[i].apply(null, eventArgs);
65 | }
66 | };
67 |
68 | window.EventEmitter = EventEmitter;
69 |
70 | })();
71 |
--------------------------------------------------------------------------------
/rbf/least_squares.go:
--------------------------------------------------------------------------------
1 | package rbf
2 |
3 | import (
4 | "github.com/unixpickle/autofunc"
5 | "github.com/unixpickle/num-analysis/linalg"
6 | "github.com/unixpickle/num-analysis/linalg/leastsquares"
7 | "github.com/unixpickle/sgd"
8 | "github.com/unixpickle/weakai/neuralnet"
9 | )
10 |
11 | // LeastSquares uses a least-squares approach to train the
12 | // output layer of a network.
13 | //
14 | // The s argument is a set of neuralnet.VectorSamples.
15 | //
16 | // The bs argument specifies the batch size for applying
17 | // the network.
18 | //
19 | // The least squares solution is returned and can be used
20 | // as n.OutLayer.
21 | // The value of n.OutLayer may be nil, but the other
22 | // layers of n should be set.
23 | func LeastSquares(n *Network, s sgd.SampleSet, bs int) *neuralnet.DenseLayer {
24 | comp := autofunc.ComposedBatcher{
25 | n.DistLayer,
26 | &autofunc.FuncBatcher{
27 | F: autofunc.ComposedFunc{n.ScaleLayer, n.ExpLayer},
28 | },
29 | }
30 |
31 | mat := linalg.NewMatrix(s.Len(), n.DistLayer.NumCenters())
32 | var outData linalg.Vector
33 | for i := 0; i < s.Len(); i += bs {
34 | b := bs
35 | if b+i > s.Len() {
36 | b = s.Len() - i
37 | }
38 |
39 | var input linalg.Vector
40 | for j := 0; j < b; j++ {
41 | sample := s.GetSample(j + i).(neuralnet.VectorSample)
42 | input = append(input, sample.Input...)
43 | outData = append(outData, sample.Output...)
44 | }
45 |
46 | res := comp.Batch(&autofunc.Variable{Vector: input}, b)
47 | copy(mat.Data[i*mat.Cols:], res.Output())
48 | }
49 |
50 | outMat := linalg.NewMatrix(s.Len(), len(outData)/s.Len())
51 | outMat.Data = outData
52 |
53 | solver := leastsquares.NewSolver(mat)
54 | var solutionData linalg.Vector
55 | for col := 0; col < outMat.Cols; col++ {
56 | outVec := outMat.Col(col)
57 | solution := solver.Solve(outVec)
58 | solutionData = append(solutionData, solution...)
59 | }
60 |
61 | resLayer := neuralnet.NewDenseLayer(n.DistLayer.NumCenters(), outMat.Cols)
62 | copy(resLayer.Weights.Data.Vector, solutionData)
63 | for i := range resLayer.Biases.Var.Vector {
64 | resLayer.Biases.Var.Vector[i] = 0
65 | }
66 | return resLayer
67 | }
68 |
--------------------------------------------------------------------------------
/rnn/seqtoseq/util.go:
--------------------------------------------------------------------------------
1 | package seqtoseq
2 |
3 | import (
4 | "sort"
5 |
6 | "github.com/unixpickle/autofunc"
7 | "github.com/unixpickle/num-analysis/linalg"
8 | "github.com/unixpickle/sgd"
9 | "github.com/unixpickle/weakai/neuralnet"
10 | )
11 |
12 | func costFuncDeriv(c neuralnet.CostFunc, expected, actual linalg.Vector) linalg.Vector {
13 | variable := &autofunc.Variable{Vector: actual}
14 | result := make(linalg.Vector, len(actual))
15 | res := c.Cost(expected, variable)
16 | res.PropagateGradient([]float64{1}, autofunc.Gradient{variable: result})
17 | return result
18 | }
19 |
20 | func costFuncRDeriv(c neuralnet.CostFunc, expected, actual,
21 | actualR linalg.Vector) (deriv, rDeriv linalg.Vector) {
22 | variable := &autofunc.RVariable{
23 | Variable: &autofunc.Variable{Vector: actual},
24 | ROutputVec: actualR,
25 | }
26 | deriv = make(linalg.Vector, len(actual))
27 | rDeriv = make(linalg.Vector, len(actual))
28 | res := c.CostR(autofunc.RVector{}, expected, variable)
29 | res.PropagateRGradient([]float64{1}, []float64{0},
30 | autofunc.RGradient{variable.Variable: rDeriv},
31 | autofunc.Gradient{variable.Variable: deriv})
32 | return
33 | }
34 |
35 | // sampleSetSlice converts a sample set into a slice
36 | // of Samples.
37 | func sampleSetSlice(s sgd.SampleSet) []Sample {
38 | res := make([]Sample, s.Len())
39 | for i := 0; i < s.Len(); i++ {
40 | res[i] = s.GetSample(i).(Sample)
41 | }
42 | return res
43 | }
44 |
45 | // sortSamples sorts the Samples in a SampleSet by size,
46 | // with the longest sequences coming first.
47 | func sortSeqs(s sgd.SampleSet) sgd.SampleSet {
48 | origSet := sampleSetSlice(s)
49 | res := make(seqSorter, len(origSet))
50 | copy(res, origSet)
51 | sort.Sort(res)
52 |
53 | resSet := make(sgd.SliceSampleSet, len(res))
54 | for i, x := range res {
55 | resSet[i] = x
56 | }
57 | return resSet
58 | }
59 |
60 | type seqSorter []Sample
61 |
62 | func (s seqSorter) Len() int {
63 | return len(s)
64 | }
65 |
66 | func (s seqSorter) Less(i, j int) bool {
67 | return len(s[i].Inputs) > len(s[j].Inputs)
68 | }
69 |
70 | func (s seqSorter) Swap(i, j int) {
71 | s[i], s[j] = s[j], s[i]
72 | }
73 |
--------------------------------------------------------------------------------
/demos/neuralnet/image_demos/imgclass/dream.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "image"
6 | "image/color"
7 | "image/png"
8 | "io/ioutil"
9 | "log"
10 | "math/rand"
11 | "os"
12 |
13 | "github.com/unixpickle/autofunc"
14 | "github.com/unixpickle/num-analysis/linalg"
15 | "github.com/unixpickle/weakai/neuralnet"
16 | )
17 |
18 | func DreamCmd(netPath, imgPath string) {
19 | networkData, err := ioutil.ReadFile(netPath)
20 | if err != nil {
21 | fmt.Fprintln(os.Stderr, "Error reading network:", err)
22 | os.Exit(1)
23 | }
24 | network, err := neuralnet.DeserializeNetwork(networkData)
25 | if err != nil {
26 | fmt.Fprintln(os.Stderr, "Error deserializing network:", err)
27 | os.Exit(1)
28 | }
29 |
30 | convIn := network[1].(*neuralnet.ConvLayer)
31 | inputImage := &autofunc.Variable{
32 | Vector: make(linalg.Vector, convIn.InputWidth*convIn.InputHeight*
33 | convIn.InputDepth),
34 | }
35 | for i := range inputImage.Vector {
36 | inputImage.Vector[i] = rand.Float64()*0.01 + 0.5
37 | }
38 |
39 | desiredOut := linalg.Vector{0, 1}
40 | cost := neuralnet.DotCost{}
41 | grad := autofunc.NewGradient([]*autofunc.Variable{inputImage})
42 | for i := 0; i < 1000; i++ {
43 | output := network.Apply(inputImage)
44 | costOut := cost.Cost(desiredOut, output)
45 | grad.Zero()
46 | log.Println("cost is", costOut.Output()[0])
47 | costOut.PropagateGradient(linalg.Vector{1}, grad)
48 | grad.AddToVars(-0.01)
49 | }
50 |
51 | newImage := image.NewRGBA(image.Rect(0, 0, convIn.InputWidth, convIn.InputHeight))
52 | var idx int
53 | for y := 0; y < convIn.InputHeight; y++ {
54 | for x := 0; x < convIn.InputWidth; x++ {
55 | r := uint8(0xff * inputImage.Vector[idx])
56 | g := uint8(0xff * inputImage.Vector[idx+1])
57 | b := uint8(0xff * inputImage.Vector[idx+2])
58 | newImage.SetRGBA(x, y, color.RGBA{
59 | R: r,
60 | G: g,
61 | B: b,
62 | A: 0xff,
63 | })
64 | idx += 3
65 | }
66 | }
67 |
68 | output, err := os.Create(imgPath)
69 | if err != nil {
70 | fmt.Fprintln(os.Stderr, "Failed to create output file:", err)
71 | os.Exit(1)
72 | }
73 | defer output.Close()
74 | png.Encode(output, newImage)
75 | }
76 |
--------------------------------------------------------------------------------
/demos/rbm/mnist_reconstruct/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "log"
5 | "runtime"
6 |
7 | "github.com/unixpickle/mnist"
8 | "github.com/unixpickle/weakai/rbm"
9 | )
10 |
11 | const (
12 | ImageSize = 28 * 28
13 | GibbsSteps = 10
14 | BigStepSize = 1e-2
15 | BigEpochs = 100
16 | SmallStepSize = 1e-3
17 | SmallEpochs = 100
18 |
19 | ReconstructionGridSize = 5
20 | )
21 |
22 | var HiddenSizes = []int{300, 100}
23 |
24 | func main() {
25 | training := mnist.LoadTrainingDataSet()
26 | samples := make([][]bool, len(training.Samples))
27 | for i, sample := range training.Samples {
28 | samples[i] = make([]bool, len(sample.Intensities))
29 | for j, x := range sample.Intensities {
30 | if x > 0.5 {
31 | samples[i][j] = true
32 | }
33 | }
34 | }
35 |
36 | layers := buildLayers()
37 | trainer := rbm.Trainer{
38 | GibbsSteps: GibbsSteps,
39 | StepSize: BigStepSize,
40 | Epochs: BigEpochs,
41 | BatchSize: runtime.GOMAXPROCS(0),
42 | }
43 | log.Println("Training...")
44 | trainer.TrainDeep(layers, samples[:1000])
45 | trainer.StepSize = SmallStepSize
46 | trainer.Epochs = SmallEpochs
47 | trainer.TrainDeep(layers, samples[:1000])
48 | log.Println("Generating outputs...")
49 |
50 | testingSamples := mnist.LoadTestingDataSet()
51 |
52 | mnist.SaveReconstructionGrid("output.png", func(img []float64) []float64 {
53 | return reconstruct(layers, img)
54 | }, testingSamples, ReconstructionGridSize, ReconstructionGridSize)
55 | }
56 |
57 | func buildLayers() rbm.DBN {
58 | res := make(rbm.DBN, len(HiddenSizes))
59 | for i, size := range HiddenSizes {
60 | inputSize := ImageSize
61 | if i > 0 {
62 | inputSize = HiddenSizes[i-1]
63 | }
64 | res[i] = rbm.NewRBM(inputSize, size)
65 | }
66 | return res
67 | }
68 |
69 | func reconstruct(dbn rbm.DBN, img []float64) []float64 {
70 | binaryInput := make([]bool, len(img))
71 | for i, x := range img {
72 | if x > 0.5 {
73 | binaryInput[i] = true
74 | }
75 | }
76 | output := dbn.Sample(nil, binaryInput)
77 | binaryInput = dbn.SampleInput(nil, output)
78 | res := make([]float64, len(binaryInput))
79 | for i, b := range binaryInput {
80 | if b {
81 | res[i] = 1
82 | }
83 | }
84 | return res
85 | }
86 |
--------------------------------------------------------------------------------
/rbm/train.go:
--------------------------------------------------------------------------------
1 | package rbm
2 |
3 | import (
4 | "math/rand"
5 | "runtime"
6 | )
7 |
8 | // A Trainer stores parameters for training an RBM.
9 | type Trainer struct {
10 | GibbsSteps int
11 | StepSize float64
12 | Epochs int
13 | BatchSize int
14 | }
15 |
16 | // Train trains the RBM for the supplied inputs.
17 | func (t *Trainer) Train(r *RBM, inputs [][]bool) {
18 | procCount := runtime.GOMAXPROCS(0)
19 | inputChan := make(chan []bool, t.BatchSize)
20 | outputChan := make(chan *RBMGradient, t.BatchSize)
21 |
22 | defer close(inputChan)
23 |
24 | for i := 0; i < procCount; i++ {
25 | go func() {
26 | source := rand.NewSource(rand.Int63())
27 | gen := rand.New(source)
28 | for input := range inputChan {
29 | grad := r.LogLikelihoodGradient(gen, [][]bool{input}, t.GibbsSteps)
30 | outputChan <- grad
31 | }
32 | }()
33 | }
34 |
35 | for i := 0; i < t.Epochs; i++ {
36 | perm := rand.Perm(len(inputs))
37 | for j := 0; j < len(inputs); j += t.BatchSize {
38 | batchCount := t.BatchSize
39 | if j+batchCount > len(inputs) {
40 | batchCount = len(inputs) - j
41 | }
42 | for k := j; k < j+batchCount; k++ {
43 | inputChan <- inputs[perm[k]]
44 | }
45 | var batch *RBMGradient
46 | for k := 0; k < batchCount; k++ {
47 | grad := <-outputChan
48 | if batch == nil {
49 | batch = grad
50 | } else {
51 | batch.HiddenBiases.Add(grad.HiddenBiases)
52 | batch.VisibleBiases.Add(grad.VisibleBiases)
53 | batch.Weights.Add(grad.Weights)
54 | }
55 | }
56 | r.HiddenBiases.Add(batch.HiddenBiases.Scale(t.StepSize))
57 | r.VisibleBiases.Add(batch.VisibleBiases.Scale(t.StepSize))
58 | r.Weights.Add(batch.Weights.Scale(t.StepSize))
59 | }
60 | }
61 | }
62 |
63 | // TrainDeep performs pre-training on a DBN.
64 | // The layers are ordered from the input layer to
65 | // the output layer.
66 | func (t *Trainer) TrainDeep(layers DBN, inputs [][]bool) {
67 | layerInputs := inputs
68 | for _, layer := range layers {
69 | t.Train(layer, layerInputs)
70 | newInputs := make([][]bool, len(layerInputs))
71 | for i, input := range layerInputs {
72 | newInputs[i] = make([]bool, len(layer.HiddenBiases))
73 | layer.SampleHidden(nil, newInputs[i], input)
74 | }
75 | layerInputs = newInputs
76 | }
77 | }
78 |
--------------------------------------------------------------------------------
/neuralnet/unstack_layer_test.go:
--------------------------------------------------------------------------------
1 | package neuralnet
2 |
3 | import (
4 | "math"
5 | "math/rand"
6 | "testing"
7 |
8 | "github.com/unixpickle/autofunc"
9 | "github.com/unixpickle/autofunc/functest"
10 | "github.com/unixpickle/num-analysis/linalg"
11 | )
12 |
13 | func TestUnstackLayerInverse(t *testing.T) {
14 | // This test utilizes the fact that gradients have to
15 | // be un-unstacked when they are propagated backwards.
16 | // Thus, we can check that unstacking and un-unstacking
17 | // are truly inverses as they should be.
18 |
19 | width := 15
20 | height := 13
21 | depth := 18
22 |
23 | inputVal := make(linalg.Vector, width*height*depth)
24 | for i := range inputVal {
25 | inputVal[i] = rand.Float64()*2 - 1
26 | }
27 | variable := &autofunc.Variable{inputVal}
28 |
29 | layer := &UnstackLayer{
30 | InputWidth: width,
31 | InputHeight: height,
32 | InputDepth: depth,
33 | InverseStride: 3,
34 | }
35 | output := layer.Apply(variable)
36 |
37 | outGrad := make(linalg.Vector, len(output.Output()))
38 | copy(outGrad, output.Output())
39 | grad := autofunc.NewGradient([]*autofunc.Variable{variable})
40 | output.PropagateGradient(outGrad, grad)
41 |
42 | original := grad[variable]
43 | if len(original) != len(inputVal) {
44 | t.Fatalf("expected output length %d got %d", len(inputVal), len(original))
45 | }
46 |
47 | for i, x := range inputVal {
48 | a := original[i]
49 | if math.Abs(a-x) > 1e-6 {
50 | t.Fatalf("entry %d should be %f but got %f", i, x, a)
51 | }
52 | }
53 | }
54 |
55 | func TestUnstackLayerRProp(t *testing.T) {
56 | width := 3
57 | height := 4
58 | depth := 18
59 |
60 | inputVal := make(linalg.Vector, width*height*depth)
61 | inputR := make(linalg.Vector, len(inputVal))
62 | for i := range inputVal {
63 | inputVal[i] = rand.Float64()*2 - 1
64 | inputR[i] = rand.Float64()*2 - 1
65 | }
66 | variable := &autofunc.Variable{inputVal}
67 | rVec := autofunc.RVector{
68 | variable: inputR,
69 | }
70 |
71 | layer := &UnstackLayer{
72 | InputWidth: width,
73 | InputHeight: height,
74 | InputDepth: depth,
75 | InverseStride: 3,
76 | }
77 |
78 | funcTest := &functest.RFuncChecker{
79 | F: layer,
80 | Vars: []*autofunc.Variable{variable},
81 | Input: variable,
82 | RV: rVec,
83 | }
84 | funcTest.FullCheck(t)
85 | }
86 |
--------------------------------------------------------------------------------
/neuralnet/single_rgradienter.go:
--------------------------------------------------------------------------------
1 | package neuralnet
2 |
3 | import (
4 | "github.com/unixpickle/autofunc"
5 | "github.com/unixpickle/num-analysis/linalg"
6 | "github.com/unixpickle/sgd"
7 | )
8 |
9 | // SingleRGradienter is an RGradienter that acts
10 | // on an underlying SingleLearner to compute
11 | // gradients for sets of BasicSamples.
12 | //
13 | // SingleRGradienter is good for nets with very few
14 | // parameters, where most of the training overhead
15 | // comes from GC overhead, function calls, etc.
16 | //
17 | // Once you use a SingleRGradienter with a Learner,
18 | // you should not ever use the same SingleRGradienter
19 | // for any learners with different parameters.
20 | type SingleRGradienter struct {
21 | Learner SingleLearner
22 | CostFunc CostFunc
23 |
24 | gradCache autofunc.Gradient
25 | rgradCache autofunc.RGradient
26 | }
27 |
28 | func (b *SingleRGradienter) Gradient(s sgd.SampleSet) autofunc.Gradient {
29 | if b.gradCache == nil {
30 | b.gradCache = autofunc.NewGradient(b.Learner.Parameters())
31 | } else {
32 | b.gradCache.Zero()
33 | }
34 |
35 | for i := 0; i < s.Len(); i++ {
36 | sample := s.GetSample(i)
37 | vs := sample.(VectorSample)
38 | output := vs.Output
39 | inVar := &autofunc.Variable{vs.Input}
40 | result := b.Learner.Apply(inVar)
41 | cost := b.CostFunc.Cost(output, result)
42 | cost.PropagateGradient(linalg.Vector{1}, b.gradCache)
43 | }
44 |
45 | return b.gradCache
46 | }
47 |
48 | func (b *SingleRGradienter) RGradient(rv autofunc.RVector, s sgd.SampleSet) (autofunc.Gradient,
49 | autofunc.RGradient) {
50 | if b.gradCache == nil {
51 | b.gradCache = autofunc.NewGradient(b.Learner.Parameters())
52 | } else {
53 | b.gradCache.Zero()
54 | }
55 | if b.rgradCache == nil {
56 | b.rgradCache = autofunc.NewRGradient(b.Learner.Parameters())
57 | } else {
58 | b.rgradCache.Zero()
59 | }
60 |
61 | for i := 0; i < s.Len(); i++ {
62 | sample := s.GetSample(i)
63 | vs := sample.(VectorSample)
64 | output := vs.Output
65 | inVar := &autofunc.Variable{vs.Input}
66 | rVar := autofunc.NewRVariable(inVar, rv)
67 | result := b.Learner.ApplyR(rv, rVar)
68 | cost := b.CostFunc.CostR(rv, output, result)
69 | cost.PropagateRGradient(linalg.Vector{1}, linalg.Vector{0},
70 | b.rgradCache, b.gradCache)
71 | }
72 |
73 | return b.gradCache, b.rgradCache
74 | }
75 |
--------------------------------------------------------------------------------
/neuralnet/gauss_noise.go:
--------------------------------------------------------------------------------
1 | package neuralnet
2 |
3 | import (
4 | "encoding/json"
5 | "math/rand"
6 |
7 | "github.com/unixpickle/autofunc"
8 | "github.com/unixpickle/num-analysis/linalg"
9 | )
10 |
11 | // A GaussNoiseLayer adds random gaussian noise to
12 | // its input vectors with the given standard
13 | // deviation.
14 | // This is similar to DropoutLayer, except that it
15 | // modifies the inputs instead of dropping them.
16 | type GaussNoiseLayer struct {
17 | // Stddev is the standard devation of the noise
18 | // added to the inputs.
19 | Stddev float64
20 |
21 | // Training is true if noise should be applied.
22 | Training bool
23 | }
24 |
25 | func DeserializeGaussNoiseLayer(d []byte) (*GaussNoiseLayer, error) {
26 | var res GaussNoiseLayer
27 | if err := json.Unmarshal(d, &res); err != nil {
28 | return nil, err
29 | }
30 | return &res, nil
31 | }
32 |
33 | func (g *GaussNoiseLayer) Apply(in autofunc.Result) autofunc.Result {
34 | if g.Training {
35 | return autofunc.Add(in, g.noise(len(in.Output())))
36 | } else {
37 | return in
38 | }
39 | }
40 |
41 | func (g *GaussNoiseLayer) ApplyR(v autofunc.RVector, in autofunc.RResult) autofunc.RResult {
42 | if g.Training {
43 | return autofunc.AddR(in, g.noiseR(len(in.Output())))
44 | } else {
45 | return in
46 | }
47 | }
48 |
49 | func (g *GaussNoiseLayer) Batch(in autofunc.Result, n int) autofunc.Result {
50 | return g.Apply(in)
51 | }
52 |
53 | func (g *GaussNoiseLayer) BatchR(v autofunc.RVector, in autofunc.RResult, n int) autofunc.RResult {
54 | return g.ApplyR(v, in)
55 | }
56 |
57 | func (g *GaussNoiseLayer) SerializerType() string {
58 | return serializerTypeGaussNoiseLayer
59 | }
60 |
61 | func (g *GaussNoiseLayer) Serialize() ([]byte, error) {
62 | return json.Marshal(g)
63 | }
64 |
65 | func (g *GaussNoiseLayer) noise(size int) autofunc.Result {
66 | vec := make(linalg.Vector, size)
67 | for i := range vec {
68 | vec[i] = rand.NormFloat64() * g.Stddev
69 | }
70 | return &autofunc.Variable{Vector: vec}
71 | }
72 |
73 | func (g *GaussNoiseLayer) noiseR(size int) autofunc.RResult {
74 | vec := make(linalg.Vector, size)
75 | for i := range vec {
76 | vec[i] = rand.NormFloat64() * g.Stddev
77 | }
78 | return &autofunc.RVariable{
79 | Variable: &autofunc.Variable{Vector: vec},
80 | ROutputVec: make(linalg.Vector, size),
81 | }
82 | }
83 |
--------------------------------------------------------------------------------
/neuralnet/residual_layer.go:
--------------------------------------------------------------------------------
1 | package neuralnet
2 |
3 | import (
4 | "github.com/unixpickle/autofunc"
5 | "github.com/unixpickle/serializer"
6 | )
7 |
8 | // A ResidualLayer wraps a set of layers and adds its input
9 | // the layers' final output.
10 | type ResidualLayer struct {
11 | Network Network
12 | }
13 |
14 | // DeserializeResidualLayer deserializes a ResidualLayer.
15 | func DeserializeResidualLayer(d []byte) (*ResidualLayer, error) {
16 | var n Network
17 | if err := serializer.DeserializeAny(d, &n); err != nil {
18 | return nil, err
19 | }
20 | return &ResidualLayer{Network: n}, nil
21 | }
22 |
23 | // Apply applies the layer.
24 | func (r *ResidualLayer) Apply(in autofunc.Result) autofunc.Result {
25 | return autofunc.Pool(in, func(inPool autofunc.Result) autofunc.Result {
26 | return autofunc.Add(inPool, r.Network.Apply(inPool))
27 | })
28 | }
29 |
30 | // ApplyR applies the layer.
31 | func (r *ResidualLayer) ApplyR(rv autofunc.RVector, in autofunc.RResult) autofunc.RResult {
32 | return autofunc.PoolR(in, func(inPool autofunc.RResult) autofunc.RResult {
33 | return autofunc.AddR(inPool, r.Network.ApplyR(rv, inPool))
34 | })
35 | }
36 |
37 | // Batch applies the layer in batch.
38 | func (r *ResidualLayer) Batch(in autofunc.Result, n int) autofunc.Result {
39 | b := r.Network.BatchLearner()
40 | return autofunc.Pool(in, func(inPool autofunc.Result) autofunc.Result {
41 | return autofunc.Add(inPool, b.Batch(inPool, n))
42 | })
43 | }
44 |
45 | // BatchR applies the layer in batch.
46 | func (r *ResidualLayer) BatchR(rv autofunc.RVector, in autofunc.RResult, n int) autofunc.RResult {
47 | b := r.Network.BatchLearner()
48 | return autofunc.PoolR(in, func(inPool autofunc.RResult) autofunc.RResult {
49 | return autofunc.AddR(inPool, b.BatchR(rv, inPool, n))
50 | })
51 | }
52 |
53 | // Parameters returns the parameters of the network.
54 | func (r *ResidualLayer) Parameters() []*autofunc.Variable {
55 | return r.Network.Parameters()
56 | }
57 |
58 | // SerializerType returns the unique ID used to serialize
59 | // a ResidualLayer with the serializer package.
60 | func (r *ResidualLayer) SerializerType() string {
61 | return serializerTypeResidualLayer
62 | }
63 |
64 | // Serialize serializes the layer.
65 | func (r *ResidualLayer) Serialize() ([]byte, error) {
66 | return serializer.SerializeAny(r.Network)
67 | }
68 |
--------------------------------------------------------------------------------
/demos/hopfield/scripts/hopfield.js:
--------------------------------------------------------------------------------
1 | (function() {
2 |
3 | var INTERVAL = 20;
4 | var ACTIVATION_THRESHOLD = 0;
5 | var CONVERGE_NODES = 5;
6 |
7 | function HopfieldNet(nodeCount) {
8 | this._weights = [];
9 | this._values = [];
10 | for (var i = 0; i < nodeCount; ++i) {
11 | this._weights[i] = [];
12 | this._values[i] = -1;
13 | for (var j = 0; j < nodeCount; ++j) {
14 | this._weights[i][j] = 1;
15 | }
16 | }
17 | }
18 |
19 | HopfieldNet.prototype.train = function(samples) {
20 | this._resetWeights();
21 | var nodeCount = this._values.length;
22 | for (var i = 0; i < nodeCount; ++i) {
23 | for (var j = 0; j < nodeCount; ++j) {
24 | for (var k = 0, len = samples.length; k < len; ++k) {
25 | if (samples[k][i] === samples[k][j]) {
26 | this._weights[i][j] += 1 / len;
27 | } else {
28 | this._weights[i][j] -= 1 / len;
29 | }
30 | }
31 | }
32 | }
33 | };
34 |
35 | HopfieldNet.prototype.vector = function() {
36 | return this._values.slice();
37 | };
38 |
39 | HopfieldNet.prototype.setVector = function(values) {
40 | for (var i = 0, len = this._values.length; i < len; ++i) {
41 | this._values[i] = values[i];
42 | }
43 | };
44 |
45 | HopfieldNet.prototype.convergeStep = function() {
46 | for (var x = 0; x < CONVERGE_NODES; ++x) {
47 | this._convergeRandomNode();
48 | }
49 | };
50 |
51 | HopfieldNet.prototype._convergeRandomNode = function() {
52 | var nodeIndex = Math.floor(Math.random() * this._values.length);
53 | var sum = 0;
54 | for (var j = 0, len = this._values.length; j < len; ++j) {
55 | if (j === nodeIndex) {
56 | continue;
57 | }
58 | var w = this._weights[nodeIndex][j];
59 | var v = this._values[j];
60 | sum += w * v;
61 | }
62 | if (sum > ACTIVATION_THRESHOLD) {
63 | this._values[nodeIndex] = 1;
64 | } else {
65 | this._values[nodeIndex] = -1;
66 | }
67 | };
68 |
69 | HopfieldNet.prototype._resetWeights = function() {
70 | var nodeCount = this._values.length;
71 | for (var i = 0; i < nodeCount; ++i) {
72 | for (var j = 0; j < nodeCount; ++j) {
73 | this._weights[i][j] = 0;
74 | }
75 | }
76 | };
77 |
78 | window.app.HopfieldNet = HopfieldNet;
79 |
80 | })();
81 |
--------------------------------------------------------------------------------
/neuralnet/softmax_layer_test.go:
--------------------------------------------------------------------------------
1 | package neuralnet
2 |
3 | import (
4 | "math"
5 | "math/rand"
6 | "testing"
7 |
8 | "github.com/unixpickle/autofunc"
9 | )
10 |
11 | func TestSoftmaxLayerOutput(t *testing.T) {
12 | input := &autofunc.Variable{
13 | Vector: []float64{
14 | 0.175974, 0.764459, 0.573260, 0.105675, 0.320708,
15 | 0.554257, 0.028740, 0.826560, 0.290679, 0.208740,
16 | },
17 | }
18 | layer := SoftmaxLayer{}
19 | output := layer.Apply(input).Output()
20 | expOutput := []float64{
21 | 0.078301, 0.141040, 0.116494, 0.072985, 0.090495,
22 | 0.114302, 0.067581, 0.150076, 0.087818, 0.080909,
23 | }
24 | for i, x := range expOutput {
25 | actual := output[i]
26 | if math.Abs(actual-x) > 1e-5 {
27 | t.Errorf("invalid output %d: got %f expected %f", i, actual, x)
28 | }
29 | }
30 | }
31 |
32 | func TestLogSoftmaxLayerOutput(t *testing.T) {
33 | input := &autofunc.Variable{
34 | Vector: []float64{
35 | 0.175974, 0.764459, 0.573260, 0.105675, 0.320708,
36 | 0.554257, 0.028740, 0.826560, 0.290679, 0.208740,
37 | },
38 | }
39 | layer := LogSoftmaxLayer{}
40 | output := layer.Apply(input).Output()
41 | expOutput := []float64{
42 | -2.547194905, -1.958711741, -2.149915509, -2.617501338, -2.402460678,
43 | -2.168911211, -2.694428401, -1.896613447, -2.432488788, -2.514430213,
44 | }
45 | for i, x := range expOutput {
46 | actual := output[i]
47 | if math.Abs(actual-x) > 1e-5 {
48 | t.Errorf("invalid output %d: got %f expected %f", i, actual, x)
49 | }
50 | }
51 | }
52 |
53 | func BenchmarkSoftmaxForward(b *testing.B) {
54 | rand.Seed(123)
55 | inputVec := make([]float64, 3000)
56 | for i := range inputVec {
57 | inputVec[i] = rand.Float64()*5 - 2.5
58 | }
59 | inputVar := &autofunc.Variable{Vector: inputVec}
60 | layer := SoftmaxLayer{}
61 |
62 | b.ResetTimer()
63 | for i := 0; i < b.N; i++ {
64 | layer.Apply(inputVar)
65 | }
66 | }
67 |
68 | func BenchmarkSoftmaxBackProp(b *testing.B) {
69 | rand.Seed(123)
70 | inputVec := make([]float64, 3000)
71 | for i := range inputVec {
72 | inputVec[i] = rand.Float64()*5 - 2.5
73 | }
74 | inputVar := &autofunc.Variable{Vector: inputVec}
75 | outGrad := autofunc.NewGradient([]*autofunc.Variable{inputVar})
76 | layer := SoftmaxLayer{}
77 |
78 | b.ResetTimer()
79 | for i := 0; i < b.N; i++ {
80 | layer.Apply(inputVar).PropagateGradient(inputVec, outGrad)
81 | }
82 | }
83 |
--------------------------------------------------------------------------------
/boosting/boosting.go:
--------------------------------------------------------------------------------
1 | // Package boosting implements boosting algorithms such
2 | // as AdaBoost and the more general Gradient Boosting.
3 | package boosting
4 |
5 | import "github.com/unixpickle/num-analysis/linalg"
6 |
7 | // A SampleList represents an ordered list of training
8 | // samples.
9 | type SampleList interface {
10 | Len() int
11 | }
12 |
13 | // A Classifier classifies samples in a SampleList.
14 | type Classifier interface {
15 | // Classify returns a vector of classifications,
16 | // one per sample in the sample list.
17 | // A classification's sign indicates whether the
18 | // sample was positive or negative, whereas its
19 | // magnitude represents the confidence in the
20 | // classification.
21 | //
22 | // The resulting vector belongs to the caller, who
23 | // may modify the vector.
24 | Classify(s SampleList) linalg.Vector
25 | }
26 |
27 | // A Pool creates or reuses classifiers which are
28 | // tuned for particular sets of weighted samples.
29 | type Pool interface {
30 | // BestClassifier creates or reuses a Classifier
31 | // which classifies s well given the weighting.
32 | //
33 | // In other words, the classifier's output vector
34 | // on s should have as much correlation to the
35 | // weight vector as possible, where correlation is
36 | // measured by cosine distance.
37 | BestClassifier(s SampleList, weights linalg.Vector) Classifier
38 | }
39 |
40 | // SumClassifier classifies samples by adding the
41 | // results of other classifiers.
42 | type SumClassifier struct {
43 | // Classifiers is a list of classifiers whose
44 | // outputs are summed in a weighted fashion.
45 | Classifiers []Classifier
46 |
47 | // Weights contains one weight per classifier, where
48 | // weights serve a scaling coefficient for a
49 | // classifier's outputs.
50 | //
51 | // This must be the same length as Classifiers.
52 | Weights []float64
53 | }
54 |
55 | func (s *SumClassifier) Classify(list SampleList) linalg.Vector {
56 | if len(s.Classifiers) == 0 {
57 | return make(linalg.Vector, list.Len())
58 | } else if len(s.Classifiers) != len(s.Weights) {
59 | panic("classifier count must match weight count")
60 | }
61 | var res linalg.Vector
62 | for i, c := range s.Classifiers {
63 | w := s.Weights[i]
64 | if res == nil {
65 | res = c.Classify(list).Scale(w)
66 | } else {
67 | res.Add(c.Classify(list).Scale(w))
68 | }
69 | }
70 | return res
71 | }
72 |
--------------------------------------------------------------------------------
/svm/random_solver.go:
--------------------------------------------------------------------------------
1 | package svm
2 |
3 | import (
4 | "math"
5 | "math/rand"
6 | )
7 |
8 | // RandomlySolveLinear guesses random pre-images for hyperplane normals and returns the best guess,
9 | // judging based on which guess yields the maximum separation between any two samples while
10 | // maintaining an optimal margin size.
11 | //
12 | // This only works well for linear kernels.
13 | // For nonlinear kernels, this may never guess the correct solution, since said solution will be in
14 | // the transformed space and may not have a corresponding vector in the sample space.
15 | func RandomlySolveLinear(p *Problem, numGuesses int, maxEntry float64) *LinearClassifier {
16 | var bestClassifier *LinearClassifier
17 | var bestTotalError float64
18 | var bestMagnitude float64
19 |
20 | for i := 0; i < numGuesses; i++ {
21 | guess := randomSample(len(p.Positives[0].V), maxEntry)
22 | mag := p.Kernel(guess, guess)
23 | threshold := idealThresholdForGuess(guess, p)
24 |
25 | totalError := 0.0
26 | for _, pos := range p.Positives {
27 | totalError += math.Max(0, 1-(p.Kernel(guess, pos)+threshold))
28 | }
29 | for _, neg := range p.Negatives {
30 | totalError += math.Max(0, 1+(p.Kernel(guess, neg)+threshold))
31 | }
32 |
33 | if i == 0 || (totalError == bestTotalError && mag < bestMagnitude) ||
34 | totalError < bestTotalError {
35 | bestTotalError = totalError
36 | bestMagnitude = mag
37 | bestClassifier = &LinearClassifier{
38 | HyperplaneNormal: guess,
39 | Threshold: threshold,
40 | Kernel: p.Kernel,
41 | }
42 | }
43 | }
44 | return bestClassifier
45 | }
46 |
47 | func randomSample(dimension int, componentMax float64) Sample {
48 | vec := make([]float64, dimension)
49 | for i := range vec {
50 | vec[i] = (rand.Float64() - 0.5) * componentMax * 2
51 | }
52 | return Sample{V: vec}
53 | }
54 |
55 | func idealThresholdForGuess(guess Sample, p *Problem) float64 {
56 | var minPositiveDot float64
57 | for i, positive := range p.Positives {
58 | product := p.Kernel(guess, positive)
59 | if i == 0 || product < minPositiveDot {
60 | minPositiveDot = product
61 | }
62 | }
63 |
64 | var maxNegativeDot float64
65 | for i, negative := range p.Negatives {
66 | product := p.Kernel(guess, negative)
67 | if i == 0 || product > maxNegativeDot {
68 | maxNegativeDot = product
69 | }
70 | }
71 |
72 | return -(minPositiveDot + maxNegativeDot) / 2
73 | }
74 |
--------------------------------------------------------------------------------
/demos/nearestneighbors/prune_index/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "encoding/json"
5 | "fmt"
6 | "io/ioutil"
7 | "os"
8 | "strconv"
9 | "strings"
10 | )
11 |
12 | type IndexEntry struct {
13 | URL string `json:"url"`
14 | Keywords map[string]int `json:"keywords"`
15 | }
16 |
17 | func main() {
18 | if len(os.Args) != 4 {
19 | fmt.Fprintln(os.Stderr, "Usage: prune_index ")
20 | fmt.Fprintln(os.Stderr, "")
21 | fmt.Fprintln(os.Stderr, "The threshold argument is a number between 0 and 1.")
22 | fmt.Fprintln(os.Stderr, "A value of 0 will remove all keywords, 1 will keep all keywords.")
23 | os.Exit(1)
24 | }
25 |
26 | threshold, err := strconv.ParseFloat(os.Args[3], 64)
27 | if err != nil || threshold < 0 || threshold > 1 {
28 | fmt.Fprintln(os.Stderr, "Invalid threshold:", os.Args[3])
29 | os.Exit(1)
30 | }
31 |
32 | var entries []IndexEntry
33 | contents, err := ioutil.ReadFile(os.Args[1])
34 | if err != nil {
35 | fmt.Fprintln(os.Stderr, err)
36 | os.Exit(1)
37 | }
38 | if err := json.Unmarshal(contents, &entries); err != nil {
39 | fmt.Fprintln(os.Stderr, err)
40 | os.Exit(1)
41 | }
42 |
43 | ubiquity := keywordUbiquity(entries)
44 | keepKeywords := map[string]bool{}
45 | pruned := []string{}
46 | for word, ubiquity := range ubiquity {
47 | if ubiquity < threshold {
48 | keepKeywords[word] = true
49 | } else {
50 | pruned = append(pruned, word)
51 | }
52 | }
53 |
54 | fmt.Println("Pruned", len(pruned), "words:", strings.Join(pruned, ", "))
55 |
56 | filterKeywords(entries, keepKeywords)
57 | data, _ := json.Marshal(entries)
58 | if err := ioutil.WriteFile(os.Args[2], data, 0755); err != nil {
59 | fmt.Fprintln(os.Stderr, err)
60 | os.Exit(1)
61 | }
62 | }
63 |
64 | func keywordUbiquity(entries []IndexEntry) map[string]float64 {
65 | ubiquity := map[string]float64{}
66 | ubiquityPerPage := 1 / float64(len(entries))
67 | for _, entry := range entries {
68 | for word := range entry.Keywords {
69 | ubiquity[word] += ubiquityPerPage
70 | }
71 | }
72 | return ubiquity
73 | }
74 |
75 | func filterKeywords(entries []IndexEntry, filter map[string]bool) {
76 | for i, entry := range entries {
77 | newKeywords := map[string]int{}
78 | for word, count := range entry.Keywords {
79 | if filter[word] {
80 | newKeywords[word] = count
81 | }
82 | }
83 | entries[i] = IndexEntry{URL: entry.URL, Keywords: newKeywords}
84 | }
85 | }
86 |
--------------------------------------------------------------------------------
/demos/svm/image_categorizing/image.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "image"
5 | "image/color"
6 | _ "image/jpeg"
7 | "image/png"
8 | "os"
9 | )
10 |
11 | // Image is a black and white bitmap of pixels.
12 | type Image struct {
13 | Vector []float64
14 | Width int
15 | Height int
16 | }
17 |
18 | // ReadImage loads an image from an image file and generates a vector which represents its black and
19 | // white pixels.
20 | func ReadImage(path string) (*Image, error) {
21 | file, err := os.Open(path)
22 | if err != nil {
23 | return nil, err
24 | }
25 | image, _, err := image.Decode(file)
26 | if err != nil {
27 | return nil, err
28 | }
29 | res := &Image{
30 | Vector: make([]float64, image.Bounds().Dx()*image.Bounds().Dy()),
31 | Width: image.Bounds().Dx(),
32 | Height: image.Bounds().Dy(),
33 | }
34 | idx := 0
35 | for y := 0; y < image.Bounds().Dy(); y++ {
36 | for x := 0; x < image.Bounds().Dx(); x++ {
37 | r, g, b, _ := image.At(x, y).RGBA()
38 | black := float64(r+g+b) / float64(0xffff*3)
39 | res.Vector[idx] = black
40 | idx++
41 | }
42 | }
43 | return res, nil
44 | }
45 |
46 | // Crop crops this image to a specific sub-region, returning an image for that sub-region.
47 | func (i *Image) Crop(rect image.Rectangle) *Image {
48 | if rect.Min.X < 0 || rect.Min.Y < 0 || rect.Max.X > i.Width || rect.Max.Y > i.Height ||
49 | rect.Dx() < 0 || rect.Dy() < 0 {
50 | panic("invalid crop rectangle")
51 | }
52 | res := &Image{
53 | Vector: make([]float64, rect.Dx()*rect.Dy()),
54 | Width: rect.Dx(),
55 | Height: rect.Dy(),
56 | }
57 | idx := 0
58 | for y := 0; y < rect.Dy(); y++ {
59 | for x := 0; x < rect.Dx(); x++ {
60 | res.Vector[idx] = i.At(x+rect.Min.X, y+rect.Min.Y)
61 | idx++
62 | }
63 | }
64 | return res
65 | }
66 |
67 | // At returns the brightness value at a given point.
68 | func (i *Image) At(x, y int) float64 {
69 | if x < 0 || x >= i.Width || y < 0 || y >= i.Height {
70 | panic("point out of bounds")
71 | }
72 | return i.Vector[x+y*i.Width]
73 | }
74 |
75 | // WriteFile saves the image as a PNG at a path.
76 | func (i *Image) WriteFile(path string) error {
77 | output, err := os.Create(path)
78 | if err != nil {
79 | return err
80 | }
81 | defer output.Close()
82 | img := image.NewRGBA(image.Rect(0, 0, i.Width, i.Height))
83 | for y := 0; y < i.Height; y++ {
84 | for x := 0; x < i.Width; x++ {
85 | brightness := i.At(x, y)
86 | val := uint8(brightness * 0xff)
87 | img.Set(x, y, color.RGBA{val, val, val, 0xff})
88 | }
89 | }
90 | return png.Encode(output, img)
91 | }
92 |
--------------------------------------------------------------------------------
/neuralnet/rescale_layer.go:
--------------------------------------------------------------------------------
1 | package neuralnet
2 |
3 | import (
4 | "encoding/json"
5 |
6 | "github.com/unixpickle/autofunc"
7 | "github.com/unixpickle/num-analysis/linalg"
8 | )
9 |
10 | // RescaleLayer is a Layer which adds a bias to its
11 | // input and scales the translated input.
12 | // It is useful for ensuring that input samples have
13 | // a mean of 0 and a standard deviation of 1.
14 | type RescaleLayer struct {
15 | Bias float64
16 | Scale float64
17 | }
18 |
19 | func DeserializeRescaleLayer(d []byte) (*RescaleLayer, error) {
20 | var res RescaleLayer
21 | if err := json.Unmarshal(d, &res); err != nil {
22 | return nil, err
23 | }
24 | return &res, nil
25 | }
26 |
27 | func (r *RescaleLayer) Apply(in autofunc.Result) autofunc.Result {
28 | return autofunc.Scale(autofunc.AddScaler(in, r.Bias), r.Scale)
29 | }
30 |
31 | func (r *RescaleLayer) ApplyR(v autofunc.RVector, in autofunc.RResult) autofunc.RResult {
32 | return autofunc.ScaleR(autofunc.AddScalerR(in, r.Bias), r.Scale)
33 | }
34 |
35 | func (r *RescaleLayer) Serialize() ([]byte, error) {
36 | return json.Marshal(r)
37 | }
38 |
39 | func (r *RescaleLayer) SerializerType() string {
40 | return serializerTypeRescaleLayer
41 | }
42 |
43 | // VecRescaleLayer is similar to a RescaleLayer, but
44 | // it applies a different bias and scale to each entry
45 | // of its input vectors.
46 | type VecRescaleLayer struct {
47 | Biases linalg.Vector
48 | Scales linalg.Vector
49 | }
50 |
51 | func DeserializeVecRescaleLayer(d []byte) (*VecRescaleLayer, error) {
52 | var res VecRescaleLayer
53 | if err := json.Unmarshal(d, &res); err != nil {
54 | return nil, err
55 | }
56 | return &res, nil
57 | }
58 |
59 | func (v *VecRescaleLayer) Apply(in autofunc.Result) autofunc.Result {
60 | return autofunc.Mul(autofunc.Add(in, &autofunc.Variable{Vector: v.Biases}),
61 | &autofunc.Variable{Vector: v.Scales})
62 | }
63 |
64 | func (v *VecRescaleLayer) ApplyR(rv autofunc.RVector, in autofunc.RResult) autofunc.RResult {
65 | zeroVec := make(linalg.Vector, len(in.Output()))
66 | biases := &autofunc.RVariable{
67 | Variable: &autofunc.Variable{Vector: v.Biases},
68 | ROutputVec: zeroVec,
69 | }
70 | scales := &autofunc.RVariable{
71 | Variable: &autofunc.Variable{Vector: v.Scales},
72 | ROutputVec: zeroVec,
73 | }
74 | return autofunc.MulR(autofunc.AddR(in, biases), scales)
75 | }
76 |
77 | func (v *VecRescaleLayer) Serialize() ([]byte, error) {
78 | return json.Marshal(v)
79 | }
80 |
81 | func (v *VecRescaleLayer) SerializerType() string {
82 | return serializerTypeVecRescaleLayer
83 | }
84 |
--------------------------------------------------------------------------------
/neuralnet/cost_func_test.go:
--------------------------------------------------------------------------------
1 | package neuralnet
2 |
3 | import (
4 | "math"
5 | "math/rand"
6 | "testing"
7 |
8 | "github.com/unixpickle/autofunc"
9 | "github.com/unixpickle/autofunc/functest"
10 | "github.com/unixpickle/num-analysis/linalg"
11 | "github.com/unixpickle/sgd"
12 | )
13 |
14 | type meanSquaredTestFunc struct {
15 | Expected linalg.Vector
16 | }
17 |
18 | func (m meanSquaredTestFunc) Apply(in autofunc.Result) autofunc.Result {
19 | return MeanSquaredCost{}.Cost(m.Expected, in)
20 | }
21 |
22 | func (m meanSquaredTestFunc) ApplyR(v autofunc.RVector, in autofunc.RResult) autofunc.RResult {
23 | return MeanSquaredCost{}.CostR(v, m.Expected, in)
24 | }
25 |
26 | func TestMeanSquaredCostGradient(t *testing.T) {
27 | actual := &autofunc.Variable{make(linalg.Vector, 10)}
28 | expected := make(linalg.Vector, len(actual.Vector))
29 | for i := range expected {
30 | expected[i] = rand.Float64()
31 | actual.Vector[i] = rand.Float64()
32 | }
33 | funcTest := &functest.FuncChecker{
34 | F: meanSquaredTestFunc{expected},
35 | Vars: []*autofunc.Variable{actual},
36 | Input: actual,
37 | }
38 | funcTest.FullCheck(t)
39 | }
40 |
41 | func TestMeanSquaredCostRGradient(t *testing.T) {
42 | actual := &autofunc.Variable{make(linalg.Vector, 10)}
43 | expected := make(linalg.Vector, len(actual.Vector))
44 | rVector := autofunc.RVector{actual: make(linalg.Vector, len(expected))}
45 | for i := range expected {
46 | expected[i] = rand.Float64()
47 | actual.Vector[i] = rand.Float64()
48 | rVector[actual][i] = rand.Float64()
49 | }
50 | funcTest := &functest.RFuncChecker{
51 | F: meanSquaredTestFunc{expected},
52 | Vars: []*autofunc.Variable{actual},
53 | Input: actual,
54 | RV: rVector,
55 | }
56 | funcTest.FullCheck(t)
57 | }
58 |
59 | func TestTotalCostBatcher(t *testing.T) {
60 | net := Network{NewDenseLayer(2, 3)}
61 | samples := sgd.SliceSampleSet{
62 | VectorSample{Input: []float64{1, -1}, Output: []float64{1, -2, 0.4}},
63 | VectorSample{Input: []float64{1, 1}, Output: []float64{1, -3, -0.4}},
64 | VectorSample{Input: []float64{-1, -1}, Output: []float64{0, -2, 0.4}},
65 | VectorSample{Input: []float64{-1, 1}, Output: []float64{1, -2, 0.9}},
66 | VectorSample{Input: []float64{0.5, 0.75}, Output: []float64{-1, 2, 0.4}},
67 | }
68 | cf := MeanSquaredCost{}
69 | expected := TotalCost(cf, net, samples)
70 | for _, batchSize := range []int{1, 0, 3, 5, 10} {
71 | actual := TotalCostBatcher(cf, net.BatchLearner(), samples, batchSize)
72 | if math.Abs(actual-expected) > 1e-5 {
73 | t.Errorf("batch %d: expected %v got %v", batchSize, expected, actual)
74 | }
75 | }
76 | }
77 |
--------------------------------------------------------------------------------
/neuralnet/dropout.go:
--------------------------------------------------------------------------------
1 | package neuralnet
2 |
3 | import (
4 | "encoding/json"
5 | "math/rand"
6 |
7 | "github.com/unixpickle/autofunc"
8 | "github.com/unixpickle/num-analysis/linalg"
9 | )
10 |
11 | // DropoutLayer implements dropout regularization,
12 | // where random inputs are "dropped" each time the
13 | // layer is evaluated.
14 | //
15 | // A DropoutLayer can either be in training mode,
16 | // where inputs are dropped stochastically, or in
17 | // usage mode, where inputs are scaled to output
18 | // their expected values.
19 | //
20 | // Unlike normal autofunc.RFuncs, a DropoutLayer in
21 | // training mode may return different values each
22 | // time it is evaluated.
23 | // As a result, it will most likely fail traditional
24 | // autofunc tests which assume consistent functions.
25 | type DropoutLayer struct {
26 | // KeepProbability is the probability that an
27 | // individual input is not dropped at each
28 | // function evaluation.
29 | KeepProbability float64
30 |
31 | // Training is true if inputs should be dropped
32 | // stochastically rather than averaged.
33 | Training bool
34 | }
35 |
36 | func DeserializeDropoutLayer(d []byte) (*DropoutLayer, error) {
37 | var res DropoutLayer
38 | if err := json.Unmarshal(d, &res); err != nil {
39 | return nil, err
40 | }
41 | return &res, nil
42 | }
43 |
44 | func (d *DropoutLayer) Apply(in autofunc.Result) autofunc.Result {
45 | if d.Training {
46 | return autofunc.Mul(in, d.dropoutMask(len(in.Output())))
47 | } else {
48 | return autofunc.Scale(in, d.KeepProbability)
49 | }
50 | }
51 |
52 | func (d *DropoutLayer) ApplyR(v autofunc.RVector, in autofunc.RResult) autofunc.RResult {
53 | if d.Training {
54 | mask := d.dropoutMask(len(in.Output()))
55 | maskVar := autofunc.NewRVariable(mask, v)
56 | return autofunc.MulR(in, maskVar)
57 | } else {
58 | return autofunc.ScaleR(in, d.KeepProbability)
59 | }
60 | }
61 |
62 | func (d *DropoutLayer) Batch(in autofunc.Result, n int) autofunc.Result {
63 | return d.Apply(in)
64 | }
65 |
66 | func (d *DropoutLayer) BatchR(v autofunc.RVector, in autofunc.RResult, n int) autofunc.RResult {
67 | return d.ApplyR(v, in)
68 | }
69 |
70 | func (d *DropoutLayer) Serialize() ([]byte, error) {
71 | return json.Marshal(d)
72 | }
73 |
74 | func (d *DropoutLayer) SerializerType() string {
75 | return serializerTypeDropoutLayer
76 | }
77 |
78 | func (d *DropoutLayer) dropoutMask(inLen int) *autofunc.Variable {
79 | resVec := make(linalg.Vector, inLen)
80 | for i := range resVec {
81 | if rand.Float64() > d.KeepProbability {
82 | resVec[i] = 0
83 | } else {
84 | resVec[i] = 1
85 | }
86 | }
87 | return &autofunc.Variable{resVec}
88 | }
89 |
--------------------------------------------------------------------------------
/rbm/gradient.go:
--------------------------------------------------------------------------------
1 | package rbm
2 |
3 | import (
4 | "math/rand"
5 |
6 | "github.com/unixpickle/num-analysis/linalg"
7 | )
8 |
9 | // RBMGradient is structured like an RBM itself,
10 | // but its values represent the partials of some
11 | // function with respect to an RBM's values.
12 | type RBMGradient RBM
13 |
14 | // LogLikelihoodGradient uses contrastive divergence
15 | // to approximate the gradient of the log likelihood
16 | // of the RBM for the given visible inputs.
17 | //
18 | // The markovSteps parameter specifies how many steps
19 | // of Gibbs sampling this should perform for
20 | // contrastive divergence.
21 | func (r *RBM) LogLikelihoodGradient(ra *rand.Rand, inputs [][]bool, gibbsSteps int) *RBMGradient {
22 | grad := RBMGradient(*NewRBM(len(r.VisibleBiases), len(r.HiddenBiases)))
23 |
24 | visibleVec := make(linalg.Vector, len(r.VisibleBiases))
25 |
26 | for _, input := range inputs {
27 | for i, x := range input {
28 | if x {
29 | visibleVec[i] = 1
30 | } else {
31 | visibleVec[i] = 0
32 | }
33 | }
34 | grad.VisibleBiases.Add(visibleVec)
35 | expHidden := r.ExpectedHidden(input)
36 | grad.HiddenBiases.Add(expHidden)
37 | for hiddenIdx := 0; hiddenIdx < grad.Weights.Rows; hiddenIdx++ {
38 | for visibleIdx := 0; visibleIdx < grad.Weights.Cols; visibleIdx++ {
39 | val := grad.Weights.Get(hiddenIdx, visibleIdx)
40 | val += expHidden[hiddenIdx] * visibleVec[visibleIdx]
41 | grad.Weights.Set(hiddenIdx, visibleIdx, val)
42 | }
43 | }
44 | }
45 |
46 | contrastiveDivergence(r, ra, &grad, len(inputs), gibbsSteps)
47 |
48 | return &grad
49 | }
50 |
51 | func contrastiveDivergence(r *RBM, ra *rand.Rand, grad *RBMGradient, sampleCount int, steps int) {
52 | visibleState := make([]bool, len(r.VisibleBiases))
53 | hiddenState := make([]bool, len(r.HiddenBiases))
54 | for i := 0; i < steps; i++ {
55 | r.SampleHidden(ra, hiddenState, visibleState)
56 | r.SampleVisible(ra, visibleState, hiddenState)
57 | }
58 |
59 | scaler := float64(sampleCount)
60 | visibleVec := make(linalg.Vector, len(visibleState))
61 | hiddenVec := make(linalg.Vector, len(hiddenState))
62 | for i, v := range visibleState {
63 | if v {
64 | visibleVec[i] = 1
65 | }
66 | }
67 | for i, h := range hiddenState {
68 | if h {
69 | hiddenVec[i] = 1
70 | }
71 | }
72 |
73 | grad.HiddenBiases.Add(hiddenVec.Copy().Scale(-scaler))
74 | grad.VisibleBiases.Add(visibleVec.Copy().Scale(-scaler))
75 |
76 | for hiddenIdx := 0; hiddenIdx < grad.Weights.Rows; hiddenIdx++ {
77 | for visibleIdx := 0; visibleIdx < grad.Weights.Cols; visibleIdx++ {
78 | val := grad.Weights.Get(hiddenIdx, visibleIdx)
79 | val -= scaler * hiddenVec[hiddenIdx] * visibleVec[visibleIdx]
80 | grad.Weights.Set(hiddenIdx, visibleIdx, val)
81 | }
82 | }
83 | }
84 |
--------------------------------------------------------------------------------
/rnn/nprnn.go:
--------------------------------------------------------------------------------
1 | package rnn
2 |
3 | import (
4 | "math"
5 | "math/rand"
6 |
7 | "github.com/unixpickle/autofunc"
8 | "github.com/unixpickle/num-analysis/linalg"
9 | "github.com/unixpickle/serializer"
10 | "github.com/unixpickle/sgd"
11 | "github.com/unixpickle/weakai/neuralnet"
12 | )
13 |
14 | const npPowerIterations = 1000
15 |
16 | // NPRNN is an RNN that implements the architecture
17 | // described in http://arxiv.org/pdf/1511.03771v3.pdf.
18 | // An NPRNN outputs its current state, so that Blocks
19 | // may be stacked on top of it to use this state.
20 | type NPRNN interface {
21 | Block
22 | serializer.Serializer
23 | sgd.Learner
24 | }
25 |
26 | // NewNPRNN creates an np-RNN with the given input
27 | // and hidden sizes.
28 | func NewNPRNN(inputSize, hiddenSize int) NPRNN {
29 | matrix := &neuralnet.DenseLayer{
30 | InputCount: inputSize + hiddenSize,
31 | OutputCount: hiddenSize,
32 | Biases: &autofunc.LinAdd{
33 | Var: &autofunc.Variable{
34 | Vector: make(linalg.Vector, hiddenSize),
35 | },
36 | },
37 | Weights: &autofunc.LinTran{
38 | Rows: hiddenSize,
39 | Cols: inputSize + hiddenSize,
40 | Data: &autofunc.Variable{
41 | Vector: make(linalg.Vector, hiddenSize*(inputSize+hiddenSize)),
42 | },
43 | },
44 | }
45 | weights := matrix.Weights.Data.Vector
46 |
47 | normalMat := linalg.NewMatrix(hiddenSize, hiddenSize)
48 | for i := range normalMat.Data {
49 | normalMat.Data[i] = rand.NormFloat64()
50 | }
51 | normalMat = normalMat.Mul(normalMat.Transpose())
52 | normalMat.Scale(1 / float64(hiddenSize))
53 | for i := 0; i < hiddenSize; i++ {
54 | normalMat.Set(i, i, 1+normalMat.Get(i, i))
55 | }
56 | normalMat.Scale(1 / maxEigenvalue(normalMat))
57 |
58 | for i := 0; i < hiddenSize; i++ {
59 | for j := 0; j < hiddenSize; j++ {
60 | x := normalMat.Get(i, j)
61 | weights[j+inputSize+i*(hiddenSize+inputSize)] = x
62 | }
63 | }
64 |
65 | weightScale := 1 / math.Sqrt(float64(inputSize))
66 | for j := 0; j < hiddenSize; j++ {
67 | for i := 0; i < inputSize; i++ {
68 | weights[i+j*(inputSize+hiddenSize)] = (rand.Float64()*2 - 1) * weightScale
69 | }
70 | }
71 |
72 | network := neuralnet.Network{
73 | matrix,
74 | &neuralnet.ReLU{},
75 | }
76 |
77 | return &StateOutBlock{
78 | Block: NewNetworkBlock(network, hiddenSize),
79 | }
80 | }
81 |
82 | func maxEigenvalue(m *linalg.Matrix) float64 {
83 | inVec := make(linalg.Vector, m.Rows)
84 | for i := range inVec {
85 | inVec[i] = rand.NormFloat64()
86 | }
87 | inMat := linalg.NewMatrixColumn(inVec)
88 | for i := 0; i < npPowerIterations; i++ {
89 | inMat = m.MulFast(inMat)
90 | vec := linalg.Vector(inMat.Data)
91 | vec.Scale(1 / vec.Mag())
92 | }
93 | outVec := linalg.Vector(m.MulFast(inMat).Data)
94 | return outVec.Mag()
95 | }
96 |
--------------------------------------------------------------------------------
/rnn/runner.go:
--------------------------------------------------------------------------------
1 | package rnn
2 |
3 | import (
4 | "github.com/unixpickle/autofunc"
5 | "github.com/unixpickle/num-analysis/linalg"
6 | )
7 |
8 | // Runner makes it easy to evaluate an RNN across time
9 | // without having to manually store states and build
10 | // BlockInputs.
11 | type Runner struct {
12 | Block Block
13 | currentState State
14 | }
15 |
16 | // Reset resets the current state, effectively going
17 | // back to time 0.
18 | func (r *Runner) Reset() {
19 | r.currentState = nil
20 | }
21 |
22 | // StepTime evaluates the block with the current state
23 | // and input, updating the internal state and returning
24 | // the output.
25 | func (r *Runner) StepTime(input linalg.Vector) linalg.Vector {
26 | if r.currentState == nil {
27 | r.currentState = r.Block.StartState()
28 | }
29 | inVar := &autofunc.Variable{Vector: input}
30 | out := r.Block.ApplyBlock([]State{r.currentState}, []autofunc.Result{inVar})
31 | r.currentState = out.States()[0]
32 | return out.Outputs()[0]
33 | }
34 |
35 | // RunAll evaluates a batch of sequences on the block and
36 | // returns the new output sequences.
37 | // All of the sequences are evaluated in a single batch.
38 | // This does not affect or use the state used by Reset and
39 | // StepTime.
40 | func (r *Runner) RunAll(inputs [][]linalg.Vector) [][]linalg.Vector {
41 | initState := r.Block.StartState()
42 | initStates := make([]State, len(inputs))
43 | for i := range initStates {
44 | initStates[i] = initState
45 | }
46 | return r.recursiveRunAll(inputs, initStates)
47 | }
48 |
49 | func (r *Runner) recursiveRunAll(seqs [][]linalg.Vector, states []State) [][]linalg.Vector {
50 | var inRes []autofunc.Result
51 | var inStates []State
52 | for i, seq := range seqs {
53 | if len(seq) > 0 {
54 | inVar := &autofunc.Variable{Vector: seq[0]}
55 | inRes = append(inRes, inVar)
56 | inStates = append(inStates, states[i])
57 | }
58 | }
59 | if len(inRes) == 0 {
60 | return make([][]linalg.Vector, len(seqs))
61 | }
62 | result := r.Block.ApplyBlock(inStates, inRes)
63 |
64 | var newStates []State
65 | var truncSecs [][]linalg.Vector
66 | usedIdx := 0
67 | for _, seq := range seqs {
68 | if len(seq) > 0 {
69 | newStates = append(newStates, result.States()[usedIdx])
70 | truncSecs = append(truncSecs, seq[1:])
71 | usedIdx++
72 | }
73 | }
74 | nextOutput := r.recursiveRunAll(truncSecs, newStates)
75 |
76 | var resVecs [][]linalg.Vector
77 | nextIdx := 0
78 | for _, seq := range seqs {
79 | if len(seq) > 0 {
80 | nextOut := nextOutput[nextIdx]
81 | thisOut := append([]linalg.Vector{result.Outputs()[nextIdx]}, nextOut...)
82 | resVecs = append(resVecs, thisOut)
83 | nextIdx++
84 | } else {
85 | resVecs = append(resVecs, nil)
86 | }
87 | }
88 |
89 | return resVecs
90 | }
91 |
--------------------------------------------------------------------------------
/demos/objectrecog/scripts/controls.js:
--------------------------------------------------------------------------------
1 | (function() {
2 |
3 | function Controls() {
4 | window.EventEmitter.call(this);
5 | this._trainButton = document.getElementById('train-button');
6 | this._recognizeButton = document.getElementById('recognize-button');
7 | this._snapButton = document.getElementById('snap-button');
8 |
9 | this._canRecognize = false;
10 | this._snapped = false;
11 | this._training = false;
12 |
13 | this._setEnabledStates([false, false, false]);
14 | this._registerEvents();
15 | }
16 |
17 | Controls.prototype = Object.create(window.EventEmitter.prototype);
18 | Controls.prototype.constructor = Controls;
19 |
20 | Controls.prototype.enable = function() {
21 | this._updateState();
22 | };
23 |
24 | Controls.prototype.setCanRecognize = function(flag) {
25 | this._canRecognize = flag;
26 | this._updateState();
27 | };
28 |
29 | Controls.prototype.setSnapped = function(flag) {
30 | this._snapped = flag;
31 | this._updateState();
32 | };
33 |
34 | Controls.prototype.setTraining = function(flag) {
35 | this._training = flag;
36 | this._updateState();
37 | };
38 |
39 | Controls.prototype._updateState = function() {
40 | if (this._snapped) {
41 | this._snapButton.innerText = 'Back to Camera';
42 | if (this._training) {
43 | this._setEnabledStates([true, false, false]);
44 | this._trainButton.innerText = 'Cancel Training';
45 | } else {
46 | this._setEnabledStates([true, this._canRecognize, true]);
47 | this._trainButton.innerText = 'Train';
48 | }
49 | } else {
50 | this._setEnabledStates([false, false, true]);
51 | this._snapButton.innerText = 'Freeze Picture';
52 | }
53 | };
54 |
55 | Controls.prototype._setEnabledStates = function(flags) {
56 | var buttons = [this._trainButton, this._recognizeButton, this._snapButton];
57 | for (var i = 0; i < 3; ++i) {
58 | buttons[i].disabled = !flags[i];
59 | }
60 | };
61 |
62 | Controls.prototype._registerEvents = function() {
63 | this._trainButton.addEventListener('click', function() {
64 | if (this._training) {
65 | this.setTraining(false);
66 | this.emit('cancelTrain');
67 | } else {
68 | this.setTraining(true);
69 | this.emit('train');
70 | }
71 | }.bind(this));
72 | this._recognizeButton.addEventListener('click', this.emit.bind(this, 'recognize'));
73 | this._snapButton.addEventListener('click', function() {
74 | if (this._snapped) {
75 | this.setSnapped(false);
76 | this.emit('unsnap');
77 | } else {
78 | this.setSnapped(true);
79 | this.emit('snap');
80 | }
81 | }.bind(this));
82 | };
83 |
84 | window.app.Controls = Controls;
85 |
86 | })();
87 |
--------------------------------------------------------------------------------
/rbf/network.go:
--------------------------------------------------------------------------------
1 | package rbf
2 |
3 | import (
4 | "github.com/unixpickle/autofunc"
5 | "github.com/unixpickle/serializer"
6 | "github.com/unixpickle/sgd"
7 | "github.com/unixpickle/weakai/neuralnet"
8 | )
9 |
10 | func init() {
11 | var n Network
12 | serializer.RegisterTypedDeserializer(n.SerializerType(), DeserializeNetwork)
13 | }
14 |
15 | // A Network is an RBF network, with a radial basis layer
16 | // represented by a Dist-Scale-Exp layer combo, and the
17 | // classification layer as a *neuralnet.DenseLayer.
18 | type Network struct {
19 | DistLayer *DistLayer
20 | ScaleLayer *ScaleLayer
21 | ExpLayer *ExpLayer
22 | OutLayer *neuralnet.DenseLayer
23 | }
24 |
25 | // DeserializeNetwork deserializes a Network.
26 | func DeserializeNetwork(d []byte) (*Network, error) {
27 | var n Network
28 | err := serializer.DeserializeAny(d, &n.DistLayer, &n.ScaleLayer, &n.ExpLayer, &n.OutLayer)
29 | if err != nil {
30 | return nil, err
31 | }
32 | return &n, nil
33 | }
34 |
35 | // Apply applies the network to an input.
36 | func (n *Network) Apply(in autofunc.Result) autofunc.Result {
37 | comp := autofunc.ComposedFunc{n.DistLayer, n.ScaleLayer, n.ExpLayer, n.OutLayer}
38 | return comp.Apply(in)
39 | }
40 |
41 | // ApplyR applies the network to an input.
42 | func (n *Network) ApplyR(rv autofunc.RVector, in autofunc.RResult) autofunc.RResult {
43 | comp := autofunc.ComposedRFunc{n.DistLayer, n.ScaleLayer, n.ExpLayer, n.OutLayer}
44 | return comp.ApplyR(rv, in)
45 | }
46 |
47 | // Batch applies the network in batch.
48 | func (n *Network) Batch(in autofunc.Result, m int) autofunc.Result {
49 | comp := autofunc.ComposedBatcher{
50 | n.DistLayer,
51 | &autofunc.FuncBatcher{
52 | F: autofunc.ComposedFunc{n.ScaleLayer, n.ExpLayer},
53 | },
54 | n.OutLayer,
55 | }
56 | return comp.Batch(in, m)
57 | }
58 |
59 | // BatchR applies the network in batch.
60 | func (n *Network) BatchR(rv autofunc.RVector, in autofunc.RResult, m int) autofunc.RResult {
61 | comp := autofunc.ComposedRBatcher{
62 | n.DistLayer,
63 | &autofunc.RFuncBatcher{
64 | F: autofunc.ComposedRFunc{n.ScaleLayer, n.ExpLayer},
65 | },
66 | n.OutLayer,
67 | }
68 | return comp.BatchR(rv, in, m)
69 | }
70 |
71 | // Parameters returns all of the network's parameters.
72 | func (n *Network) Parameters() []*autofunc.Variable {
73 | learners := []sgd.Learner{n.DistLayer, n.ScaleLayer, n.OutLayer}
74 | var res []*autofunc.Variable
75 | for _, l := range learners {
76 | res = append(res, l.Parameters()...)
77 | }
78 | return res
79 | }
80 |
81 | // SerializerType returns the unique ID used to serialize
82 | // a Network with the serializer package.
83 | func (n *Network) SerializerType() string {
84 | return "github.com/unixpickle/weakai/rbf.Network"
85 | }
86 |
87 | // Serialize serializes the network.
88 | func (n *Network) Serialize() ([]byte, error) {
89 | return serializer.SerializeAny(n.DistLayer, n.ScaleLayer, n.ExpLayer, n.OutLayer)
90 | }
91 |
--------------------------------------------------------------------------------
/demos/rnn/basic_demo/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "math/rand"
6 | "time"
7 |
8 | "github.com/unixpickle/num-analysis/linalg"
9 | "github.com/unixpickle/sgd"
10 | "github.com/unixpickle/weakai/neuralnet"
11 | "github.com/unixpickle/weakai/rnn"
12 | "github.com/unixpickle/weakai/rnn/seqtoseq"
13 | )
14 |
15 | const (
16 | StepSize = 0.05
17 | Epochs = 60
18 | BatchSize = 10
19 |
20 | TrainingCount = 100
21 | TestingCount = 100
22 | MaxSeqLen = 30
23 | MinSeqLen = 10
24 |
25 | HiddenSize = 20
26 | )
27 |
28 | func main() {
29 | rand.Seed(time.Now().UnixNano())
30 |
31 | sampleSet := sgd.SliceSampleSet{}
32 | for i := 0; i < TrainingCount; i++ {
33 | inSeq, outSeq := genEvenOddSeq(rand.Intn(MaxSeqLen-MinSeqLen) + MinSeqLen)
34 | sampleSet = append(sampleSet, seqtoseq.Sample{
35 | Inputs: inSeq,
36 | Outputs: outSeq,
37 | })
38 | }
39 |
40 | outNet := neuralnet.Network{
41 | &neuralnet.DenseLayer{
42 | InputCount: HiddenSize,
43 | OutputCount: 2,
44 | },
45 | }
46 | outNet.Randomize()
47 | outBlock := rnn.NewNetworkBlock(outNet, 0)
48 | lstm := rnn.NewLSTM(2, HiddenSize)
49 | net := rnn.StackedBlock{lstm, outBlock}
50 |
51 | gradienter := &sgd.RMSProp{
52 | Gradienter: &seqtoseq.Gradienter{
53 | SeqFunc: &rnn.BlockSeqFunc{B: net},
54 | Learner: net,
55 | CostFunc: neuralnet.SigmoidCECost{},
56 | MaxLanes: 1,
57 | },
58 | }
59 |
60 | sgd.SGD(gradienter, sampleSet, StepSize, Epochs, BatchSize)
61 |
62 | outNet = append(outNet, neuralnet.Sigmoid{})
63 |
64 | var scoreSum float64
65 | var scoreTotal float64
66 | for i := 0; i < TestingCount; i++ {
67 | size := rand.Intn(MaxSeqLen-MinSeqLen) + MinSeqLen
68 | ins, outs := genEvenOddSeq(size)
69 | score := runTestSample(ins, outs, net)
70 | scoreSum += score
71 | scoreTotal += 1
72 | }
73 |
74 | fmt.Println("Testing success rate:", scoreSum/scoreTotal)
75 | }
76 |
77 | func genEvenOddSeq(size int) (ins, outs []linalg.Vector) {
78 | var odds [2]bool
79 | for i := 0; i < size; i++ {
80 | sampleIdx := rand.Intn(2)
81 | odds[sampleIdx] = !odds[sampleIdx]
82 | if sampleIdx == 0 {
83 | ins = append(ins, linalg.Vector{1, 0})
84 | } else {
85 | ins = append(ins, linalg.Vector{0, 1})
86 | }
87 | out := linalg.Vector{0, 0}
88 | for j, o := range odds {
89 | if o {
90 | out[j] = 1
91 | }
92 | }
93 | outs = append(outs, out)
94 | }
95 | return
96 | }
97 |
98 | func runTestSample(ins, outs []linalg.Vector, b rnn.Block) float64 {
99 | var correct int
100 | var total int
101 | r := &rnn.Runner{Block: b}
102 | for i, in := range ins {
103 | out := r.StepTime(in)
104 | if out[0] < 0.5 == (outs[i][0] < 0.5) {
105 | correct++
106 | }
107 | if out[1] < 0.5 == (outs[i][1] < 0.5) {
108 | correct++
109 | }
110 | total += 2
111 | }
112 |
113 | return float64(correct) / float64(total)
114 | }
115 |
--------------------------------------------------------------------------------
/demos/objectrecog/scripts/main.js:
--------------------------------------------------------------------------------
1 | (function() {
2 |
3 | var camera = null;
4 | var controls = null;
5 | var canvas = null;
6 | var cropper = null;
7 | var lastFrame = null;
8 |
9 | function App() {
10 | this._camera = new window.app.Camera();
11 | this._controls = new window.app.Controls();
12 | this._canvas = document.getElementById('canvas');
13 |
14 | this._cropper = null;
15 | this._lastFrame = null;
16 | this._trainedImage = null;
17 |
18 | this._registerCameraEvents();
19 | this._registerControlEvents();
20 |
21 | this._camera.start();
22 | }
23 |
24 | App.prototype.beginTraining = function() {
25 | this._cropper = new window.app.Cropper(this._canvas, this._lastFrame);
26 |
27 | this._cropper.on('cancel', function() {
28 | this._cropper = null;
29 | this._controls.setTraining(false);
30 | this._drawLastFrame();
31 | }.bind(this));
32 |
33 | this._cropper.on('crop', function(image) {
34 | this._cropper = null;
35 | this._trainedImage = image;
36 | this._controls.setTraining(false);
37 | this._controls.setCanRecognize(true);
38 | this._drawLastFrame();
39 | }.bind(this));
40 | };
41 |
42 | App.prototype.recognize = function() {
43 | var match = window.app.findSubImage(this._canvas, this._trainedImage);
44 |
45 | this._drawLastFrame();
46 | var ctx = this._canvas.getContext('2d');
47 | ctx.strokeStyle = 'red';
48 | ctx.lineWidth = 2;
49 |
50 | ctx.beginPath();
51 | ctx.rect(match.x, match.y, this._trainedImage.width, this._trainedImage.height);
52 | ctx.stroke();
53 | };
54 |
55 | App.prototype._registerCameraEvents = function() {
56 | this._camera.on('error', function() {
57 | document.body.innerHTML = 'Camera API not supported
';
58 | });
59 |
60 | this._camera.on('start', function(dimensions) {
61 | this._canvas.width = dimensions.width;
62 | this._canvas.height = dimensions.height;
63 | this._canvas.style.left = 'calc(50% - ' + Math.round(dimensions.width/2) + 'px)';
64 | }.bind(this));
65 |
66 | this._camera.on('frame', function(frame) {
67 | this._controls.enable();
68 | this._lastFrame = frame;
69 | this._drawLastFrame();
70 | }.bind(this));
71 | };
72 |
73 | App.prototype._registerControlEvents = function() {
74 | this._controls.on('snap', this._camera.pause.bind(this._camera));
75 | this._controls.on('unsnap', this._camera.resume.bind(this._camera));
76 | this._controls.on('train', this.beginTraining.bind(this));
77 | this._controls.on('cancelTrain', function() {
78 | this._cropper.cancel();
79 | this._cropper = null;
80 | this._drawLastFrame();
81 | }.bind(this));
82 | this._controls.on('recognize', this.recognize.bind(this));
83 | };
84 |
85 | App.prototype._drawLastFrame = function() {
86 | this._canvas.getContext('2d').drawImage(this._lastFrame, 0, 0);
87 | };
88 |
89 | window.addEventListener('load', function() {
90 | new App();
91 | });
92 |
93 | })();
94 |
--------------------------------------------------------------------------------
/neuralnet/softmax_layer.go:
--------------------------------------------------------------------------------
1 | package neuralnet
2 |
3 | import (
4 | "encoding/json"
5 |
6 | "github.com/unixpickle/autofunc"
7 | "github.com/unixpickle/num-analysis/linalg"
8 | )
9 |
10 | type SoftmaxLayer autofunc.Softmax
11 |
12 | func DeserializeSoftmaxLayer(d []byte) (*SoftmaxLayer, error) {
13 | var res SoftmaxLayer
14 | if err := json.Unmarshal(d, &res); err != nil {
15 | return nil, err
16 | }
17 | return &res, nil
18 | }
19 |
20 | func (s *SoftmaxLayer) Apply(in autofunc.Result) autofunc.Result {
21 | soft := (*autofunc.Softmax)(s)
22 | return soft.Apply(in)
23 | }
24 |
25 | func (s *SoftmaxLayer) ApplyR(v autofunc.RVector, in autofunc.RResult) autofunc.RResult {
26 | soft := (*autofunc.Softmax)(s)
27 | return soft.ApplyR(v, in)
28 | }
29 |
30 | func (s *SoftmaxLayer) Serialize() ([]byte, error) {
31 | return json.Marshal(s)
32 | }
33 |
34 | func (s *SoftmaxLayer) SerializerType() string {
35 | return serializerTypeSoftmaxLayer
36 | }
37 |
38 | // LogSoftmaxLayer is a layer which returns the log
39 | // of the softmax function (with temperature = 1).
40 | type LogSoftmaxLayer struct{}
41 |
42 | func DeserializeLogSoftmaxLayer(d []byte) (*LogSoftmaxLayer, error) {
43 | return &LogSoftmaxLayer{}, nil
44 | }
45 |
46 | func (s *LogSoftmaxLayer) Apply(in autofunc.Result) autofunc.Result {
47 | return autofunc.Pool(in, func(in autofunc.Result) autofunc.Result {
48 | // Compute the log of the sum of the exponents by
49 | // factoring out the largest exponent so that all
50 | // the exponentials fit nicely inside floats.
51 | maxIdx := maxVecIdx(in.Output())
52 | maxValue := autofunc.Slice(in, maxIdx, maxIdx+1)
53 | exponents := autofunc.AddFirst(in, autofunc.Scale(maxValue, -1))
54 | expSum := autofunc.SumAll(autofunc.Exp{}.Apply(exponents))
55 | expLog := autofunc.Log{}.Apply(expSum)
56 | denomLog := autofunc.Add(expLog, maxValue)
57 | return autofunc.AddFirst(in, autofunc.Scale(denomLog, -1))
58 | })
59 | }
60 |
61 | func (s *LogSoftmaxLayer) ApplyR(v autofunc.RVector, in autofunc.RResult) autofunc.RResult {
62 | return autofunc.PoolR(in, func(in autofunc.RResult) autofunc.RResult {
63 | // See comment in Apply() for details on how this works.
64 | maxIdx := maxVecIdx(in.Output())
65 | maxValue := autofunc.SliceR(in, maxIdx, maxIdx+1)
66 | exponents := autofunc.AddFirstR(in, autofunc.ScaleR(maxValue, -1))
67 | expSum := autofunc.SumAllR(autofunc.Exp{}.ApplyR(v, exponents))
68 | expLog := autofunc.Log{}.ApplyR(v, expSum)
69 | denomLog := autofunc.AddR(expLog, maxValue)
70 | return autofunc.AddFirstR(in, autofunc.ScaleR(denomLog, -1))
71 | })
72 | }
73 |
74 | func (s *LogSoftmaxLayer) Serialize() ([]byte, error) {
75 | return []byte{}, nil
76 | }
77 |
78 | func (s *LogSoftmaxLayer) SerializerType() string {
79 | return serializerTypeLogSoftmaxLayer
80 | }
81 |
82 | func maxVecIdx(v linalg.Vector) int {
83 | var maxVal float64
84 | var maxIdx int
85 | for i, x := range v {
86 | if i == 0 {
87 | maxVal = x
88 | maxIdx = i
89 | } else if x > maxVal {
90 | maxVal = x
91 | maxIdx = i
92 | }
93 | }
94 | return maxIdx
95 | }
96 |
--------------------------------------------------------------------------------
/demos/svm/nonlinear/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "math/rand"
6 | "time"
7 |
8 | "github.com/unixpickle/weakai/svm"
9 | )
10 |
11 | func main() {
12 | rand.Seed(time.Now().UnixNano())
13 |
14 | positives := []svm.Sample{
15 | {V: []float64{0.5, 0}},
16 | {V: []float64{0.4, 0.1}},
17 | {V: []float64{0.3, 0.2}},
18 | {V: []float64{0.5, 0.9}},
19 | {V: []float64{0.6, 1}},
20 | {V: []float64{0.4, 0.85}},
21 | {V: []float64{0.5, 0.5}},
22 | }
23 |
24 | negatives := []svm.Sample{
25 | {V: []float64{0, 0.5}},
26 | {V: []float64{0.1, 0.4}},
27 | {V: []float64{0.9, 0.5}},
28 | {V: []float64{1, 0.6}},
29 | {V: []float64{0, 0.46}},
30 | }
31 |
32 | problem := &svm.Problem{
33 | Positives: positives,
34 | Negatives: negatives,
35 | Kernel: svm.PolynomialKernel(1, 2),
36 | }
37 |
38 | printProblem(problem)
39 |
40 | printSolution("random linear", svm.RandomlySolveLinear(problem, 100000, 5), problem)
41 |
42 | subgradientSolver := &svm.SubgradientSolver{
43 | Tradeoff: 0.001,
44 | StepSize: 0.1,
45 | Steps: 100000,
46 | }
47 | printSolution("subgradient descent", subgradientSolver.Solve(problem), problem)
48 |
49 | gradientSolver := &svm.GradientDescentSolver{
50 | Tradeoff: 0.001,
51 | Timeout: time.Minute,
52 | }
53 | printSolution("gradient descent", gradientSolver.Solve(problem), problem)
54 | }
55 |
56 | func printProblem(p *svm.Problem) {
57 | grid := make([]rune, 10*20)
58 | x := func(samples []svm.Sample, char rune) {
59 | for _, s := range samples {
60 | x, y := int(s.V[0]*20), int(s.V[1]*10)
61 | if x >= 20 {
62 | x = 19
63 | }
64 | if y >= 10 {
65 | y = 9
66 | }
67 | grid[x+y*20] = char
68 | }
69 | }
70 | x(p.Positives, '+')
71 | x(p.Negatives, '-')
72 |
73 | fmt.Println("Problem:")
74 | fmt.Println("----------------------")
75 | for y := 0; y < 10; y++ {
76 | fmt.Print("|")
77 | for x := 0; x < 20; x++ {
78 | if ch := grid[x+y*20]; ch == 0 {
79 | fmt.Print(" ")
80 | } else {
81 | fmt.Print(string(ch))
82 | }
83 | }
84 | fmt.Println("|")
85 | }
86 | fmt.Println("----------------------")
87 | }
88 |
89 | func printSolution(solverType string, solution svm.Classifier, p *svm.Problem) {
90 | fmt.Println("Got solution from", solverType)
91 |
92 | incorrectCount := 0
93 | for _, pos := range p.Positives {
94 | if !solution.Classify(pos) {
95 | incorrectCount++
96 | }
97 | }
98 | for _, neg := range p.Negatives {
99 | if solution.Classify(neg) {
100 | incorrectCount++
101 | }
102 | }
103 |
104 | fmt.Println("Number of incorrect classifications:", incorrectCount)
105 |
106 | fmt.Println("Visual representation:")
107 | for y := 0.0; y <= 1.0; y += 0.1 {
108 | fmt.Print(" ")
109 | for x := 0.0; x < 1.0; x += 0.05 {
110 | classification := solution.Classify(svm.Sample{V: []float64{x, y}})
111 | if classification {
112 | fmt.Print("+")
113 | } else {
114 | fmt.Print("-")
115 | }
116 | }
117 | fmt.Println("")
118 | }
119 | fmt.Println("")
120 | }
121 |
--------------------------------------------------------------------------------
/neuralnet/batch_rgradienter.go:
--------------------------------------------------------------------------------
1 | package neuralnet
2 |
3 | import (
4 | "github.com/unixpickle/autofunc"
5 | "github.com/unixpickle/num-analysis/linalg"
6 | "github.com/unixpickle/sgd"
7 | )
8 |
9 | // BatchRGradienter is an RGradienter that computes
10 | // the gradients of BatchLearners using SampleSets
11 | // of VectorSamples.
12 | //
13 | // A BatchRGradienter is suitable for training tasks
14 | // with lots of learnable parameters, since said tasks
15 | // can benefit from parallelism.
16 | //
17 | // After you use a BatchRGradienter with a given
18 | // BatchLearner, you should never use the same
19 | // BatchRGradienter for any BatchLearner with
20 | // different parameters.
21 | type BatchRGradienter struct {
22 | Learner BatchLearner
23 | CostFunc CostFunc
24 |
25 | // MaxGoroutines is the maximum number of Goroutines
26 | // the BatchRGradienter will use simultaneously.
27 | // If this is 0, a reasonable default is used.
28 | MaxGoroutines int
29 |
30 | // MaxBatchSize is the maximum number of samples the
31 | // BatchRGradienter will pass to the learner at once.
32 | // If this is 0, a reasonable default is used.
33 | MaxBatchSize int
34 |
35 | helper *GradHelper
36 | }
37 |
38 | func (b *BatchRGradienter) Gradient(s sgd.SampleSet) autofunc.Gradient {
39 | return b.makeHelper().Gradient(s)
40 | }
41 |
42 | func (b *BatchRGradienter) RGradient(v autofunc.RVector, s sgd.SampleSet) (autofunc.Gradient,
43 | autofunc.RGradient) {
44 | return b.makeHelper().RGradient(v, s)
45 | }
46 |
47 | func (b *BatchRGradienter) makeHelper() *GradHelper {
48 | if b.helper != nil {
49 | b.helper.MaxConcurrency = b.MaxGoroutines
50 | b.helper.MaxSubBatch = b.MaxBatchSize
51 | return b.helper
52 | }
53 | b.helper = &GradHelper{
54 | MaxConcurrency: b.MaxGoroutines,
55 | MaxSubBatch: b.MaxBatchSize,
56 | Learner: b.Learner,
57 |
58 | CompGrad: func(g autofunc.Gradient, s sgd.SampleSet) {
59 | b.runBatch(nil, nil, g, s)
60 | },
61 | CompRGrad: b.runBatch,
62 | }
63 | return b.helper
64 | }
65 |
66 | func (b *BatchRGradienter) runBatch(rv autofunc.RVector, rgrad autofunc.RGradient,
67 | grad autofunc.Gradient, s sgd.SampleSet) {
68 | if s.Len() == 0 {
69 | return
70 | }
71 |
72 | sampleCount := s.Len()
73 | firstSample := s.GetSample(0).(VectorSample)
74 | inputSize := len(firstSample.Input)
75 | outputSize := len(firstSample.Output)
76 | inVec := make(linalg.Vector, sampleCount*inputSize)
77 | outVec := make(linalg.Vector, sampleCount*outputSize)
78 |
79 | for i := 0; i < s.Len(); i++ {
80 | sample := s.GetSample(i)
81 | vs := sample.(VectorSample)
82 | copy(inVec[i*inputSize:], vs.Input)
83 | copy(outVec[i*outputSize:], vs.Output)
84 | }
85 |
86 | inVar := &autofunc.Variable{inVec}
87 | if rgrad != nil {
88 | rVar := autofunc.NewRVariable(inVar, rv)
89 | result := b.Learner.BatchR(rv, rVar, sampleCount)
90 | cost := b.CostFunc.CostR(rv, outVec, result)
91 | cost.PropagateRGradient(linalg.Vector{1}, linalg.Vector{0},
92 | rgrad, grad)
93 | } else {
94 | result := b.Learner.Batch(inVar, sampleCount)
95 | cost := b.CostFunc.Cost(outVec, result)
96 | cost.PropagateGradient(linalg.Vector{1}, grad)
97 | }
98 | }
99 |
--------------------------------------------------------------------------------
/demos/neuralnet/image_demos/fetch/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "bytes"
5 | "image"
6 | "image/color"
7 | _ "image/jpeg"
8 | "image/png"
9 | "io/ioutil"
10 | "log"
11 | "net/http"
12 | "path/filepath"
13 | "strconv"
14 | "strings"
15 | "sync"
16 |
17 | "github.com/nfnt/resize"
18 | )
19 |
20 | const (
21 | // Download http://www.cs.columbia.edu/CAVE/databases/pubfig/download/dev_urls.txt
22 | // and remove the comments at the top of the file.
23 | FilePath = "dev_urls.txt"
24 |
25 | OutputDir = "./faces"
26 | RoutineCount = 10
27 | OutputSize = 96
28 | )
29 |
30 | type requestInfo struct {
31 | index int
32 | line string
33 | }
34 |
35 | func main() {
36 | dbData, err := ioutil.ReadFile(FilePath)
37 | if err != nil {
38 | panic(err)
39 | }
40 |
41 | reqChan := make(chan requestInfo)
42 | var wg sync.WaitGroup
43 |
44 | for i := 0; i < RoutineCount; i++ {
45 | wg.Add(1)
46 | go func() {
47 | defer wg.Done()
48 | for req := range reqChan {
49 | parts := strings.Split(req.line, "\t")
50 | url := parts[2]
51 | name := strings.Join(parts[:2], " ")
52 | rectParts := strings.Split(parts[3], ",")
53 | var rectNums [4]int
54 | for i, r := range rectParts {
55 | n, _ := strconv.Atoi(r)
56 | rectNums[i] = n
57 | }
58 | face, err := fetchFace(url, rectNums)
59 | if err != nil {
60 | log.Printf("Error for face %d: %s", req.index, err.Error())
61 | } else {
62 | imageData := encodeImage(face)
63 | filePath := filepath.Join(OutputDir, name+".png")
64 | if err := ioutil.WriteFile(filePath, imageData, 0755); err != nil {
65 | log.Printf("Error writing: %s", filePath)
66 | }
67 | }
68 | }
69 | }()
70 | }
71 |
72 | lines := strings.Split(string(dbData), "\n")
73 | for i, line := range lines {
74 | reqChan <- requestInfo{i, line}
75 | }
76 |
77 | close(reqChan)
78 | wg.Wait()
79 | }
80 |
81 | func fetchFace(url string, rect [4]int) (image.Image, error) {
82 | log.Println("fetching", url)
83 | resp, err := http.Get(url)
84 | if err != nil {
85 | return nil, err
86 | }
87 | defer resp.Body.Close()
88 | img, _, err := image.Decode(resp.Body)
89 | if err != nil {
90 | return nil, err
91 | }
92 |
93 | cropped := croppedImage{img, rect}
94 | return resize.Resize(OutputSize, OutputSize, cropped, resize.Bilinear), nil
95 | }
96 |
97 | func encodeImage(img image.Image) []byte {
98 | var buf bytes.Buffer
99 | if err := png.Encode(&buf, img); err != nil {
100 | panic("failed to encode PNG: " + err.Error())
101 | }
102 | return buf.Bytes()
103 | }
104 |
105 | type croppedImage struct {
106 | img image.Image
107 | bounds [4]int
108 | }
109 |
110 | func (c croppedImage) ColorModel() color.Model {
111 | return c.img.ColorModel()
112 | }
113 |
114 | func (c croppedImage) At(x, y int) color.Color {
115 | return c.img.At(x, y)
116 | }
117 |
118 | func (c croppedImage) Bounds() image.Rectangle {
119 | b1 := c.img.Bounds()
120 | b1.Min.X = c.bounds[0]
121 | b1.Min.Y = c.bounds[1]
122 | b1.Max.X = c.bounds[2]
123 | b1.Max.Y = c.bounds[3]
124 | return b1
125 | }
126 |
--------------------------------------------------------------------------------
/demos/objectrecog/scripts/camera.js:
--------------------------------------------------------------------------------
1 | (function() {
2 |
3 | var FRAME_RATE = 500;
4 |
5 | function Camera() {
6 | window.EventEmitter.call(this);
7 | this._video = null;
8 | this._paused = false;
9 | }
10 |
11 | Camera.prototype = Object.create(window.EventEmitter.prototype);
12 | Camera.prototype.constructor = Camera;
13 |
14 | Camera.prototype.start = function() {
15 | getUserMedia(function(err, stream) {
16 | if (err !== null) {
17 | this.emit('error', err);
18 | } else {
19 | this._startWithStream(stream);
20 | }
21 | }.bind(this));
22 | };
23 |
24 | Camera.prototype.pause = function() {
25 | this._paused = true;
26 | };
27 |
28 | Camera.prototype.resume = function() {
29 | this._paused = false;
30 | };
31 |
32 | Camera.prototype.outputDimensions = function() {
33 | var width = this._video.videoWidth;
34 | var height = this._video.videoHeight;
35 | var scaleFactor = 192 / height;
36 | return {
37 | width: Math.round(width * scaleFactor),
38 | height: Math.round(height * scaleFactor)
39 | };
40 | };
41 |
42 | Camera.prototype._startWithStream = function(stream) {
43 | this._video = document.createElement('video');
44 | this._video.src = window.URL.createObjectURL(stream);
45 | this._video.play();
46 |
47 | // On chrome, onloadedmetadata will never be called, so we
48 | // use a timeout to start emitting frames anyway.
49 | var loadInterval;
50 | loadInterval = setInterval(function() {
51 | if (this._video.videoWidth) {
52 | clearInterval(loadInterval);
53 | this._video.onloadedmetadata = null;
54 | this._beginEmittingFrames();
55 | }
56 | }.bind(this), 500);
57 |
58 | this._video.onloadedmetadata = function() {
59 | clearInterval(loadInterval);
60 | this._beginEmittingFrames();
61 | }.bind(this);
62 | };
63 |
64 | Camera.prototype._beginEmittingFrames = function() {
65 | this.emit('start', this.outputDimensions());
66 | setInterval(function() {
67 | if (!this._paused) {
68 | this.emit('frame', this._generateFrameCanvas());
69 | }
70 | }.bind(this), FRAME_RATE);
71 | };
72 |
73 | Camera.prototype._generateFrameCanvas = function() {
74 | var dims = this.outputDimensions();
75 | var canvas = document.createElement('canvas');
76 | canvas.width = dims.width;
77 | canvas.height = dims.height;
78 | var ctx = canvas.getContext('2d');
79 | ctx.drawImage(this._video, 0, 0, dims.width, dims.height);
80 | return canvas;
81 | };
82 |
83 | window.app.Camera = Camera;
84 |
85 | function getUserMedia(cb) {
86 | var gum = (navigator.getUserMedia || navigator.webkitGetUserMedia ||
87 | navigator.mozGetUserMedia || navigator.msGetUserMedia);
88 | if (!gum) {
89 | setTimeout(function() {
90 | cb('getUserMedia() is not available.', null);
91 | }, 10);
92 | return;
93 | }
94 | gum.call(navigator, {audio: false, video: true},
95 | function(stream) {
96 | cb(null, stream);
97 | },
98 | function(err) {
99 | cb(err, null);
100 | }
101 | );
102 | }
103 |
104 | })();
105 |
--------------------------------------------------------------------------------
/rnn/block.go:
--------------------------------------------------------------------------------
1 | package rnn
2 |
3 | import (
4 | "github.com/unixpickle/autofunc"
5 | "github.com/unixpickle/num-analysis/linalg"
6 | )
7 |
8 | // A Block is a differentiable unit in a recurrent model
9 | // that transforms input/state pairs into output/state
10 | // pairs.
11 | type Block interface {
12 | // StartState returns the initial state for the Block.
13 | StartState() State
14 |
15 | // StartRState is like StartState but for an RState.
16 | StartRState(rv autofunc.RVector) RState
17 |
18 | // PropagateStart performs back-propagation through the
19 | // start state.
20 | // It takes the list of start states, their upstream
21 | // gradients, and the output gradient.
22 | PropagateStart(s []State, u []StateGrad, g autofunc.Gradient)
23 |
24 | // PropagateStartR is like PropagateStart, but for an
25 | // RStateGrad.
26 | // The g argument may be nil.
27 | PropagateStartR(s []RState, u []RStateGrad, rg autofunc.RGradient, g autofunc.Gradient)
28 |
29 | // ApplyBlock applies the block to a batch of inputs.
30 | // The result is valid so long as neither the inputs
31 | // nor the Block are changed.
32 | //
33 | // The input slice must be non-empty.
34 | // It is not possible to apply a block to zero inputs.
35 | ApplyBlock(s []State, in []autofunc.Result) BlockResult
36 |
37 | // ApplyBlockR is like ApplyBlock, but with support for
38 | // the R operator.
39 | //
40 | // It is necessary to provide an RVector so that the
41 | // block knows how much each of its hidden parameters
42 | // changes with respect to R.
43 | ApplyBlockR(v autofunc.RVector, s []RState, in []autofunc.RResult) BlockRResult
44 | }
45 |
46 | // A BlockResult represents the output of a Block.
47 | type BlockResult interface {
48 | // Outputs returns the vector outputs of the block.
49 | Outputs() []linalg.Vector
50 |
51 | // States returns the new states of the block.
52 | States() []State
53 |
54 | // PropagateGradient performs back-propagation.
55 | // It returns the gradient with respect to the input
56 | // states.
57 | //
58 | // A nil argument stands for a 0 gradient.
59 | // Upstream, s, and/or some entries in s may be nil.
60 | //
61 | // This should not modify the upstream information.
62 | PropagateGradient(upstream []linalg.Vector, s []StateGrad, g autofunc.Gradient) []StateGrad
63 | }
64 |
65 | // A BlockRResult is like a BlockResult, but for RStates.
66 | type BlockRResult interface {
67 | // Outputs returns the vector outputs of the block.
68 | Outputs() []linalg.Vector
69 |
70 | // ROutputs returns the derivative of Outputs() with
71 | // respect to R.
72 | ROutputs() []linalg.Vector
73 |
74 | // RStates returns the new states of the block.
75 | RStates() []RState
76 |
77 | // PropagateRGradient performs back-propagation.
78 | // It returns the gradient with respect to the input
79 | // states.
80 | //
81 | // A nil upstream or s stands for a 0 gradient.
82 | // Upstream, s, and/or some entries in s may be nil.
83 | // The upstream argument may be nil if and only if the
84 | // upstreamR argument is also nil.
85 | // The g argument may also be nil if the gradients are
86 | // not desired.
87 | //
88 | // This should not modify the upstream information.
89 | PropagateRGradient(upstream, upstreamR []linalg.Vector, s []RStateGrad,
90 | rg autofunc.RGradient, g autofunc.Gradient) []RStateGrad
91 | }
92 |
--------------------------------------------------------------------------------
/demos/nearestneighbors/search/index.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "encoding/json"
5 | "io/ioutil"
6 | "math"
7 | "sort"
8 | "strings"
9 | )
10 |
11 | type rawIndexEntry struct {
12 | URL string `json:"url"`
13 | Keywords map[string]int `json:"keywords"`
14 | }
15 |
16 | type normalizedIndexEntry struct {
17 | URL string
18 | Vector []float64
19 | }
20 |
21 | func (n normalizedIndexEntry) relevanceForQuery(vector []float64) float64 {
22 | var dotProduct float64
23 | for i, x := range vector {
24 | dotProduct += x * n.Vector[i]
25 | }
26 | return dotProduct
27 | }
28 |
29 | type Index struct {
30 | entries []normalizedIndexEntry
31 | keywords []string
32 | }
33 |
34 | func ReadIndex(path string) (idx *Index, err error) {
35 | contents, err := ioutil.ReadFile(path)
36 | if err != nil {
37 | return
38 | }
39 |
40 | var rawEntries []rawIndexEntry
41 | if err := json.Unmarshal(contents, &rawEntries); err != nil {
42 | return nil, err
43 | }
44 |
45 | keywords := keywordsFromEntries(rawEntries)
46 | idx = &Index{entries: []normalizedIndexEntry{}, keywords: keywords}
47 |
48 | for _, raw := range rawEntries {
49 | idx.addRawEntry(raw)
50 | }
51 |
52 | return idx, nil
53 | }
54 |
55 | func (i *Index) SearchResults(query string) []string {
56 | fields := strings.Fields(strings.ToLower(query))
57 | wordFrequency := map[string]int{}
58 | for _, f := range fields {
59 | wordFrequency[f] = wordFrequency[f] + 1
60 | }
61 |
62 | vector := i.vectorForKeywordMap(wordFrequency)
63 | if vector == nil {
64 | // None of the words in the user's search were indexed keywords.
65 | return []string{}
66 | }
67 |
68 | entries := make([]normalizedIndexEntry, len(i.entries))
69 | copy(entries, i.entries)
70 | sorter := searchSorter{Search: vector, Entries: entries}
71 | sort.Sort(sorter)
72 |
73 | res := make([]string, 0, len(sorter.Entries))
74 | for _, entry := range sorter.Entries {
75 | if entry.relevanceForQuery(vector) > 0 {
76 | res = append(res, entry.URL)
77 | }
78 | }
79 | return res
80 | }
81 |
82 | func (i *Index) vectorForKeywordMap(m map[string]int) []float64 {
83 | vector := make([]float64, len(i.keywords))
84 | var magnitude float64
85 | for keyword, count := range m {
86 | idx := sort.SearchStrings(i.keywords, keyword)
87 | if idx == len(i.keywords) || i.keywords[idx] != keyword {
88 | continue
89 | }
90 | vector[idx] += float64(count)
91 | magnitude += float64(count)
92 | }
93 | magnitude = math.Sqrt(magnitude)
94 | if magnitude == 0 {
95 | return nil
96 | }
97 | for j, val := range vector {
98 | vector[j] = val / magnitude
99 | }
100 | return vector
101 | }
102 |
103 | func (i *Index) addRawEntry(raw rawIndexEntry) {
104 | vec := i.vectorForKeywordMap(raw.Keywords)
105 | if vec == nil {
106 | return
107 | }
108 | i.entries = append(i.entries, normalizedIndexEntry{URL: raw.URL, Vector: vec})
109 | }
110 |
111 | func keywordsFromEntries(entries []rawIndexEntry) []string {
112 | words := map[string]bool{}
113 | for _, entry := range entries {
114 | for word := range entry.Keywords {
115 | words[word] = true
116 | }
117 | }
118 |
119 | res := make([]string, 0, len(words))
120 | for word := range words {
121 | res = append(res, word)
122 | }
123 |
124 | sort.Strings(res)
125 |
126 | return res
127 | }
128 |
--------------------------------------------------------------------------------
/demos/neuralnet/image_demos/autoencoder/autoencode.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "errors"
5 | "image"
6 | "log"
7 | "math"
8 |
9 | "github.com/unixpickle/hessfree"
10 | "github.com/unixpickle/num-analysis/linalg"
11 | "github.com/unixpickle/tensor"
12 | "github.com/unixpickle/weakai/neuralnet"
13 | )
14 |
15 | const (
16 | HiddenSize1 = 300
17 | HiddenSize2 = 100
18 | MaxSubBatch = 20
19 | )
20 |
21 | func Autoencode(images <-chan image.Image) (neuralnet.Network, error) {
22 | firstImage := <-images
23 | if firstImage == nil {
24 | return nil, errors.New("no readable images")
25 | }
26 |
27 | width := firstImage.Bounds().Dx()
28 | height := firstImage.Bounds().Dy()
29 |
30 | log.Print("Reading images...")
31 |
32 | tensors := []*tensor.Float64{ImageTensor(firstImage)}
33 | for img := range images {
34 | if img.Bounds().Dx() != width || img.Bounds().Dy() != height {
35 | log.Printf("Image size %d,%d does not match %d,%d",
36 | img.Bounds().Dx(), img.Bounds().Dy(),
37 | width, height)
38 | } else {
39 | tensors = append(tensors, ImageTensor(img))
40 | }
41 | }
42 |
43 | log.Print("Training network (ctrl+c to finish)...")
44 |
45 | tensorSlices := make([]linalg.Vector, len(tensors))
46 | for i, tensor := range tensors {
47 | tensorSlices[i] = tensor.Data
48 | }
49 | samples := neuralnet.VectorSampleSet(tensorSlices, tensorSlices)
50 |
51 | average, stddev := statisticalInfo(tensorSlices)
52 |
53 | network := neuralnet.Network{
54 | &neuralnet.RescaleLayer{
55 | Bias: -average,
56 | Scale: 1 / stddev,
57 | },
58 | &neuralnet.DenseLayer{
59 | InputCount: width * height * 3,
60 | OutputCount: HiddenSize1,
61 | },
62 | neuralnet.Sigmoid{},
63 | &neuralnet.DenseLayer{
64 | InputCount: HiddenSize1,
65 | OutputCount: HiddenSize2,
66 | },
67 | neuralnet.Sigmoid{},
68 | &neuralnet.DenseLayer{
69 | InputCount: HiddenSize2,
70 | OutputCount: HiddenSize1,
71 | },
72 | neuralnet.Sigmoid{},
73 | &neuralnet.DenseLayer{
74 | InputCount: HiddenSize1,
75 | OutputCount: width * height * 3,
76 | },
77 | }
78 | network.Randomize()
79 |
80 | ui := hessfree.NewConsoleUI()
81 | learner := &hessfree.DampingLearner{
82 | WrappedLearner: &hessfree.NeuralNetLearner{
83 | Layers: network,
84 | Output: nil,
85 | Cost: neuralnet.SigmoidCECost{},
86 | MaxSubBatch: MaxSubBatch,
87 | MaxConcurrency: 2,
88 | },
89 | DampingCoeff: 2,
90 | UI: ui,
91 | }
92 | trainer := hessfree.Trainer{
93 | Learner: learner,
94 | Samples: samples,
95 | BatchSize: samples.Len(),
96 | UI: ui,
97 | Convergence: hessfree.ConvergenceCriteria{
98 | MinK: 5,
99 | },
100 | }
101 | trainer.Train()
102 |
103 | network = append(network, neuralnet.Sigmoid{})
104 |
105 | return network, nil
106 | }
107 |
108 | func statisticalInfo(samples []linalg.Vector) (mean, stddev float64) {
109 | var count int
110 | for _, v := range samples {
111 | for _, x := range v {
112 | mean += x
113 | count++
114 | }
115 | }
116 |
117 | mean /= float64(count)
118 |
119 | for _, v := range samples {
120 | for _, x := range v {
121 | stddev += math.Pow(x-mean, 2)
122 | count++
123 | }
124 | }
125 | stddev /= float64(count)
126 | stddev = math.Sqrt(stddev)
127 |
128 | return
129 | }
130 |
--------------------------------------------------------------------------------
/neuralnet/serializer.go:
--------------------------------------------------------------------------------
1 | package neuralnet
2 |
3 | import "github.com/unixpickle/serializer"
4 |
5 | const (
6 | serializerTypePrefix = "github.com/unixpickle/weakai/neuralnet."
7 | serializerTypeHyperbolicTangent = serializerTypePrefix + "HyperbolicTangent"
8 | serializerTypeSigmoid = serializerTypePrefix + "Sigmoid"
9 | serializerTypeSin = serializerTypePrefix + "Sin"
10 | serializerTypeBorderLayer = serializerTypePrefix + "BorderLayer"
11 | serializerTypeUnstackLayer = serializerTypePrefix + "UnstackLayer"
12 | serializerTypeConvLayer = serializerTypePrefix + "ConvLayer"
13 | serializerTypeDenseLayer = serializerTypePrefix + "DenseLayer"
14 | serializerTypeMaxPoolingLayer = serializerTypePrefix + "MaxPoolingLayer"
15 | serializerTypeSoftmaxLayer = serializerTypePrefix + "SoftmaxLayer"
16 | serializerTypeLogSoftmaxLayer = serializerTypePrefix + "LogSoftmaxLayer"
17 | serializerTypeNetwork = serializerTypePrefix + "Network"
18 | serializerTypeReLU = serializerTypePrefix + "ReLU"
19 | serializerTypeRescaleLayer = serializerTypePrefix + "RescaleLayer"
20 | serializerTypeDropoutLayer = serializerTypePrefix + "DropoutLayer"
21 | serializerTypeVecRescaleLayer = serializerTypePrefix + "VecRescaleLayer"
22 | serializerTypeGaussNoiseLayer = serializerTypePrefix + "GaussNoiseLayer"
23 | serializerTypeResidualLayer = serializerTypePrefix + "ResidualLayer"
24 | )
25 |
26 | func init() {
27 | serializer.RegisterDeserializer(serializerTypeSigmoid,
28 | func(d []byte) (serializer.Serializer, error) {
29 | return &Sigmoid{}, nil
30 | })
31 | serializer.RegisterDeserializer(serializerTypeReLU,
32 | func(d []byte) (serializer.Serializer, error) {
33 | return &ReLU{}, nil
34 | })
35 | serializer.RegisterDeserializer(serializerTypeHyperbolicTangent,
36 | func(d []byte) (serializer.Serializer, error) {
37 | return &HyperbolicTangent{}, nil
38 | })
39 | serializer.RegisterDeserializer(serializerTypeSin,
40 | func(d []byte) (serializer.Serializer, error) {
41 | return &Sin{}, nil
42 | })
43 | serializer.RegisterTypedDeserializer(serializerTypeConvLayer,
44 | DeserializeConvLayer)
45 | serializer.RegisterTypedDeserializer(serializerTypeDenseLayer,
46 | DeserializeDenseLayer)
47 | serializer.RegisterTypedDeserializer(serializerTypeNetwork,
48 | DeserializeNetwork)
49 | serializer.RegisterTypedDeserializer(serializerTypeBorderLayer,
50 | DeserializeBorderLayer)
51 | serializer.RegisterTypedDeserializer(serializerTypeSoftmaxLayer,
52 | DeserializeSoftmaxLayer)
53 | serializer.RegisterTypedDeserializer(serializerTypeLogSoftmaxLayer,
54 | DeserializeLogSoftmaxLayer)
55 | serializer.RegisterTypedDeserializer(serializerTypeMaxPoolingLayer,
56 | DeserializeMaxPoolingLayer)
57 | serializer.RegisterTypedDeserializer(serializerTypeUnstackLayer,
58 | DeserializeUnstackLayer)
59 | serializer.RegisterTypedDeserializer(serializerTypeRescaleLayer,
60 | DeserializeRescaleLayer)
61 | serializer.RegisterTypedDeserializer(serializerTypeDropoutLayer,
62 | DeserializeDropoutLayer)
63 | serializer.RegisterTypedDeserializer(serializerTypeVecRescaleLayer,
64 | DeserializeVecRescaleLayer)
65 | serializer.RegisterTypedDeserializer(serializerTypeGaussNoiseLayer,
66 | DeserializeGaussNoiseLayer)
67 | serializer.RegisterTypedDeserializer(serializerTypeResidualLayer,
68 | DeserializeResidualLayer)
69 | }
70 |
--------------------------------------------------------------------------------
/neuralnet/batch_rgradienter_test.go:
--------------------------------------------------------------------------------
1 | package neuralnet
2 |
3 | import (
4 | "math"
5 | "math/rand"
6 | "runtime"
7 | "testing"
8 |
9 | "github.com/unixpickle/autofunc"
10 | "github.com/unixpickle/num-analysis/linalg"
11 | )
12 |
13 | const (
14 | batchRGradienterSeed = 123123
15 | batchRGradienterTestPrec = 1e-5
16 | )
17 |
18 | func TestBatchRGradienterSimple(t *testing.T) {
19 | testBatchRGradienter(t, 1, &BatchRGradienter{
20 | CostFunc: MeanSquaredCost{},
21 | MaxGoroutines: 1,
22 | MaxBatchSize: 1,
23 | })
24 | }
25 |
26 | func TestBatchRGradienterUnevenBatch(t *testing.T) {
27 | testBatchRGradienter(t, 16, &BatchRGradienter{
28 | CostFunc: MeanSquaredCost{},
29 | MaxGoroutines: 1,
30 | MaxBatchSize: 3,
31 | })
32 | }
33 |
34 | func TestBatchRGradienterUnevenConcurrent(t *testing.T) {
35 | n := runtime.GOMAXPROCS(0)
36 | runtime.GOMAXPROCS(8)
37 | testBatchRGradienter(t, 16, &BatchRGradienter{
38 | CostFunc: MeanSquaredCost{},
39 | MaxGoroutines: 8,
40 | MaxBatchSize: 1,
41 | })
42 | runtime.GOMAXPROCS(n)
43 | }
44 |
45 | func testBatchRGradienter(t *testing.T, batchSize int, b *BatchRGradienter) {
46 | rand.Seed(batchRGradienterSeed)
47 |
48 | net := Network{
49 | &DenseLayer{
50 | InputCount: 10,
51 | OutputCount: 30,
52 | },
53 | &Sigmoid{},
54 | &DenseLayer{
55 | InputCount: 30,
56 | OutputCount: 3,
57 | },
58 | &Sigmoid{},
59 | }
60 | net.Randomize()
61 | b.Learner = net.BatchLearner()
62 |
63 | inputs := make([]linalg.Vector, batchSize)
64 | outputs := make([]linalg.Vector, batchSize)
65 | for i := range inputs {
66 | inputVec := make(linalg.Vector, 10)
67 | outputVec := make(linalg.Vector, 3)
68 | for j := range inputVec {
69 | inputVec[j] = rand.NormFloat64()
70 | }
71 | for j := range outputVec {
72 | outputVec[j] = rand.Float64()
73 | }
74 | inputs[i] = inputVec
75 | outputs[i] = outputVec
76 | }
77 | samples := VectorSampleSet(inputs, outputs)
78 |
79 | rVector := autofunc.RVector(autofunc.NewGradient(net.Parameters()))
80 | for _, vec := range rVector {
81 | for i := range vec {
82 | vec[i] = rand.NormFloat64()
83 | }
84 | }
85 |
86 | single := SingleRGradienter{Learner: net, CostFunc: b.CostFunc}
87 | expectedGrad := single.Gradient(samples)
88 | actualGrad := b.Gradient(samples)
89 |
90 | if !vecMapsEqual(expectedGrad, actualGrad) {
91 | t.Error("bad gradient from Gradient()")
92 | }
93 |
94 | expectedGrad, expectedRGrad := single.RGradient(rVector, samples)
95 | actualGrad, actualRGrad := b.RGradient(rVector, samples)
96 |
97 | if !vecMapsEqual(expectedGrad, actualGrad) {
98 | t.Error("bad gradient from RGradient()")
99 | }
100 | if !vecMapsEqual(expectedRGrad, actualRGrad) {
101 | t.Error("bad r-gradient from RGradient()")
102 | }
103 | }
104 |
105 | func vecMapsEqual(m1, m2 map[*autofunc.Variable]linalg.Vector) bool {
106 | for k := range m1 {
107 | if _, ok := m2[k]; !ok {
108 | return false
109 | }
110 | }
111 | for k := range m2 {
112 | if _, ok := m1[k]; !ok {
113 | return false
114 | }
115 | }
116 |
117 | for k, v := range m1 {
118 | v1 := m2[k]
119 | if len(v) != len(v1) {
120 | return false
121 | }
122 | for i, x := range v {
123 | if math.Abs(x-v1[i]) > batchRGradienterTestPrec {
124 | return false
125 | }
126 | }
127 | }
128 |
129 | return true
130 | }
131 |
--------------------------------------------------------------------------------
/rnn/seqtoseq/cost.go:
--------------------------------------------------------------------------------
1 | package seqtoseq
2 |
3 | import (
4 | "github.com/unixpickle/autofunc"
5 | "github.com/unixpickle/autofunc/seqfunc"
6 | "github.com/unixpickle/num-analysis/linalg"
7 | "github.com/unixpickle/sgd"
8 | "github.com/unixpickle/weakai/neuralnet"
9 | "github.com/unixpickle/weakai/rnn"
10 | )
11 |
12 | // TotalCostBlock runs an rnn.Block on a set of Samples
13 | // and evaluates the total output cost.
14 | //
15 | // The batchSize specifies how many samples to run in
16 | // batches while computing the cost.
17 | // If it is 0, the whole thing is run in one batch.
18 | func TotalCostBlock(b rnn.Block, batchSize int, s sgd.SampleSet, c neuralnet.CostFunc) float64 {
19 | if batchSize == 0 {
20 | return blockBatchCost(b, s, c)
21 | }
22 | var cost float64
23 | for i := 0; i < s.Len(); i += batchSize {
24 | bs := batchSize
25 | if bs > s.Len()-i {
26 | bs = s.Len() - i
27 | }
28 | cost += blockBatchCost(b, s.Subset(i, i+bs), c)
29 | }
30 | return cost
31 | }
32 |
33 | // TotalCostSeqFunc runs a seqfunc.RFunc on a set of
34 | // Samples and evaluates the total output cost.
35 | //
36 | // The batchSize specifies how many samples to run in
37 | // batches while computing the cost.
38 | func TotalCostSeqFunc(f seqfunc.RFunc, batchSize int, s sgd.SampleSet,
39 | c neuralnet.CostFunc) float64 {
40 | var totalCost float64
41 | for i := 0; i < s.Len(); i += batchSize {
42 | var inSeqs [][]linalg.Vector
43 | var outSeqs [][]linalg.Vector
44 | for j := i; j < i+batchSize && j < s.Len(); j++ {
45 | seq := s.GetSample(j).(Sample)
46 | inSeqs = append(inSeqs, seq.Inputs)
47 | outSeqs = append(outSeqs, seq.Outputs)
48 | }
49 | output := f.ApplySeqs(seqfunc.ConstResult(inSeqs))
50 | for j, actualSeq := range output.OutputSeqs() {
51 | expectedSeq := outSeqs[j]
52 | for k, actual := range actualSeq {
53 | expected := expectedSeq[k]
54 | actualVar := &autofunc.Variable{Vector: actual}
55 | totalCost += c.Cost(expected, actualVar).Output()[0]
56 | }
57 | }
58 | }
59 | return totalCost
60 | }
61 |
62 | func blockBatchCost(b rnn.Block, s sgd.SampleSet, c neuralnet.CostFunc) float64 {
63 | states := make([]rnn.State, s.Len())
64 | inputs := make([][]linalg.Vector, s.Len())
65 | outputs := make([][]linalg.Vector, s.Len())
66 | maxLen := 0
67 | for i := range states {
68 | states[i] = b.StartState()
69 | s := s.GetSample(i).(Sample)
70 | inputs[i] = s.Inputs
71 | outputs[i] = s.Outputs
72 | if len(s.Inputs) > maxLen {
73 | maxLen = len(s.Inputs)
74 | }
75 | }
76 | var totalCost float64
77 | for i := 0; i < maxLen; i++ {
78 | inStates := make([]rnn.State, 0, s.Len())
79 | ins := make([]autofunc.Result, 0, s.Len())
80 | outs := make([]linalg.Vector, 0, s.Len())
81 | for j, x := range inputs {
82 | if len(x) > 0 {
83 | ins = append(ins, &autofunc.Variable{Vector: x[0]})
84 | outs = append(outs, outputs[j][0])
85 | inStates = append(inStates, states[j])
86 | }
87 | }
88 | result := b.ApplyBlock(inStates, ins)
89 | for j, out := range result.Outputs() {
90 | totalCost += c.Cost(outs[j], &autofunc.Variable{Vector: out}).Output()[0]
91 | }
92 | var stateIdx int
93 | for j, x := range inputs {
94 | if len(x) > 0 {
95 | inputs[j] = x[1:]
96 | outputs[j] = outputs[j][1:]
97 | states[j] = result.States()[stateIdx]
98 | stateIdx++
99 | }
100 | }
101 | }
102 | return totalCost
103 | }
104 |
--------------------------------------------------------------------------------
/demos/idtrees/solve_csv/csv.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "encoding/csv"
5 | "errors"
6 | "io"
7 | "strconv"
8 | "strings"
9 |
10 | "github.com/unixpickle/weakai/idtrees"
11 | )
12 |
13 | func ReadCSV(r io.Reader) ([]idtrees.Sample, []idtrees.Attr, error) {
14 | records, err := readStringRecords(r)
15 | if err != nil {
16 | return nil, nil, err
17 | }
18 | rawVals := stringsToValues(records)
19 | key, err := classKey(rawVals)
20 | if err != nil {
21 | return nil, nil, err
22 | }
23 | res := make([]idtrees.Sample, len(rawVals))
24 | for i, x := range rawVals {
25 | res[i] = &csvSample{
26 | Map: x,
27 | ClassAttr: key,
28 | }
29 | }
30 | return res, trainingKeys(rawVals), nil
31 | }
32 |
33 | type csvSample struct {
34 | Map map[string]interface{}
35 | ClassAttr string
36 | }
37 |
38 | func (c *csvSample) Attr(a idtrees.Attr) idtrees.Val {
39 | return c.Map[a.(string)]
40 | }
41 |
42 | func (c *csvSample) Class() idtrees.Class {
43 | return c.Map[c.ClassAttr]
44 | }
45 |
46 | // trainingKeys returns the keys in the data set to
47 | // use for training.
48 | func trainingKeys(c []map[string]interface{}) []idtrees.Attr {
49 | var res []idtrees.Attr
50 | if len(c) > 0 {
51 | for key := range c[0] {
52 | if !strings.HasPrefix(key, "_") && !strings.HasPrefix(key, "*") {
53 | res = append(res, key)
54 | }
55 | }
56 | }
57 | return res
58 | }
59 |
60 | func classKey(c []map[string]interface{}) (string, error) {
61 | var useKey string
62 | if len(c) > 0 {
63 | for key := range c[0] {
64 | if strings.HasPrefix(key, "*") {
65 | if useKey != "" {
66 | return "", errors.New("multiple keys begin with asterisk")
67 | }
68 | useKey = key
69 | }
70 | }
71 | }
72 | if useKey == "" {
73 | return "", errors.New("missing class attribute " +
74 | "(name prefixed with asterisk)")
75 | }
76 | return useKey, nil
77 | }
78 |
79 | func readStringRecords(r io.Reader) ([]map[string]string, error) {
80 | csvReader := csv.NewReader(r)
81 | allRecords, err := csvReader.ReadAll()
82 | if err != nil {
83 | return nil, err
84 | }
85 | if len(allRecords) < 2 {
86 | return nil, nil
87 | }
88 | header := allRecords[0]
89 | res := make([]map[string]string, len(allRecords)-1)
90 | for i, x := range allRecords[1:] {
91 | r := map[string]string{}
92 | for j, val := range x {
93 | r[header[j]] = val
94 | }
95 | res[i] = r
96 | }
97 | return res, nil
98 | }
99 |
100 | func stringsToValues(strs []map[string]string) []map[string]interface{} {
101 | if len(strs) == 0 {
102 | return nil
103 | }
104 | res := make([]map[string]interface{}, len(strs))
105 | for i := range res {
106 | res[i] = map[string]interface{}{}
107 | }
108 | for key := range strs[0] {
109 | allInt, allFloat := true, true
110 | for _, x := range strs {
111 | _, intErr := strconv.ParseInt(x[key], 0, 64)
112 | _, floatErr := strconv.ParseFloat(x[key], 64)
113 | if intErr != nil {
114 | allInt = false
115 | }
116 | if floatErr != nil {
117 | allFloat = false
118 | }
119 | }
120 | if allInt {
121 | for i, x := range strs {
122 | num, _ := strconv.ParseInt(x[key], 0, 64)
123 | res[i][key] = num
124 | }
125 | } else if allFloat {
126 | for i, x := range strs {
127 | num, _ := strconv.ParseFloat(x[key], 64)
128 | res[i][key] = num
129 | }
130 | } else {
131 | for i, x := range strs {
132 | res[i][key] = x[key]
133 | }
134 | }
135 | }
136 | return res
137 | }
138 |
--------------------------------------------------------------------------------
/demos/mapcolor/usa.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | var AdjacentUSAStates = map[string][]string{
4 | "AK": {},
5 | "AL": {"TN", "GA", "FL", "MS"},
6 | "AR": {"MO", "TN", "MS", "LA", "TX", "OK"},
7 | "AZ": {"UT", "CO", "NM", "CA", "NV"},
8 | "CA": {"OR", "NV", "AZ", "HI"},
9 | "CO": {"WY", "NE", "KS", "OK", "NM", "AZ", "UT"},
10 | "CT": {"MA", "RI", "NY"},
11 | "DC": {"MD", "VA"},
12 | "DE": {"PA", "NJ", "MD"},
13 | "FL": {"GA", "AL"},
14 | "GA": {"NC", "SC", "FL", "AL", "TN"},
15 | "HI": {},
16 | "IA": {"MN", "WI", "IL", "MO", "NE", "SD"},
17 | "ID": {"MT", "WY", "UT", "NV", "OR", "WA"},
18 | "IL": {"WI", "IN", "KY", "MO", "IA"},
19 | "IN": {"MI", "OH", "KY", "IL"},
20 | "KS": {"NE", "MO", "OK", "CO"},
21 | "KY": {"OH", "WV", "VA", "TN", "MO", "IL", "IN"},
22 | "LA": {"AR", "MS", "TX"},
23 | "MA": {"NH", "RI", "CT", "NY", "VT"},
24 | "MD": {"PA", "DE", "DC", "VA", "WV"},
25 | "ME": {"NH"},
26 | "MI": {"OH", "IN", "WI"},
27 | "MN": {"WI", "IA", "SD", "ND"},
28 | "MO": {"IA", "IL", "KY", "TN", "AR", "OK", "KS", "NE"},
29 | "MS": {"TN", "AL", "LA", "AR"},
30 | "MT": {"ND", "SD", "WY", "ID"},
31 | "NC": {"VA", "SC", "GA", "TN"},
32 | "ND": {"MN", "SD", "MT"},
33 | "NE": {"SD", "IA", "MO", "KS", "CO", "WY"},
34 | "NH": {"ME", "MA", "VT"},
35 | "NJ": {"NY", "DE", "PA"},
36 | "NM": {"CO", "OK", "TX", "AZ", "UT"},
37 | "NV": {"ID", "UT", "AZ", "CA", "OR"},
38 | "NY": {"VT", "MA", "CT", "NJ", "PA"},
39 | "OH": {"PA", "WV", "KY", "IN", "MI"},
40 | "OK": {"KS", "MO", "AR", "TX", "NM", "CO"},
41 | "OR": {"WA", "ID", "NV", "CA"},
42 | "PA": {"NY", "NJ", "DE", "MD", "WV", "OH"},
43 | "RI": {"MA", "CT"},
44 | "SC": {"NC", "GA"},
45 | "SD": {"ND", "MN", "IA", "NE", "WY", "MT"},
46 | "TN": {"KY", "VA", "NC", "GA", "AL", "MS", "AR", "MO"},
47 | "TX": {"OK", "AR", "LA", "NM"},
48 | "UT": {"ID", "WY", "CO", "NM", "AZ", "NV"},
49 | "VA": {"MD", "DC", "NC", "TN", "KY", "WV"},
50 | "VT": {"NH", "MA", "NY"},
51 | "WA": {"AK", "ID", "OR"},
52 | "WI": {"MI", "IL", "IA", "MN"},
53 | "WV": {"PA", "MD", "VA", "KY", "OH"},
54 | "WY": {"MT", "SD", "NE", "CO", "UT", "ID"},
55 | }
56 |
57 | type StateVariable struct {
58 | Name string
59 | constraints []Constraint
60 | }
61 |
62 | func (s *StateVariable) Constraints() []Constraint {
63 | return s.constraints
64 | }
65 |
66 | func (s *StateVariable) InitialDomain() Domain {
67 | return Domain{"#65bcd4", "#9b59b6", "#e36a8f", "#33be6e"}
68 | }
69 |
70 | type StateConstraint []Variable
71 |
72 | func (c StateConstraint) Variables() []Variable {
73 | return c
74 | }
75 |
76 | func (c StateConstraint) RestrictedDomain(idx int, d Domain) Domain {
77 | if len(d) != 1 {
78 | return c[idx].InitialDomain()
79 | }
80 | res := Domain{}
81 | for _, x := range c[idx].InitialDomain() {
82 | if !d.Contains(x) {
83 | res = append(res, x)
84 | }
85 | }
86 | return res
87 | }
88 |
89 | func VariablesForStates() []Variable {
90 | vars := map[string]Variable{}
91 | result := make([]Variable, 0, len(AdjacentUSAStates))
92 | for stateName := range AdjacentUSAStates {
93 | v := &StateVariable{Name: stateName, constraints: []Constraint{}}
94 | vars[stateName] = v
95 | result = append(result, v)
96 | }
97 | for stateName, connections := range AdjacentUSAStates {
98 | state := vars[stateName]
99 | for _, connection := range connections {
100 | c := StateConstraint{state, vars[connection]}
101 | state.(*StateVariable).constraints = append(state.(*StateVariable).constraints, c)
102 | }
103 | }
104 | return result
105 | }
106 |
--------------------------------------------------------------------------------
/demos/svm/image_categorizing/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "image"
6 | "os"
7 | "path/filepath"
8 | "strings"
9 | "time"
10 |
11 | "github.com/unixpickle/weakai/svm"
12 | )
13 |
14 | func main() {
15 | if len(os.Args) != 4 {
16 | fmt.Fprintln(os.Stderr, "Usage: face_recog ")
17 | os.Exit(1)
18 | }
19 |
20 | fmt.Println("Loading images...")
21 | negatives := loadImages(os.Args[1])
22 | positives := loadImages(os.Args[2])
23 |
24 | allImages := make([]*Image, len(negatives)+len(positives))
25 | copy(allImages, negatives)
26 | copy(allImages[len(negatives):], positives)
27 | width, height := minimumSize(allImages)
28 |
29 | problem := &svm.Problem{
30 | Kernel: svm.CachedKernel(svm.LinearKernel),
31 | Positives: make([]svm.Sample, len(positives)),
32 | Negatives: make([]svm.Sample, len(negatives)),
33 | }
34 |
35 | fmt.Println("Cropping samples...")
36 | // TODO: perhaps normalize the image vectors.
37 | for i, x := range negatives {
38 | vec := cutOutMiddle(x, width, height).Vector
39 | problem.Negatives[i] = svm.Sample{V: vec, UserInfo: i + 1}
40 | }
41 | for i, x := range positives {
42 | vec := cutOutMiddle(x, width, height).Vector
43 | problem.Positives[i] = svm.Sample{V: vec, UserInfo: len(negatives) + 1 + i}
44 | }
45 |
46 | fmt.Println("Solving...")
47 | solver := svm.GradientDescentSolver{
48 | Tradeoff: 0.0001,
49 | Timeout: time.Minute,
50 | }
51 | classifier := solver.Solve(problem).Linearize()
52 |
53 | image := imageForSolution(width, height, classifier.HyperplaneNormal.V)
54 | if err := image.WriteFile(os.Args[3]); err != nil {
55 | fmt.Fprintln(os.Stderr, err)
56 | os.Exit(1)
57 | }
58 | }
59 |
60 | func loadImages(dirPath string) []*Image {
61 | file, err := os.Open(dirPath)
62 | if err != nil {
63 | fmt.Fprintln(os.Stderr, err)
64 | os.Exit(1)
65 | }
66 | defer file.Close()
67 | names, err := file.Readdirnames(-1)
68 | if err != nil {
69 | fmt.Fprintln(os.Stderr, err)
70 | os.Exit(1)
71 | }
72 | res := make([]*Image, 0, len(names))
73 | for _, name := range names {
74 | if strings.HasPrefix(name, ".") {
75 | continue
76 | }
77 | image, err := ReadImage(filepath.Join(dirPath, name))
78 | if err != nil {
79 | fmt.Fprintln(os.Stderr, err)
80 | os.Exit(1)
81 | }
82 | res = append(res, image)
83 | }
84 | return res
85 | }
86 |
87 | func minimumSize(images []*Image) (width, height int) {
88 | width, height = images[0].Width, images[0].Height
89 | for _, img := range images {
90 | if img.Width < width {
91 | width = img.Width
92 | }
93 | if img.Height < height {
94 | height = img.Height
95 | }
96 | }
97 | return
98 | }
99 |
100 | func cutOutMiddle(img *Image, width, height int) *Image {
101 | cutLeft := (img.Width - width) / 2
102 | cutTop := (img.Height - height) / 2
103 | return img.Crop(image.Rect(cutLeft, cutTop, cutLeft+width, cutTop+height))
104 | }
105 |
106 | func imageForSolution(width, height int, solution []float64) *Image {
107 | var minPixel float64
108 | var maxPixel float64
109 | for i, sample := range solution {
110 | if sample < minPixel || i == 0 {
111 | minPixel = sample
112 | }
113 | if sample > maxPixel || i == 0 {
114 | maxPixel = sample
115 | }
116 | }
117 | for i := range solution {
118 | solution[i] += minPixel
119 | solution[i] /= (maxPixel - minPixel)
120 | }
121 |
122 | return &Image{
123 | Vector: solution,
124 | Width: width,
125 | Height: height,
126 | }
127 | }
128 |
--------------------------------------------------------------------------------
/rbm/rbm.go:
--------------------------------------------------------------------------------
1 | // Package rbm implements Restricted Boltzmann Machines.
2 | package rbm
3 |
4 | import (
5 | "math"
6 | "math/rand"
7 |
8 | "github.com/unixpickle/num-analysis/kahan"
9 | "github.com/unixpickle/num-analysis/linalg"
10 | )
11 |
12 | // An RBM stores the parameters of a
13 | // Restricted Boltzmann Machine.
14 | type RBM struct {
15 | Weights *linalg.Matrix
16 | HiddenBiases linalg.Vector
17 | VisibleBiases linalg.Vector
18 | }
19 |
20 | func NewRBM(visibleCount, hiddenCount int) *RBM {
21 | return &RBM{
22 | Weights: linalg.NewMatrix(hiddenCount, visibleCount),
23 | HiddenBiases: make(linalg.Vector, hiddenCount),
24 | VisibleBiases: make(linalg.Vector, visibleCount),
25 | }
26 | }
27 |
28 | // Randomize initializes the weights randomly.
29 | // The random values will be clamped to
30 | // the range [-randMag, randMag].
31 | func (r *RBM) Randomize(randMag float64) {
32 | for i := range r.Weights.Data {
33 | r.Weights.Data[i] = rand.Float64()*randMag*2 - randMag
34 | }
35 | }
36 |
37 | // SampleVisible generates a random visible vector
38 | // given a vector of hidden layer values.
39 | // The visible vector will be written to output,
40 | // allowing the caller to cache a slice for visible
41 | // samples.
42 | //
43 | // If ra is nil, this uses the rand package's
44 | // default generator.
45 | func (r *RBM) SampleVisible(ra *rand.Rand, output, hiddenValues []bool) {
46 | expected := r.ExpectedVisible(hiddenValues)
47 | sampleVector(ra, output, expected)
48 | }
49 |
50 | // SampleHidden generates a random hidden vector
51 | // given a vector of visible values.
52 | // The hidden values will be written to output,
53 | // allowing the caller to cache a slice for hidden
54 | // samples.
55 | //
56 | // If ra is nil, this uses the rand package's
57 | // default generator.
58 | func (r *RBM) SampleHidden(ra *rand.Rand, output, visibleValues []bool) {
59 | expected := r.ExpectedHidden(visibleValues)
60 | sampleVector(ra, output, expected)
61 | }
62 |
63 | // ExpectedVisible returns the expected value of
64 | // the visible layer given a hidden vector.
65 | func (r *RBM) ExpectedVisible(hidden []bool) linalg.Vector {
66 | result := make(linalg.Vector, len(r.VisibleBiases))
67 | for i := range result {
68 | var sum kahan.Summer64
69 | for j, h := range hidden {
70 | if h {
71 | sum.Add(r.Weights.Get(j, i))
72 | }
73 | }
74 | result[i] = sum.Sum()
75 | }
76 |
77 | result.Add(r.VisibleBiases)
78 | mapSigmoid(result)
79 |
80 | return result
81 | }
82 |
83 | // ExpectedHidden returns the expected value of
84 | // the hidden layer given a visible vector.
85 | func (r *RBM) ExpectedHidden(visible []bool) linalg.Vector {
86 | result := make(linalg.Vector, len(r.HiddenBiases))
87 | for i := range result {
88 | var sum kahan.Summer64
89 | for j, v := range visible {
90 | if v {
91 | sum.Add(r.Weights.Get(i, j))
92 | }
93 | }
94 | result[i] = sum.Sum()
95 | }
96 |
97 | result.Add(r.HiddenBiases)
98 | mapSigmoid(result)
99 |
100 | return result
101 | }
102 |
103 | func mapSigmoid(v linalg.Vector) {
104 | for i, x := range v {
105 | e := math.Exp(x)
106 | v[i] = e / (1 + e)
107 | }
108 | }
109 |
110 | func sampleVector(r *rand.Rand, output []bool, expected linalg.Vector) {
111 | for i, prob := range expected {
112 | var num float64
113 | if r != nil {
114 | num = r.Float64()
115 | } else {
116 | num = rand.Float64()
117 | }
118 | if num >= prob {
119 | output[i] = false
120 | } else {
121 | output[i] = true
122 | }
123 | }
124 | }
125 |
--------------------------------------------------------------------------------
/rnn/seqtoseq/seq_func.go:
--------------------------------------------------------------------------------
1 | package seqtoseq
2 |
3 | import (
4 | "github.com/unixpickle/autofunc"
5 | "github.com/unixpickle/autofunc/seqfunc"
6 | "github.com/unixpickle/num-analysis/linalg"
7 | "github.com/unixpickle/sgd"
8 | "github.com/unixpickle/weakai/neuralnet"
9 | )
10 |
11 | // A Gradienter is an sgd.RGradienter which works on a
12 | // seqfunc.RFunc for sample sets of Sequence objectg.
13 | //
14 | // After an instance is used once, it should never be
15 | // reused with different parameterg.
16 | type Gradienter struct {
17 | SeqFunc seqfunc.RFunc
18 | Learner sgd.Learner
19 | CostFunc neuralnet.CostFunc
20 |
21 | // MaxLanes specifies the maximum number of lanes
22 | // any BlockInput or BlockRInput may have at once
23 | // while computing gradientg.
24 | // If this is 0, a reasonable default is used.
25 | MaxLanes int
26 |
27 | // MaxGoroutines specifies the maximum number of
28 | // Goroutines on which to invoke Batch or BatchR
29 | // on Learner at once.
30 | MaxGoroutines int
31 |
32 | helper *neuralnet.GradHelper
33 | }
34 |
35 | func (g *Gradienter) Gradient(set sgd.SampleSet) autofunc.Gradient {
36 | return g.makeHelper().Gradient(sortSeqs(set))
37 | }
38 |
39 | func (g *Gradienter) RGradient(v autofunc.RVector,
40 | set sgd.SampleSet) (autofunc.Gradient, autofunc.RGradient) {
41 | return g.makeHelper().RGradient(v, sortSeqs(set))
42 | }
43 |
44 | func (g *Gradienter) makeHelper() *neuralnet.GradHelper {
45 | if g.helper != nil {
46 | g.helper.MaxConcurrency = g.MaxGoroutines
47 | g.helper.MaxSubBatch = g.MaxLanes
48 | return g.helper
49 | }
50 | g.helper = &neuralnet.GradHelper{
51 | MaxConcurrency: g.MaxGoroutines,
52 | MaxSubBatch: g.MaxLanes,
53 | Learner: g.Learner,
54 | CompGrad: g.runBatch,
55 | CompRGrad: g.runBatchR,
56 | }
57 | return g.helper
58 | }
59 |
60 | func (g *Gradienter) runBatch(grad autofunc.Gradient, set sgd.SampleSet) {
61 | seqs := sampleSetSlice(set)
62 | var seqIns [][]linalg.Vector
63 | for _, s := range seqs {
64 | seqIns = append(seqIns, s.Inputs)
65 | }
66 | output := g.SeqFunc.ApplySeqs(seqfunc.ConstResult(seqIns))
67 |
68 | upstream := make([][]linalg.Vector, len(seqIns))
69 | for i, outSeq := range output.OutputSeqs() {
70 | us := make([]linalg.Vector, len(outSeq))
71 | expectedSeq := seqs[i].Outputs
72 | for j, actual := range outSeq {
73 | expected := expectedSeq[j]
74 | us[j] = costFuncDeriv(g.CostFunc, expected, actual)
75 | }
76 | upstream[i] = us
77 | }
78 |
79 | output.PropagateGradient(upstream, grad)
80 | }
81 |
82 | func (g *Gradienter) runBatchR(rv autofunc.RVector, rg autofunc.RGradient,
83 | grad autofunc.Gradient, set sgd.SampleSet) {
84 | seqs := sampleSetSlice(set)
85 | seqIns := make([][]linalg.Vector, len(seqs))
86 | for _, s := range seqs {
87 | seqIns = append(seqIns, s.Inputs)
88 | }
89 | output := g.SeqFunc.ApplySeqsR(rv, seqfunc.ConstRResult(seqIns))
90 |
91 | upstream := make([][]linalg.Vector, len(seqIns))
92 | upstreamR := make([][]linalg.Vector, len(seqIns))
93 | for i, outSeq := range output.OutputSeqs() {
94 | rOutSeq := output.ROutputSeqs()[i]
95 | us := make([]linalg.Vector, len(outSeq))
96 | usR := make([]linalg.Vector, len(outSeq))
97 | expectedSeq := seqs[i].Outputs
98 | for j, actual := range outSeq {
99 | expected := expectedSeq[j]
100 | us[j], usR[j] = costFuncRDeriv(g.CostFunc, expected, actual, rOutSeq[j])
101 | }
102 | upstream[i] = us
103 | upstreamR[i] = usR
104 | }
105 |
106 | output.PropagateRGradient(upstream, upstreamR, rg, grad)
107 | }
108 |
--------------------------------------------------------------------------------
/demos/neuralnet/image_demos/mnist/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "log"
5 |
6 | "github.com/unixpickle/autofunc"
7 | "github.com/unixpickle/mnist"
8 | "github.com/unixpickle/num-analysis/linalg"
9 | "github.com/unixpickle/sgd"
10 | "github.com/unixpickle/weakai/neuralnet"
11 | )
12 |
13 | const (
14 | HiddenSize = 300
15 | LabelCount = 10
16 | StepSize = 1e-2
17 | BatchSize = 20
18 |
19 | FilterSize = 3
20 | FilterCount = 5
21 | FilterStride = 1
22 |
23 | MaxPoolingSpan = 3
24 | )
25 |
26 | func main() {
27 | training := mnist.LoadTrainingDataSet()
28 | crossValidation := mnist.LoadTestingDataSet()
29 |
30 | net := createNet(training)
31 |
32 | trainingSamples := dataSetSamples(training)
33 | gradienter := &neuralnet.BatchRGradienter{
34 | Learner: net.BatchLearner(),
35 | CostFunc: neuralnet.MeanSquaredCost{},
36 | }
37 | rmsGrad := &sgd.RMSProp{Gradienter: gradienter}
38 |
39 | sgd.SGDInteractive(rmsGrad, trainingSamples, StepSize, BatchSize, func() bool {
40 | log.Println("Printing score...")
41 | printScore("Cross", net, crossValidation)
42 | log.Println("Running training round...")
43 | return true
44 | })
45 | }
46 |
47 | func createNet(d mnist.DataSet) neuralnet.Network {
48 | convOutWidth := (d.Width-FilterSize)/FilterStride + 1
49 | convOutHeight := (d.Height-FilterSize)/FilterStride + 1
50 |
51 | poolOutWidth := convOutWidth / MaxPoolingSpan
52 | if convOutWidth%MaxPoolingSpan != 0 {
53 | poolOutWidth++
54 | }
55 | poolOutHeight := convOutWidth / MaxPoolingSpan
56 | if convOutHeight%MaxPoolingSpan != 0 {
57 | poolOutHeight++
58 | }
59 |
60 | net := neuralnet.Network{
61 | &neuralnet.ConvLayer{
62 | FilterCount: FilterCount,
63 | FilterWidth: FilterSize,
64 | FilterHeight: FilterSize,
65 | Stride: FilterStride,
66 | InputWidth: d.Width,
67 | InputHeight: d.Height,
68 | InputDepth: 1,
69 | },
70 | &neuralnet.Sigmoid{},
71 | &neuralnet.MaxPoolingLayer{
72 | XSpan: MaxPoolingSpan,
73 | YSpan: MaxPoolingSpan,
74 | InputWidth: convOutWidth,
75 | InputHeight: convOutHeight,
76 | InputDepth: FilterCount,
77 | },
78 | &neuralnet.DenseLayer{
79 | InputCount: poolOutWidth * poolOutHeight * FilterCount,
80 | OutputCount: HiddenSize,
81 | },
82 | &neuralnet.Sigmoid{},
83 | &neuralnet.DenseLayer{
84 | InputCount: HiddenSize,
85 | OutputCount: LabelCount,
86 | },
87 | &neuralnet.SoftmaxLayer{},
88 | }
89 | net.Randomize()
90 |
91 | return net
92 | }
93 |
94 | func printScore(prefix string, n neuralnet.Network, d mnist.DataSet) {
95 | classifier := func(v []float64) int {
96 | result := n.Apply(&autofunc.Variable{v})
97 | return outputIdx(result)
98 | }
99 | correctCount := d.NumCorrect(classifier)
100 | histogram := d.CorrectnessHistogram(classifier)
101 | log.Printf("%s: %d/%d - %s", prefix, correctCount, len(d.Samples), histogram)
102 | }
103 |
104 | func outputIdx(r autofunc.Result) int {
105 | out := r.Output()
106 | var maxIdx int
107 | var max float64
108 | for i, x := range out {
109 | if i == 0 || x > max {
110 | max = x
111 | maxIdx = i
112 | }
113 | }
114 | return maxIdx
115 | }
116 |
117 | func dataSetSamples(d mnist.DataSet) sgd.SampleSet {
118 | labelVecs := d.LabelVectors()
119 | inputVecs := d.IntensityVectors()
120 | return neuralnet.VectorSampleSet(vecVec(inputVecs), vecVec(labelVecs))
121 | }
122 |
123 | func vecVec(f [][]float64) []linalg.Vector {
124 | res := make([]linalg.Vector, len(f))
125 | for i, x := range f {
126 | res[i] = x
127 | }
128 | return res
129 | }
130 |
--------------------------------------------------------------------------------