├── Ch10 ├── PBIL.py ├── billsfit.py ├── exhaustiveKnapsack.py ├── fourpeaks.py ├── ga.py ├── greedyKnapsack.py ├── knapsack.py ├── onemax.py └── run_ga.py ├── Ch11 ├── SARSA.py ├── SARSA_cliff.py ├── TDZero.py └── TDZero_cliff.py ├── Ch12 ├── dtree.py ├── party.data └── party.py ├── Ch13 ├── .DS_Store ├── bagging.py ├── boost.py ├── car.data ├── car.py ├── car1.py ├── dtree.py ├── dtw.py ├── mushroom.py ├── party.py └── randomforest.py ├── Ch14 ├── iris.py ├── iris_somperc.py ├── kmeans.py ├── kmeansnet.py ├── moredemos.py ├── som.py └── somdemo.py ├── Ch15 ├── BoxMuller.py ├── Gibbs.py ├── MH.py ├── SIR.py ├── importancesampling.py ├── lcg.py └── rejectionsampling.py ├── Ch16 ├── EKF.py ├── Gibbs.py ├── HMM.py ├── Kalman.py ├── Kalman_full.py ├── MRF.py ├── graphdemo.py ├── particle_filter.py ├── pftrack.py ├── world.gif └── world.png ├── Ch17 ├── binaryalphadigs.mat ├── dbn.py ├── hopfield.py └── rbm.py ├── Ch18 ├── data.txt ├── gp.py ├── gpc.py ├── gpcdemo.py ├── plotdist.py └── plotgp.py ├── Ch2 ├── gaussian.py └── plotGaussian.py ├── Ch3 ├── autompg.py ├── linreg.py ├── linreg_logic_eg.py ├── logic.py ├── mnist.py ├── pcn.py ├── pcn_logic_eg.py └── pima.py ├── Ch4 ├── PNOz.py ├── iris.py ├── iris_proc.data ├── logic.py ├── mlp.py ├── mnist.py └── sinewave.py ├── Ch5 ├── iris.py ├── least_squares.py └── rbf.py ├── Ch6 ├── ecoli.py ├── factoranalysis.py ├── floyd.py ├── iris.py ├── isomap.py ├── kernelpca.py ├── kpcademo.py ├── lda.py ├── lle.py ├── pca.py └── pcademo.py ├── Ch7 ├── GMM.py ├── kdtree.py ├── knn.py └── knnSmoother.py ├── Ch8 ├── svm.py ├── svmdemo.py └── svmdemo2.py ├── Ch9 ├── CG.py ├── LevenbergMarquardt.py ├── LevenbergMarquardt_leastsq.py ├── Newton.py ├── TSP.py ├── iris.py ├── mlp_cg.py └── steepest.py ├── Data ├── PNoz.dat ├── PROSTATE_TEST.TXT ├── PROSTATE_TRAIN.TXT ├── ruapehu.dat └── shortecoli.dat └── README.md /Ch10/PBIL.py: -------------------------------------------------------------------------------- 1 | 2 | # Code from Chapter 10 of Machine Learning: An Algorithmic Perspective (2nd Edition) 3 | # by Stephen Marsland (http://stephenmonika.net) 4 | 5 | # You are free to use, change, or redistribute the code in any way you wish for 6 | # non-commercial purposes, but please maintain the name of the original author. 7 | # This code comes with no warranty of any kind. 8 | 9 | # Stephen Marsland, 2008, 2014 10 | 11 | # The Population Based Incremental Learning algorithm 12 | # Comment and uncomment fitness functions as appropriate (as an import and the fitnessFunction variable) 13 | 14 | import pylab as pl 15 | import numpy as np 16 | 17 | #import fourpeaks as fF 18 | import knapsack as fF 19 | 20 | def PBIL(): 21 | pl.ion() 22 | 23 | populationSize = 100 24 | stringLength = 20 25 | eta = 0.005 26 | 27 | #fitnessFunction = 'fF.fourpeaks' 28 | fitnessFunction = 'fF.knapsack' 29 | p = 0.5*np.ones(stringLength) 30 | best = np.zeros(501,dtype=float) 31 | 32 | for count in range(501): 33 | # Generate samples 34 | population = np.random.rand(populationSize,stringLength) 35 | for i in range(stringLength): 36 | population[:,i] = np.where(population[:,i]maxSize,500-2*(fitness-maxSize),fitness) 23 | 24 | return fitness 25 | -------------------------------------------------------------------------------- /Ch10/exhaustiveKnapsack.py: -------------------------------------------------------------------------------- 1 | 2 | # Code from Chapter 10 of Machine Learning: An Algorithmic Perspective (2nd Edition) 3 | # by Stephen Marsland (http://stephenmonika.net) 4 | 5 | # You are free to use, change, or redistribute the code in any way you wish for 6 | # non-commercial purposes, but please maintain the name of the original author. 7 | # This code comes with no warranty of any kind. 8 | 9 | # Stephen Marsland, 2008, 2014 10 | 11 | # An exhaustive search to solve the Knapsack problem 12 | import numpy as np 13 | 14 | def exhaustive(): 15 | maxSize = 500 16 | sizes = np.array([109.60,125.48,52.16,195.55,58.67,61.87,92.95,93.14,155.05,110.89,13.34,132.49,194.03,121.29,179.33,139.02,198.78,192.57,81.66,128.90]) 17 | 18 | best = 0 19 | 20 | twos = np.arange(-len(sizes),0,1) 21 | twos = 2.0**twos 22 | 23 | for i in range(2**len(sizes)-1): 24 | string = np.remainder(np.floor(i*twos),2) 25 | fitness = np.sum(string*sizes) 26 | if fitness > best and fitness<500: 27 | best = fitness 28 | bestString = string 29 | print best 30 | print bestString 31 | 32 | exhaustive() 33 | -------------------------------------------------------------------------------- /Ch10/fourpeaks.py: -------------------------------------------------------------------------------- 1 | 2 | # Code from Chapter 10 of Machine Learning: An Algorithmic Perspective (2nd Edition) 3 | # by Stephen Marsland (http://stephenmonika.net) 4 | 5 | # You are free to use, change, or redistribute the code in any way you wish for 6 | # non-commercial purposes, but please maintain the name of the original author. 7 | # This code comes with no warranty of any kind. 8 | 9 | # Stephen Marsland, 2008, 2014 10 | 11 | # The four peaks fitness function 12 | import numpy as np 13 | def fourpeaks(population): 14 | 15 | T = 15 16 | start = np.zeros((np.shape(population)[0],1)) 17 | finish = np.zeros((np.shape(population)[0],1)) 18 | 19 | fitness = np.zeros((np.shape(population)[0],1)) 20 | 21 | for i in range(np.shape(population)[0]): 22 | s = np.where(population[i,:]==1) 23 | f = np.where(population[i,:]==0) 24 | if np.size(s)>0: 25 | start = s[0][0] 26 | else: 27 | start = 0 28 | 29 | if np.size(f)>0: 30 | finish = np.shape(population)[1] - f[-1][-1] -1 31 | else: 32 | finish = 0 33 | 34 | if start>T and finish>T: 35 | fitness[i] = np.maximum(start,finish)+100 36 | else: 37 | fitness[i] = np.maximum(start,finish) 38 | 39 | fitness = np.squeeze(fitness) 40 | return fitness 41 | -------------------------------------------------------------------------------- /Ch10/ga.py: -------------------------------------------------------------------------------- 1 | 2 | # Code from Chapter 10 of Machine Learning: An Algorithmic Perspective (2nd Edition) 3 | # by Stephen Marsland (http://stephenmonika.net) 4 | 5 | # You are free to use, change, or redistribute the code in any way you wish for 6 | # non-commercial purposes, but please maintain the name of the original author. 7 | # This code comes with no warranty of any kind. 8 | 9 | # Stephen Marsland, 2008, 2014 10 | 11 | 12 | # The Genetic algorithm 13 | # Comment and uncomment fitness functions as appropriate (as an import and the fitnessFunction variable) 14 | 15 | import pylab as pl 16 | import numpy as np 17 | import fourpeaks as fF 18 | 19 | class ga: 20 | 21 | def __init__(self,stringLength,fitnessFunction,nEpochs,populationSize=100,mutationProb=-1,crossover='un',nElite=4,tournament=True): 22 | """ Constructor""" 23 | self.stringLength = stringLength 24 | 25 | # Population size should be even 26 | if np.mod(populationSize,2)==0: 27 | self.populationSize = populationSize 28 | else: 29 | self.populationSize = populationSize+1 30 | 31 | if mutationProb < 0: 32 | self.mutationProb = 1/stringLength 33 | else: 34 | self.mutationProb = mutationProb 35 | 36 | self.nEpochs = nEpochs 37 | 38 | self.fitnessFunction = fitnessFunction 39 | 40 | self.crossover = crossover 41 | self.nElite = nElite 42 | self.tournment = tournament 43 | 44 | self.population = np.random.rand(self.populationSize,self.stringLength) 45 | self.population = np.where(self.population<0.5,0,1) 46 | 47 | def runGA(self,plotfig): 48 | """The basic loop""" 49 | pl.ion() 50 | #plotfig = pl.figure() 51 | bestfit = np.zeros(self.nEpochs) 52 | 53 | for i in range(self.nEpochs): 54 | # Compute fitness of the population 55 | fitness = eval(self.fitnessFunction)(self.population) 56 | 57 | # Pick parents -- can do in order since they are randomised 58 | newPopulation = self.fps(self.population,fitness) 59 | 60 | # Apply the genetic operators 61 | if self.crossover == 'sp': 62 | newPopulation = self.spCrossover(newPopulation) 63 | elif self.crossover == 'un': 64 | newPopulation = self.uniformCrossover(newPopulation) 65 | newPopulation = self.mutate(newPopulation) 66 | 67 | # Apply elitism and tournaments if using 68 | if self.nElite>0: 69 | newPopulation = self.elitism(self.population,newPopulation,fitness) 70 | 71 | if self.tournament: 72 | newPopulation = self.tournament(self.population,newPopulation,fitness,self.fitnessFunction) 73 | 74 | self.population = newPopulation 75 | bestfit[i] = fitness.max() 76 | 77 | if (np.mod(i,100)==0): 78 | print i, fitness.max() 79 | #pl.plot([i],[fitness.max()],'r+') 80 | pl.plot(bestfit,'kx-') 81 | #pl.show() 82 | 83 | def fps(self,population,fitness): 84 | 85 | # Scale fitness by total fitness 86 | fitness = fitness/np.sum(fitness) 87 | fitness = 10*fitness/fitness.max() 88 | 89 | # Put repeated copies of each string in according to fitness 90 | # Deal with strings with very low fitness 91 | j=0 92 | while np.round(fitness[j])<1: 93 | j = j+1 94 | 95 | newPopulation = np.kron(np.ones((np.round(fitness[j]),1)),population[j,:]) 96 | 97 | # Add multiple copies of strings into the newPopulation 98 | for i in range(j+1,self.populationSize): 99 | if np.round(fitness[i])>=1: 100 | newPopulation = np.concatenate((newPopulation,np.kron(np.ones((np.round(fitness[i]),1)),population[i,:])),axis=0) 101 | 102 | # Shuffle the order (note that there are still too many) 103 | indices = range(np.shape(newPopulation)[0]) 104 | np.random.shuffle(indices) 105 | newPopulation = newPopulation[indices[:self.populationSize],:] 106 | return newPopulation 107 | 108 | def spCrossover(self,population): 109 | # Single point crossover 110 | newPopulation = np.zeros(np.shape(population)) 111 | crossoverPoint = np.random.randint(0,self.stringLength,self.populationSize) 112 | for i in range(0,self.populationSize,2): 113 | newPopulation[i,:crossoverPoint[i]] = population[i,:crossoverPoint[i]] 114 | newPopulation[i+1,:crossoverPoint[i]] = population[i+1,:crossoverPoint[i]] 115 | newPopulation[i,crossoverPoint[i]:] = population[i+1,crossoverPoint[i]:] 116 | newPopulation[i+1,crossoverPoint[i]:] = population[i,crossoverPoint[i]:] 117 | return newPopulation 118 | 119 | def uniformCrossover(self,population): 120 | # Uniform crossover 121 | newPopulation = np.zeros(np.shape(population)) 122 | which = np.random.rand(self.populationSize,self.stringLength) 123 | which1 = which>=0.5 124 | for i in range(0,self.populationSize,2): 125 | newPopulation[i,:] = population[i,:]*which1[i,:] + population[i+1,:]*(1-which1[i,:]) 126 | newPopulation[i+1,:] = population[i,:]*(1-which1[i,:]) + population[i+1,:]*which1[i,:] 127 | return newPopulation 128 | 129 | def mutate(self,population): 130 | # Mutation 131 | whereMutate = np.random.rand(np.shape(population)[0],np.shape(population)[1]) 132 | population[np.where(whereMutate < self.mutationProb)] = 1 - population[np.where(whereMutate < self.mutationProb)] 133 | return population 134 | 135 | def elitism(self,oldPopulation,population,fitness): 136 | best = np.argsort(fitness) 137 | best = np.squeeze(oldPopulation[best[-self.nElite:],:]) 138 | indices = range(np.shape(population)[0]) 139 | np.random.shuffle(indices) 140 | population = population[indices,:] 141 | population[0:self.nElite,:] = best 142 | return population 143 | 144 | def tournament(self,oldPopulation,population,fitness,fitnessFunction): 145 | newFitness = eval(self.fitnessFunction)(population) 146 | for i in range(0,np.shape(population)[0],2): 147 | f = np.concatenate((fitness[i:i+2],newFitness[i:i+2]),axis=1) 148 | indices = np.argsort(f) 149 | if indices[-1]<2 and indices[-2]<2: 150 | population[i,:] = oldPopulation[i,:] 151 | population[i+1,:] = oldPopulation[i+1,:] 152 | elif indices[-1]<2: 153 | if indices[0]>=2: 154 | population[i+indices[0]-2,:] = oldPopulation[i+indices[-1]] 155 | else: 156 | population[i+indices[1]-2,:] = oldPopulation[i+indices[-1]] 157 | elif indices[-2]<2: 158 | if indices[0]>=2: 159 | population[i+indices[0]-2,:] = oldPopulation[i+indices[-2]] 160 | else: 161 | population[i+indices[1]-2,:] = oldPopulation[i+indices[-2]] 162 | return population 163 | 164 | -------------------------------------------------------------------------------- /Ch10/greedyKnapsack.py: -------------------------------------------------------------------------------- 1 | 2 | # Code from Chapter 10 of Machine Learning: An Algorithmic Perspective (2nd Edition) 3 | # by Stephen Marsland (http://stephenmonika.net) 4 | 5 | # You are free to use, change, or redistribute the code in any way you wish for 6 | # non-commercial purposes, but please maintain the name of the original author. 7 | # This code comes with no warranty of any kind. 8 | 9 | # Stephen Marsland, 2008, 2014 10 | 11 | # A greedy algorithm to solve the Knapsack problem 12 | import numpy as np 13 | 14 | def greedy(): 15 | maxSize = 500 16 | sizes = np.array([109.60,125.48,52.16,195.55,58.67,61.87,92.95,93.14,155.05,110.89,13.34,132.49,194.03,121.29,179.33,139.02,198.78,192.57,81.66,128.90]) 17 | 18 | sizes.sort() 19 | newSizes = sizes[-1:0:-1] 20 | space = maxSize 21 | 22 | while len(newSizes)>0 and space>newSizes[-1]: 23 | # Pick largest item that will fit 24 | item = np.where(space>newSizes)[0][0] 25 | print newSizes[item] 26 | space = space-newSizes[item] 27 | newSizes = np.concatenate((newSizes[:item],newSizes[item+1:])) 28 | print "Size = ",maxSize-space 29 | 30 | greedy() 31 | -------------------------------------------------------------------------------- /Ch10/knapsack.py: -------------------------------------------------------------------------------- 1 | 2 | # Code from Chapter 10 of Machine Learning: An Algorithmic Perspective (2nd Edition) 3 | # by Stephen Marsland (http://stephenmonika.net) 4 | 5 | # You are free to use, change, or redistribute the code in any way you wish for 6 | # non-commercial purposes, but please maintain the name of the original author. 7 | # This code comes with no warranty of any kind. 8 | 9 | # Stephen Marsland, 2008, 2014 10 | 11 | # A fitness function for the Knapsack problem 12 | import numpy as np 13 | 14 | def knapsack(pop): 15 | maxSize = 500 16 | #sizes = np.array([193.71,60.15,89.08,88.98,15.39,238.14,68.78,107.47,119.66,183.70]) 17 | 18 | sizes = np.array([109.60,125.48,52.16,195.55,58.67,61.87,92.95,93.14,155.05,110.89,13.34,132.49,194.03,121.29,179.33,139.02,198.78,192.57,81.66,128.90]) 19 | 20 | fitness = np.sum(sizes*pop,axis=1) 21 | fitness = np.where(fitness>maxSize,500-2*(fitness-maxSize),fitness) 22 | 23 | return fitness 24 | -------------------------------------------------------------------------------- /Ch10/onemax.py: -------------------------------------------------------------------------------- 1 | 2 | # Code from Chapter 10 of Machine Learning: An Algorithmic Perspective (2nd Edition) 3 | # by Stephen Marsland (http://stephenmonika.net) 4 | 5 | # You are free to use, change, or redistribute the code in any way you wish for 6 | # non-commercial purposes, but please maintain the name of the original author. 7 | # This code comes with no warranty of any kind. 8 | 9 | # Stephen Marsland, 2008, 2014 10 | 11 | # A fitness function for the onemax problem 12 | import numpy as np 13 | 14 | def onemax(pop): 15 | 16 | fitness = np.sum(pop,axis=1) 17 | 18 | return fitness 19 | -------------------------------------------------------------------------------- /Ch10/run_ga.py: -------------------------------------------------------------------------------- 1 | 2 | # Code from Chapter 10 of Machine Learning: An Algorithmic Perspective (2nd Edition) 3 | # by Stephen Marsland (http://stephenmonika.net) 4 | 5 | # You are free to use, change, or redistribute the code in any way you wish for 6 | # non-commercial purposes, but please maintain the name of the original author. 7 | # This code comes with no warranty of any kind. 8 | 9 | # Stephen Marsland, 2008, 2014 10 | 11 | # A runner for the Genetic Algorithm 12 | import ga 13 | import pylab as pl 14 | 15 | pl.ion() 16 | pl.show() 17 | 18 | plotfig = pl.figure() 19 | 20 | ga = ga.ga(30,'fF.fourpeaks',301,100,-1,'un',4,True) 21 | ga.runGA(plotfig) 22 | 23 | pl.pause(0) 24 | #pl.show() 25 | -------------------------------------------------------------------------------- /Ch11/SARSA.py: -------------------------------------------------------------------------------- 1 | 2 | # Code from Chapter 11 of Machine Learning: An Algorithmic Perspective (2nd Edition) 3 | # by Stephen Marsland (http://stephenmonika.net) 4 | 5 | # You are free to use, change, or redistribute the code in any way you wish for 6 | # non-commercial purposes, but please maintain the name of the original author. 7 | # This code comes with no warranty of any kind. 8 | 9 | # Stephen Marsland, 2008, 2014 10 | 11 | # The basic SARSA algorithm with the Europe example 12 | 13 | import numpy as np 14 | def SARSA(): 15 | 16 | R = np.array([[-5,0,-np.inf,-np.inf,-np.inf,-np.inf],[0,-5,0,0,-np.inf,-np.inf],[-np.inf,0,-5,0,-np.inf,100],[-np.inf,0,0,-5,0,-np.inf],[-np.inf,-np.inf,-np.inf,0,-5,100],[-np.inf,-np.inf,0,-np.inf,-np.inf,0]]) 17 | t = np.array([[1,1,0,0,0,0],[1,1,1,1,0,0],[0,1,1,1,0,1],[0,1,1,1,1,0],[0,0,0,1,1,1],[0,0,1,0,1,1]]) 18 | 19 | nStates = np.shape(R)[0] 20 | nActions = np.shape(R)[1] 21 | Q = np.random.rand(nStates,nActions)*0.1-0.05 22 | mu = 0.7 23 | gamma = 0.4 24 | epsilon = 0.1 25 | nits = 0 26 | 27 | while nits < 1000: 28 | # Pick initial state 29 | s = np.random.randint(nStates) 30 | # epsilon-greedy 31 | if (np.random.rand()0: 54 | t[i,j,k,0] = i-1 55 | t[i,j,k,1] = j 56 | else: 57 | t[i,j,k,0] = i 58 | t[i,j,k,1] = j 59 | 60 | if i==1 and 1<=j<=5: 61 | t[i,j,k,0] = 0 62 | t[i,j,k,1] = 0 63 | else: 64 | if j>0: 65 | t[i,j,k,0] = i 66 | t[i,j,k,1] = j-1 67 | else: 68 | t[i,j,k,0] = i 69 | t[i,j,k,1] = j 70 | if i==0 and j==6: 71 | t[i,j,k,0] = 0 72 | t[i,j,k,1] = 0 73 | 74 | #print t[:,:,3,0] ,t[:,:,3,1] 75 | 76 | Q = np.random.random_sample(np.shape(R))*0.1-0.05 77 | mu = 0.7 78 | gamma = 0.4 79 | epsilon = 0.05 80 | nits = 0 81 | 82 | while nits < 1000: 83 | # Pick initial state 84 | s = np.array([0,0]) #np.array([np.random.randint(4),np.random.randint(7)]) 85 | 86 | r=-np.inf 87 | while r==-np.inf: 88 | # epsilon-greedy 89 | if (np.random.rand()0: 53 | t[i,j,k,0] = i-1 54 | t[i,j,k,1] = j 55 | else: 56 | t[i,j,k,0] = i 57 | t[i,j,k,1] = j 58 | 59 | if i==1 and 1<=j<=5: 60 | t[i,j,k,0] = 0 61 | t[i,j,k,1] = 0 62 | else: 63 | if j>0: 64 | t[i,j,k,0] = i 65 | t[i,j,k,1] = j-1 66 | else: 67 | t[i,j,k,0] = i 68 | t[i,j,k,1] = j 69 | if i==0 and j==6: 70 | t[i,j,k,0] = 0 71 | t[i,j,k,1] = 0 72 | 73 | #print t[:,:,3,0] ,t[:,:,3,1] 74 | 75 | #Q = np.random.random_sample(np.shape(R))*0.1-0.05 76 | Q = np.zeros(np.shape(R)) 77 | mu = 0.7 78 | gamma = 0.4 79 | epsilon = 0.05 80 | nits = 0 81 | 82 | while nits < 1000: 83 | # Pick initial state 84 | s = np.array([0,0]) #np.array([np.random.randint(4),np.random.randint(7)]) 85 | 86 | #print s, np.shape(s) 87 | #print np.shape(Q), np.shape(Q[s[0],s[1],:]) 88 | inEpisode = 1 89 | # Stop when the accepting state is reached 90 | while inEpisode: 91 | r=-np.inf 92 | while r==-np.inf: 93 | # epsilon-greedy 94 | if (np.random.rand()0: 62 | for each in out: 63 | frequency[index] = outputs.count(each) 64 | index += 1 65 | decision.append(out[frequency.argmax()]) 66 | else: 67 | decision.append(None) 68 | return decision 69 | -------------------------------------------------------------------------------- /Ch13/car.py: -------------------------------------------------------------------------------- 1 | 2 | # Code from Chapter 13 of Machine Learning: An Algorithmic Perspective (2nd Edition) 3 | # by Stephen Marsland (http://stephenmonika.net) 4 | 5 | # You are free to use, change, or redistribute the code in any way you wish for 6 | # non-commercial purposes, but please maintain the name of the original author. 7 | # This code comes with no warranty of any kind. 8 | 9 | # Stephen Marsland, 2008, 2014 10 | 11 | # An example of bagging on the Car Safety dataset 12 | import numpy as np 13 | import dtree 14 | import bagging 15 | import randomforest 16 | 17 | tree = dtree.dtree() 18 | bagger = bagging.bagger() 19 | forest = randomforest.randomforest() 20 | 21 | data,classes,features = tree.read_data('car.data') 22 | 23 | train = data[::2][:] 24 | test = data[1::2][:] 25 | trainc = classes[::2] 26 | testc = classes[1::2] 27 | 28 | t=tree.make_tree(train,trainc,features) 29 | out = tree.classifyAll(t,test) 30 | tree.printTree(t,' ') 31 | 32 | a = np.zeros(len(out)) 33 | b = np.zeros(len(out)) 34 | d = np.zeros(len(out)) 35 | 36 | for i in range(len(out)): 37 | if testc[i] == 'good' or testc[i]== 'v-good': 38 | b[i] = 1 39 | if out[i] == testc[i]: 40 | d[i] = 1 41 | if out[i] == testc[i]: 42 | a[i] = 1 43 | 44 | print "Tree" 45 | print "Number correctly predicted",np.sum(a) 46 | print "Number of testpoints ",len(a) 47 | print "Percentage Accuracy ",np.sum(a)/len(a)*100.0 48 | print "" 49 | print "Number of cars rated as good or very good", np.sum(b) 50 | print "Number correctly identified as good or very good",np.sum(d) 51 | print "Percentage Accuracy",np.sum(d)/np.sum(b)*100.0 52 | 53 | c=bagger.bag(train,trainc,features,100) 54 | out = bagger.bagclass(c,test) 55 | 56 | a = np.zeros(len(out)) 57 | b = np.zeros(len(out)) 58 | d = np.zeros(len(out)) 59 | 60 | for i in range(len(out)): 61 | if testc[i] == 'good' or testc[i]== 'v-good': 62 | b[i] = 1 63 | if out[i] == testc[i]: 64 | d[i] = 1 65 | if out[i] == testc[i]: 66 | a[i] = 1 67 | print "-----" 68 | print "Bagger" 69 | print "Number correctly predicted",np.sum(a) 70 | print "Number of testpoints ",len(a) 71 | print "Percentage Accuracy ",np.sum(a)/len(a)*100.0 72 | print "" 73 | print "Number of cars rated as good or very good", np.sum(b) 74 | print "Number correctly identified as good or very good",np.sum(d) 75 | print "Percentage Accuracy",np.sum(d)/np.sum(b)*100.0 76 | 77 | f=f = forest.rf(train,trainc,features,100,200,2) 78 | out = forest.rfclass(f,test) 79 | 80 | a = np.zeros(len(out)) 81 | b = np.zeros(len(out)) 82 | d = np.zeros(len(out)) 83 | 84 | for i in range(len(out)): 85 | if testc[i] == 'good' or testc[i]== 'v-good': 86 | b[i] = 1 87 | if out[i] == testc[i]: 88 | d[i] = 1 89 | if out[i] == testc[i]: 90 | a[i] = 1 91 | print "-----" 92 | print "Forest" 93 | print "Number correctly predicted",np.sum(a) 94 | print "Number of testpoints ",len(a) 95 | print "Percentage Accuracy ",np.sum(a)/len(a)*100.0 96 | print "" 97 | print "Number of cars rated as good or very good", np.sum(b) 98 | print "Number correctly identified as good or very good",np.sum(d) 99 | print "Percentage Accuracy",np.sum(d)/np.sum(b)*100.0 100 | -------------------------------------------------------------------------------- /Ch13/car1.py: -------------------------------------------------------------------------------- 1 | 2 | # Code from Chapter 13 of Machine Learning: An Algorithmic Perspective (2nd Edition) 3 | # by Stephen Marsland (http://stephenmonika.net) 4 | 5 | # You are free to use, change, or redistribute the code in any way you wish for 6 | # non-commercial purposes, but please maintain the name of the original author. 7 | # This code comes with no warranty of any kind. 8 | 9 | # Stephen Marsland, 2008, 2014 10 | 11 | import dtree 12 | import randomforest 13 | tree = dtree.dtree() 14 | forest = randomforest.randomforest() 15 | data,classes,features = tree.read_data('car.data') 16 | train = data[::2][:] 17 | test = data[1::2][:] 18 | trainc = classes[::2] 19 | testc = classes[1::2] 20 | f=f = forest.rf(train,trainc,features,50,100,2,maxlevel=3) 21 | #f=f = forest.rf(train,trainc,features,100,200,2) 22 | out = forest.rfclass(f,test) 23 | 24 | import numpy as np 25 | 26 | a = np.zeros(len(out)) 27 | b = np.zeros(len(out)) 28 | d = np.zeros(len(out)) 29 | 30 | for i in range(len(out)): 31 | if testc[i] == 'good' or testc[i]== 'v-good': 32 | b[i] = 1 33 | if out[i] == testc[i]: 34 | d[i] = 1 35 | if out[i] == testc[i]: 36 | a[i] = 1 37 | print "-----" 38 | print "Forest" 39 | print "Number correctly predicted",np.sum(a) 40 | print "Number of testpoints ",len(a) 41 | print "Percentage Accuracy ",np.sum(a)/len(a)*100.0 42 | print "" 43 | print "Number of cars rated as good or very good", np.sum(b) 44 | print "Number correctly identified as good or very good",np.sum(d) 45 | print "Percentage Accuracy",np.sum(d)/np.sum(b)*100.0 46 | -------------------------------------------------------------------------------- /Ch13/mushroom.py: -------------------------------------------------------------------------------- 1 | 2 | # Code from Chapter 13 of Machine Learning: An Algorithmic Perspective (2nd Edition) 3 | # by Stephen Marsland (http://stephenmonika.net) 4 | 5 | # You are free to use, change, or redistribute the code in any way you wish for 6 | # non-commercial purposes, but please maintain the name of the original author. 7 | # This code comes with no warranty of any kind. 8 | 9 | # Stephen Marsland, 2008, 2014 10 | 11 | # Comparison of stumping and bagging on the mushroom dataset 12 | import numpy as np 13 | import dtw 14 | import bagging 15 | import randomforest 16 | 17 | tree = dtw.dtree() 18 | bagger = bagging.bagger() 19 | forest = randomforest.randomforest() 20 | mushroom,classes,features = tree.read_data('agaricus-lepiota.data') 21 | 22 | w = np.ones((np.shape(mushroom)[0]),dtype = float)/np.shape(mushroom)[0] 23 | 24 | f = forest.rf(mushroom,classes,features,10,7,2) 25 | print forest.rfclass(f,mushroom) 26 | 27 | t=tree.make_tree(mushroom,w,classes,features,1) 28 | tree.printTree(t,' ') 29 | 30 | print "Tree Stump Prediction" 31 | print tree.classifyAll(t,mushroom) 32 | print "True Classes" 33 | print classes 34 | 35 | c=bagger.bag(mushroom,classes,features,20) 36 | print "Bagged Results" 37 | print bagger.bagclass(c,mushroom) 38 | -------------------------------------------------------------------------------- /Ch13/party.py: -------------------------------------------------------------------------------- 1 | 2 | # Code from Chapter 13 of Machine Learning: An Algorithmic Perspective (2nd Edition) 3 | # by Stephen Marsland (http://stephenmonika.net) 4 | 5 | # You are free to use, change, or redistribute the code in any way you wish for 6 | # non-commercial purposes, but please maintain the name of the original author. 7 | # This code comes with no warranty of any kind. 8 | 9 | # Stephen Marsland, 2008, 2014 10 | 11 | # Comparison of stumping and bagging on the Party dataset 12 | import numpy as np 13 | #import dtree 14 | import dtw 15 | import bagging 16 | import randomforest 17 | 18 | tree = dtw.dtree() 19 | #tree = dtree.dtree() 20 | bagger = bagging.bagger() 21 | forest = randomforest.randomforest() 22 | party,classes,features = tree.read_data('../6 Trees/party.data') 23 | 24 | #w = np.random.rand((np.shape(party)[0]))/np.shape(party)[0] 25 | w = np.ones((np.shape(party)[0]),dtype = float)/np.shape(party)[0] 26 | 27 | f = forest.rf(party,classes,features,10,7,2,maxlevel=2) 28 | print "RF prediction" 29 | print forest.rfclass(f,party) 30 | 31 | #t=tree.make_tree(party,classes,features) 32 | t=tree.make_tree(party,w,classes,features) 33 | #tree.printTree(t,' ') 34 | print "Decision Tree prediction" 35 | print tree.classifyAll(t,party) 36 | 37 | print "Tree Stump Prediction" 38 | print tree.classifyAll(t,party) 39 | 40 | c=bagger.bag(party,classes,features,20) 41 | print "Bagged Results" 42 | print bagger.bagclass(c,party) 43 | 44 | print "True Classes" 45 | print classes 46 | -------------------------------------------------------------------------------- /Ch13/randomforest.py: -------------------------------------------------------------------------------- 1 | 2 | # Code from Chapter 13 of Machine Learning: An Algorithmic Perspective (2nd Edition) 3 | # by Stephen Marsland (http://stephenmonika.net) 4 | 5 | # You are free to use, change, or redistribute the code in any way you wish for 6 | # non-commercial purposes, but please maintain the name of the original author. 7 | # This code comes with no warranty of any kind. 8 | 9 | # Stephen Marsland, 2014 10 | 11 | import numpy as np 12 | import dtree 13 | 14 | class randomforest: 15 | 16 | """The random forest algorithm based on the decision tree of Chapter 6""" 17 | def __init__(self): 18 | """ Constructor """ 19 | self.tree = dtree.dtree() 20 | 21 | 22 | def rf(self,data,targets,features,nTrees,nSamples,nFeatures,maxlevel=5): 23 | 24 | nPoints = np.shape(data)[0] 25 | nDim = np.shape(data)[1] 26 | self.nSamples = nSamples 27 | self.nTrees = nTrees 28 | 29 | classifiers = [] 30 | 31 | for i in range(nTrees): 32 | print i 33 | # Compute bootstrap samples 34 | samplePoints = np.random.randint(0,nPoints,(nPoints,nSamples)) 35 | 36 | for j in range(nSamples): 37 | sample = [] 38 | sampleTarget = [] 39 | for k in range(nPoints): 40 | sample.append(data[samplePoints[k,j]]) 41 | sampleTarget.append(targets[samplePoints[k,j]]) 42 | # Train classifiers 43 | classifiers.append(self.tree.make_tree(sample,sampleTarget,features,maxlevel,forest=nFeatures)) 44 | return classifiers 45 | 46 | def rfclass(self,classifiers,data): 47 | 48 | decision = [] 49 | # Majority voting 50 | for j in range(len(data)): 51 | outputs = [] 52 | #print data[j] 53 | for i in range(self.nTrees): 54 | out = self.tree.classify(classifiers[i],data[j]) 55 | if out is not None: 56 | outputs.append(out) 57 | # List the possible outputs 58 | out = [] 59 | for each in outputs: 60 | if out.count(each)==0: 61 | out.append(each) 62 | frequency = np.zeros(len(out)) 63 | 64 | index = 0 65 | if len(out)>0: 66 | for each in out: 67 | frequency[index] = outputs.count(each) 68 | index += 1 69 | decision.append(out[frequency.argmax()]) 70 | else: 71 | decision.append(None) 72 | return decision 73 | -------------------------------------------------------------------------------- /Ch14/iris.py: -------------------------------------------------------------------------------- 1 | 2 | # Code from Chapter 14 of Machine Learning: An Algorithmic Perspective (2nd Edition) 3 | # by Stephen Marsland (http://stephenmonika.net) 4 | 5 | # You are free to use, change, or redistribute the code in any way you wish for 6 | # non-commercial purposes, but please maintain the name of the original author. 7 | # This code comes with no warranty of any kind. 8 | 9 | # Stephen Marsland, 2008, 2014 10 | 11 | # Examples of using the k-means and SOM algorithms on the Iris dataset 12 | 13 | import pylab as pl 14 | import numpy as np 15 | 16 | iris = np.loadtxt('../3 MLP/iris_proc.data',delimiter=',') 17 | iris[:,:4] = iris[:,:4]-iris[:,:4].mean(axis=0) 18 | imax = np.concatenate((iris.max(axis=0)*np.ones((1,5)),iris.min(axis=0)*np.ones((1,5))),axis=0).max(axis=0) 19 | iris[:,:4] = iris[:,:4]/imax[:4] 20 | 21 | target = iris[:,4] 22 | 23 | order = range(np.shape(iris)[0]) 24 | np.random.shuffle(order) 25 | iris = iris[order,:] 26 | target = target[order] 27 | 28 | train = iris[::2,0:4] 29 | traint = target[::2] 30 | valid = iris[1::4,0:4] 31 | validt = target[1::4] 32 | test = iris[3::4,0:4] 33 | testt = target[3::4] 34 | 35 | #print train.max(axis=0), train.min(axis=0) 36 | 37 | import kmeansnet 38 | #import kmeans as kmeansnet 39 | net = kmeansnet.kmeans(3,train) 40 | net.kmeanstrain(train) 41 | cluster = net.kmeansfwd(test) 42 | print 1.*cluster 43 | print iris[3::4,4] 44 | 45 | import som 46 | net = som.som(6,6,train) 47 | net.somtrain(train,400) 48 | 49 | best = np.zeros(np.shape(train)[0],dtype=int) 50 | for i in range(np.shape(train)[0]): 51 | best[i],activation = net.somfwd(train[i,:]) 52 | 53 | pl.plot(net.map[0,:],net.map[1,:],'k.',ms=15) 54 | where = pl.find(traint == 0) 55 | pl.plot(net.map[0,best[where]],net.map[1,best[where]],'rs',ms=30) 56 | where = pl.find(traint == 1) 57 | pl.plot(net.map[0,best[where]],net.map[1,best[where]],'gv',ms=30) 58 | where = pl.find(traint == 2) 59 | pl.plot(net.map[0,best[where]],net.map[1,best[where]],'b^',ms=30) 60 | pl.axis([-0.1,1.1,-0.1,1.1]) 61 | pl.axis('off') 62 | pl.figure(2) 63 | 64 | best = np.zeros(np.shape(test)[0],dtype=int) 65 | for i in range(np.shape(test)[0]): 66 | best[i],activation = net.somfwd(test[i,:]) 67 | 68 | pl.plot(net.map[0,:],net.map[1,:],'k.',ms=15) 69 | where = pl.find(testt == 0) 70 | pl.plot(net.map[0,best[where]],net.map[1,best[where]],'rs',ms=30) 71 | where = pl.find(testt == 1) 72 | pl.plot(net.map[0,best[where]],net.map[1,best[where]],'gv',ms=30) 73 | where = pl.find(testt == 2) 74 | pl.plot(net.map[0,best[where]],net.map[1,best[where]],'b^',ms=30) 75 | pl.axis([-0.1,1.1,-0.1,1.1]) 76 | pl.axis('off') 77 | pl.show() 78 | -------------------------------------------------------------------------------- /Ch14/iris_somperc.py: -------------------------------------------------------------------------------- 1 | 2 | # Code from Chapter 14 of Machine Learning: An Algorithmic Perspective (2nd Edition) 3 | # by Stephen Marsland (http://stephenmonika.net) 4 | 5 | # You are free to use, change, or redistribute the code in any way you wish for 6 | # non-commercial purposes, but please maintain the name of the original author. 7 | # This code comes with no warranty of any kind. 8 | 9 | # Stephen Marsland, 2008, 2014 10 | 11 | # Examples of using the k-means and SOM algorithms on the Iris dataset 12 | 13 | import pylab as pl 14 | import numpy as np 15 | 16 | iris = np.loadtxt('../3 MLP/iris_proc.data',delimiter=',') 17 | iris[:,:4] = iris[:,:4]-iris[:,:4].mean(axis=0) 18 | imax = np.concatenate((iris.max(axis=0)*np.ones((1,5)),iris.min(axis=0)*np.ones((1,5))),axis=0).max(axis=0) 19 | iris[:,:4] = iris[:,:4]/imax[:4] 20 | 21 | target = iris[:,4] 22 | 23 | order = range(np.shape(iris)[0]) 24 | np.random.shuffle(order) 25 | iris = iris[order,:] 26 | target = target[order] 27 | 28 | train = iris[::2,0:4] 29 | traint = target[::2] 30 | valid = iris[1::4,0:4] 31 | validt = target[1::4] 32 | test = iris[3::4,0:4] 33 | testt = target[3::4] 34 | 35 | #print train.max(axis=0), train.min(axis=0) 36 | 37 | import som 38 | net = som.som(6,6,train) 39 | net.somtrain(train,400) 40 | 41 | best = np.zeros(np.shape(train)[0],dtype=int) 42 | for i in range(np.shape(train)[0]): 43 | best[i],activation = net.somfwd(train[i,:]) 44 | 45 | pl.plot(net.map[0,:],net.map[1,:],'k.',ms=15) 46 | where = pl.find(traint == 0) 47 | pl.plot(net.map[0,best[where]],net.map[1,best[where]],'rs',ms=30) 48 | where = pl.find(traint == 1) 49 | pl.plot(net.map[0,best[where]],net.map[1,best[where]],'gv',ms=30) 50 | where = pl.find(traint == 2) 51 | pl.plot(net.map[0,best[where]],net.map[1,best[where]],'b^',ms=30) 52 | pl.axis([-0.1,1.1,-0.1,1.1]) 53 | pl.axis('off') 54 | pl.figure(2) 55 | 56 | best = np.zeros(np.shape(test)[0],dtype=int) 57 | for i in range(np.shape(test)[0]): 58 | best[i],activation = net.somfwd(test[i,:]) 59 | 60 | pl.plot(net.map[0,:],net.map[1,:],'k.',ms=15) 61 | where = pl.find(testt == 0) 62 | pl.plot(net.map[0,best[where]],net.map[1,best[where]],'rs',ms=30) 63 | where = pl.find(testt == 1) 64 | pl.plot(net.map[0,best[where]],net.map[1,best[where]],'gv',ms=30) 65 | where = pl.find(testt == 2) 66 | pl.plot(net.map[0,best[where]],net.map[1,best[where]],'b^',ms=30) 67 | pl.axis([-0.1,1.1,-0.1,1.1]) 68 | pl.axis('off') 69 | pl.show() 70 | -------------------------------------------------------------------------------- /Ch14/kmeans.py: -------------------------------------------------------------------------------- 1 | 2 | # Code from Chapter 14 of Machine Learning: An Algorithmic Perspective (2nd Edition) 3 | # by Stephen Marsland (http://stephenmonika.net) 4 | 5 | # You are free to use, change, or redistribute the code in any way you wish for 6 | # non-commercial purposes, but please maintain the name of the original author. 7 | # This code comes with no warranty of any kind. 8 | 9 | # Stephen Marsland, 2008, 2014 10 | 11 | import numpy as np 12 | 13 | class kmeans: 14 | """ The k-Means algorithm""" 15 | def __init__(self,k,data): 16 | 17 | self.nData = np.shape(data)[0] 18 | self.nDim = np.shape(data)[1] 19 | self.k = k 20 | 21 | def kmeanstrain(self,data,maxIterations=10): 22 | 23 | # Find the minimum and maximum values for each feature 24 | minima = data.min(axis=0) 25 | maxima = data.max(axis=0) 26 | 27 | # Pick the centre locations randomly 28 | self.centres = np.random.rand(self.k,self.nDim)*(maxima-minima)+minima 29 | oldCentres = np.random.rand(self.k,self.nDim)*(maxima-minima)+minima 30 | 31 | count = 0 32 | #print centres 33 | while np.sum(np.sum(oldCentres-self.centres))!= 0 and count0: 51 | self.centres[j,:] = np.sum(data*thisCluster,axis=0)/np.sum(thisCluster) 52 | #plot(data[:,0],data[:,1],'kx') 53 | #plot(centres[:,0],centres[:,1],'ro') 54 | return self.centres 55 | 56 | def kmeansfwd(self,data): 57 | 58 | nData = np.shape(data)[0] 59 | # Compute distances 60 | distances = np.ones((1,nData))*np.sum((data-self.centres[0,:])**2,axis=1) 61 | for j in range(self.k-1): 62 | distances = np.append(distances,np.ones((1,nData))*np.sum((data-self.centres[j+1,:])**2,axis=1),axis=0) 63 | 64 | # Identify the closest cluster 65 | cluster = distances.argmin(axis=0) 66 | cluster = np.transpose(cluster*np.ones((1,nData))) 67 | 68 | return cluster 69 | -------------------------------------------------------------------------------- /Ch14/kmeansnet.py: -------------------------------------------------------------------------------- 1 | 2 | # Code from Chapter 14 of Machine Learning: An Algorithmic Perspective (2nd Edition) 3 | # by Stephen Marsland (http://stephenmonika.net) 4 | 5 | # You are free to use, change, or redistribute the code in any way you wish for 6 | # non-commercial purposes, but please maintain the name of the original author. 7 | # This code comes with no warranty of any kind. 8 | 9 | # Stephen Marsland, 2008, 2014 10 | 11 | import numpy as np 12 | 13 | class kmeans: 14 | """The k-Means Algorithm implemented as a neural network""" 15 | def __init__(self,k,data,nEpochs=1000,eta=0.25): 16 | 17 | self.nData = np.shape(data)[0] 18 | self.nDim = np.shape(data)[1] 19 | self.k = k 20 | self.nEpochs = nEpochs 21 | self.weights = np.random.rand(self.nDim,self.k) 22 | self.eta = eta 23 | 24 | def kmeanstrain(self,data): 25 | # Preprocess data (won't work if (0,0,...0) is in data) 26 | normalisers = np.sqrt(np.sum(data**2,axis=1))*np.ones((1,np.shape(data)[0])) 27 | data = np.transpose(np.transpose(data)/normalisers) 28 | 29 | for i in range(self.nEpochs): 30 | for j in range(self.nData): 31 | activation = np.sum(self.weights*np.transpose(data[j:j+1,:]),axis=0) 32 | winner = np.argmax(activation) 33 | self.weights[:,winner] += self.eta * data[j,:] - self.weights[:,winner] 34 | 35 | def kmeansfwd(self,data): 36 | best = np.zeros(np.shape(data)[0]) 37 | for i in range(np.shape(data)[0]): 38 | activation = np.sum(self.weights*np.transpose(data[i:i+1,:]),axis=0) 39 | best[i] = np.argmax(activation) 40 | return best 41 | 42 | -------------------------------------------------------------------------------- /Ch14/moredemos.py: -------------------------------------------------------------------------------- 1 | 2 | # Code from Chapter 14 of Machine Learning: An Algorithmic Perspective (2nd Edition) 3 | # by Stephen Marsland (http://stephenmonika.net) 4 | 5 | # You are free to use, change, or redistribute the code in any way you wish for 6 | # non-commercial purposes, but please maintain the name of the original author. 7 | # This code comes with no warranty of any kind. 8 | 9 | # Stephen Marsland, 2008, 2014 10 | 11 | # Demonstration of the SOM algorithm on the Wine dataset (and the e-coli dataset) 12 | import pylab as pl 13 | import numpy as np 14 | import som 15 | 16 | wine = np.loadtxt('wine.data',delimiter=',') 17 | 18 | classes = wine[:,0] 19 | data = wine[:,1:] 20 | data -= np.mean(data,axis=0) 21 | data /= data.max(axis=0) 22 | 23 | #ecoli = loadtxt('shortecoli.dat') 24 | #classes = ecoli[:,7:] 25 | #data = ecoli[:,:7] 26 | #data -= mean(data,axis=0) 27 | #data /= data.max(axis=0) 28 | 29 | order = range(shape(data)[0]) 30 | np.random.shuffle(order) 31 | split = int(np.round(np.shape(data)[0]/2)) 32 | train = data[order[:split],:] 33 | target = classes[order[:split],:] 34 | 35 | test = data[order[split:],:] 36 | ttarget = classes[order[:split],:] 37 | 38 | net = som.som(15,15,train,eta_b=0.3,eta_n=0.1,nSize=0.5,alpha=1,usePCA=1,useBCs=1,eta_bfinal=0.03,eta_nfinal=0.01,nSizefinal=0.05) 39 | net.somtrain(train,12000) 40 | 41 | best = np.zeros(shape(test)[0],dtype=int) 42 | 43 | for i in range(shape(test)[0]): 44 | best[i],activation = net.somfwd(train[i,:]) 45 | 46 | #print best 47 | #print ttarget 48 | 49 | pl.plot(net.map[0,:],net.map[1,:],'k.',ms=15) 50 | where = pl.find(target == 0) 51 | pl.plot(net.map[0,best[where]],net.map[1,best[where]],'rs',ms=30) 52 | where = pl.find(target == 1) 53 | pl.plot(net.map[0,best[where]],net.map[1,best[where]],'gv',ms=30) 54 | where = pl.find(target == 2) 55 | pl.plot(net.map[0,best[where]],net.map[1,best[where]],'b^',ms=30) 56 | pl.axis([-0.1,1.1,-0.1,1.1]) 57 | pl.axis('off') 58 | 59 | pl.figure(2) 60 | best = np.zeros(shape(test)[0],dtype=int) 61 | 62 | for i in range(shape(test)[0]): 63 | best[i],activation = net.somfwd(test[i,:]) 64 | 65 | pl.plot(net.map[0,:],net.map[1,:],'k.',ms=15) 66 | where = pl.find(ttarget == 0) 67 | pl.plot(net.map[0,best[where]],net.map[1,best[where]],'rs',ms=30) 68 | where = pl.find(ttarget == 1) 69 | pl.plot(net.map[0,best[where]],net.map[1,best[where]],'gv',ms=30) 70 | where = pl.find(ttarget == 2) 71 | pl.plot(net.map[0,best[where]],net.map[1,best[where]],'b^',ms=30) 72 | pl.axis([-0.1,1.1,-0.1,1.1]) 73 | pl.axis('off') 74 | 75 | pl.show() 76 | -------------------------------------------------------------------------------- /Ch14/som.py: -------------------------------------------------------------------------------- 1 | 2 | # Code from Chapter 14 of Machine Learning: An Algorithmic Perspective (2nd Edition) 3 | # by Stephen Marsland (http://stephenmonika.net) 4 | 5 | # You are free to use, change, or redistribute the code in any way you wish for 6 | # non-commercial purposes, but please maintain the name of the original author. 7 | # This code comes with no warranty of any kind. 8 | 9 | # Stephen Marsland, 2008, 2014 10 | 11 | import numpy as np 12 | import pca 13 | 14 | class som: 15 | """A Basic 2D Self-Organising Map 16 | The map connections can be initialised randomly or with PCA""" 17 | def __init__(self,x,y,inputs,eta_b=0.3,eta_n=0.1,nSize=0.5,alpha=1,usePCA=1,useBCs=0,eta_bfinal=0.03,eta_nfinal=0.01,nSizefinal=0.05): 18 | self.nData = np.shape(inputs)[0] 19 | self.nDim = np.shape(inputs)[1] 20 | self.mapDim = 2 21 | 22 | self.x = x 23 | self.y = y 24 | self.eta_b = eta_b 25 | self.eta_bfinal = eta_bfinal 26 | self.eta_n = eta_n 27 | self.eta_nfinal = eta_nfinal 28 | self.nSize = nSize 29 | self.nSizefinal = nSizefinal 30 | self.alpha = alpha 31 | 32 | self.map = np.mgrid[0:1:np.complex(0,x),0:1:np.complex(0,y)] 33 | self.map = np.reshape(self.map,(2,x*y)) 34 | 35 | if usePCA: 36 | dummy1,dummy2,evals,evecs = pca.pca(inputs,2) 37 | self.weights = np.zeros((self.nDim,x*y)) 38 | for i in range(x*y): 39 | for j in range(self.mapDim): 40 | self.weights[:,i] += (self.map[j,i]-0.5)*2*evecs[:,j] 41 | else: 42 | self.weights = (np.random.rand(self.nDim,x*y)-0.5)*2 43 | 44 | self.mapDist = np.zeros((self.x*self.y,self.x*self.y)) 45 | if useBCs: 46 | for i in range(self.x*self.y): 47 | for j in range(i+1,self.x*self.y): 48 | xdist = np.min((self.map[0,i]-self.map[0,j])**2,(self.map[0,i]+1+1./self.x-self.map[0,j])**2,(self.map[0,i]-1-1./self.x-self.map[0,j])**2,(self.map[0,i]-self.map[0,j]+1+1./self.x)**2,(self.map[0,i]-self.map[0,j]-1-1./self.x)**2) 49 | ydist = np.min((self.map[1,i]-self.map[1,j])**2,(self.map[1,i]+1+1./self.y-self.map[1,j])**2,(self.map[1,i]-1-1./self.y-self.map[1,j])**2,(self.map[1,i]-self.map[1,j]+1+1./self.y)**2,(self.map[1,i]-self.map[1,j]-1-1./self.y)**2) 50 | self.mapDist[i,j] = np.sqrt(xdist+ydist) 51 | self.mapDist[j,i] = self.mapDist[i,j] 52 | else: 53 | for i in range(self.x*self.y): 54 | for j in range(i+1,self.x*self.y): 55 | self.mapDist[i,j] = np.sqrt((self.map[0,i] - self.map[0,j])**2 + (self.map[1,i] - self.map[1,j])**2) 56 | self.mapDist[j,i] = self.mapDist[i,j] 57 | 58 | def somtrain(self,inputs,nIterations): 59 | self.eta_binit = self.eta_b 60 | self.eta_ninit = self.eta_n 61 | self.nSizeinit = self.nSize 62 | 63 | for iterations in range(nIterations): 64 | for i in range(self.nData): 65 | #print inputs[i,:] 66 | best,activation = self.somfwd(inputs[i,:]) 67 | # Update the weights of the best match 68 | self.weights[:,best] += self.eta_b * (inputs[i,:] - self.weights[:,best]) 69 | #print self.weights 70 | # Find the neighbours and update their weights 71 | neighbours = np.where(self.mapDist[best,:]<=self.nSize,1,0) 72 | neighbours[best] = 0 73 | #print neighbours 74 | self.weights += self.eta_n * neighbours*np.transpose((inputs[i,:] - np.transpose(self.weights))) 75 | #print self.weights 76 | # Modify learning rates 77 | self.eta_b = self.eta_binit*np.power(self.eta_bfinal/self.eta_binit,float(iterations)/nIterations) 78 | self.eta_n = self.eta_ninit*np.power(self.eta_nfinal/self.eta_ninit,float(iterations)/nIterations) 79 | 80 | # Modify neighbourhood size 81 | self.nSize = self.nSizeinit*np.power(self.nSizefinal/self.nSizeinit,float(iterations)/nIterations) 82 | 83 | def somfwd(self,inputs): 84 | activations = np.sum((np.transpose(np.tile(inputs,(self.x*self.y,1)))-self.weights)**2,axis=0) 85 | best = np.argmin(activations) 86 | return best,activations 87 | -------------------------------------------------------------------------------- /Ch14/somdemo.py: -------------------------------------------------------------------------------- 1 | 2 | # Code from Chapter 14 of Machine Learning: An Algorithmic Perspective (2nd Edition) 3 | # by Stephen Marsland (http://stephenmonika.net) 4 | 5 | # You are free to use, change, or redistribute the code in any way you wish for 6 | # non-commercial purposes, but please maintain the name of the original author. 7 | # This code comes with no warranty of any kind. 8 | 9 | # Stephen Marsland, 2008, 2014 10 | 11 | # A simple example of using the SOM on a 2D dataset showing the neighbourhood connections 12 | 13 | import pylab as pl 14 | import numpy as np 15 | 16 | import som 17 | nNodesEdge = 8 18 | data = (np.random.rand(2000,2)-0.5)*2 19 | 20 | # Set up the network and decide on parameters 21 | net = som.som(nNodesEdge,nNodesEdge,data,usePCA=0) 22 | step = 0.2 23 | 24 | pl.figure(1) 25 | pl.plot(data[:,0],data[:,1],'.') 26 | # Train the network for 0 iterations (to get the position of the nodes) 27 | net.somtrain(data,0) 28 | for i in range(net.x*net.y): 29 | neighbours = np.where(net.mapDist[i,:]<=step) 30 | 31 | t = np.zeros((np.shape(neighbours)[1]*2,np.shape(net.weights)[0])) 32 | t[::2,:] = np.tile(net.weights[:,i],(np.shape(neighbours)[1],1)) 33 | t[1::2,:] = np.transpose(net.weights[:,neighbours[0][:]]) 34 | pl.plot(t[:,0],t[:,1],'g-') 35 | pl.axis('off') 36 | 37 | pl.figure(2) 38 | pl.plot(data[:,0],data[:,1],'.') 39 | net.somtrain(data,5) 40 | for i in range(net.x*net.y): 41 | neighbours = np.where(net.mapDist[i,:]<=step) 42 | 43 | t = np.zeros((np.shape(neighbours)[1]*2,np.shape(net.weights)[0])) 44 | t[::2,:] = np.tile(net.weights[:,i],(np.shape(neighbours)[1],1)) 45 | t[1::2,:] = np.transpose(net.weights[:,neighbours[0][:]]) 46 | pl.plot(t[:,0],t[:,1],'g-') 47 | pl.axis([-1,1,-1,1]) 48 | pl.axis('off') 49 | 50 | net.somtrain(data,100) 51 | pl.figure(3) 52 | pl.plot(data[:,0],data[:,1],'.') 53 | for i in range(net.x*net.y): 54 | neighbours = np.where(net.mapDist[i,:]<=step) 55 | #print neighbours 56 | #n = tile(net.weights[:,i],(shape(neighbours)[1],1)) 57 | t = np.zeros((np.shape(neighbours)[1]*2,np.shape(net.weights)[0])) 58 | t[::2,:] = np.tile(net.weights[:,i],(np.shape(neighbours)[1],1)) 59 | t[1::2,:] = np.transpose(net.weights[:,neighbours[0][:]]) 60 | pl.plot(t[:,0],t[:,1],'g-') 61 | 62 | #net.somtrain(data,100) 63 | #pl.figure(4) 64 | #pl.plot(data[:,0],data[:,1],'.') 65 | #for i in range(net.x*net.y): 66 | # neighbours = np.where(net.mapDist[i,:]<=step) 67 | # #print neighbours 68 | # #n = np.tile(net.weights[:,i],(np.shape(neighbours)[1],1)) 69 | # t = np.zeros((np.shape(neighbours)[1]*2,np.shape(net.weights)[0])) 70 | # t[::2,:] = np.tile(net.weights[:,i],(np.shape(neighbours)[1],1)) 71 | # t[1::2,:] = np.transpose(net.weights[:,neighbours[0][:]]) 72 | # pl.plot(t[:,0],t[:,1],'g-') 73 | # 74 | #net.somtrain(data,100) 75 | #pl.figure(5) 76 | #pl.plot(data[:,0],data[:,1],'.') 77 | #for i in range(net.x*net.y): 78 | # neighbours = np.where(net.mapDist[i,:]<=step) 79 | # #print neighbours 80 | # #n = np.tile(net.weights[:,i],(snp.hape(neighbours)[1],1)) 81 | # t = np.zeros((np.shape(neighbours)[1]*2,np.shape(net.weights)[0])) 82 | # t[::2,:] = np.tile(net.weights[:,i],(np.shape(neighbours)[1],1)) 83 | # t[1::2,:] = np.transpose(net.weights[:,neighbours[0][:]]) 84 | # pl.plot(t[:,0],t[:,1],'g-') 85 | 86 | pl.show() 87 | -------------------------------------------------------------------------------- /Ch15/BoxMuller.py: -------------------------------------------------------------------------------- 1 | 2 | # Code from Chapter 15 of Machine Learning: An Algorithmic Perspective (2nd Edition) 3 | # by Stephen Marsland (http://stephenmonika.net) 4 | 5 | # You are free to use, change, or redistribute the code in any way you wish for 6 | # non-commercial purposes, but please maintain the name of the original author. 7 | # This code comes with no warranty of any kind. 8 | 9 | # Stephen Marsland, 2008, 2014 10 | 11 | # The Box-Muller algorithm for constructing pseudo-random Gaussian-distributed numbers 12 | 13 | import pylab as pl 14 | import numpy as np 15 | 16 | def boxmuller(n): 17 | 18 | x = np.zeros((n,2)) 19 | y = np.zeros((n,2)) 20 | 21 | for i in range(n): 22 | x[i,:] = np.array([2,2]) 23 | x2 = x[i,0]*x[i,0]+x[i,1]*x[i,1] 24 | while (x2)>1: 25 | x[i,:] = np.random.rand(2)*2-1 26 | x2 = x[i,0]*x[i,0]+x[i,1]*x[i,1] 27 | 28 | y[i,:] = x[i,:] * np.sqrt((-2*np.log(x2))/x2) 29 | 30 | y = np.reshape(y,2*n,1) 31 | return y 32 | 33 | y = boxmuller(1000) 34 | pl.hist(y,normed=1,fc='k') 35 | x = np.arange(-4,4,0.1) 36 | pl.plot(x,1/np.sqrt(2*np.pi)*np.exp(-0.5*x**2),'k',lw=6) 37 | pl.xlabel('x',fontsize=24) 38 | pl.ylabel('p(x)',fontsize=24) 39 | pl.show() 40 | 41 | 42 | -------------------------------------------------------------------------------- /Ch15/Gibbs.py: -------------------------------------------------------------------------------- 1 | 2 | # Code from Chapter 15 of Machine Learning: An Algorithmic Perspective (2nd Edition) 3 | # by Stephen Marsland (http://stephenmonika.net) 4 | 5 | # You are free to use, change, or redistribute the code in any way you wish for 6 | # non-commercial purposes, but please maintain the name of the original author. 7 | # This code comes with no warranty of any kind. 8 | 9 | # Stephen Marsland, 2008, 2014 10 | 11 | # A simple Gibbs sampler 12 | import pylab as pl 13 | import numpy as np 14 | 15 | def pxgiveny(y,mx,my,s1,s2): 16 | return np.random.normal(mx + (y-my)/s2,s1) 17 | #return random.binomial(16,y,1) 18 | 19 | def pygivenx(x,mx,my,s1,s2): 20 | return np.random.normal(my + (x-mx)/s1,s2) 21 | #return random.beta(x+2,16-x+4,1) 22 | 23 | def gibbs(N=500): 24 | k=10 25 | x0 = np.zeros(N,dtype=float) 26 | m1 = 10 27 | m2 = 20 28 | s1 = 2 29 | s2 = 3 30 | for i in range(N): 31 | y = np.random.rand(1) 32 | for j in range(k): 33 | x = pxgiveny(y,m1,m2,s1,s2) 34 | y = pygivenx(x,m1,m2,s1,s2) 35 | x0[i] = x 36 | 37 | return x0 38 | 39 | #def f(x): 40 | # n = 16 41 | # alph = 2 42 | # bet = 4 43 | # return 20.0*(np.factorial(n)/(np.factorial(x)*np.factorial(n-x)))*np.factorial(x+1)*np.factorial(19-x)/np.factorial(21) 44 | # 45 | #def factorial(n): 46 | # x = 1 47 | # for i in range(n): 48 | # x *= (i+1) 49 | # return x 50 | 51 | def f(x): 52 | return np.exp(-(x-10)**2/10) 53 | 54 | N=500 55 | s=gibbs(N) 56 | x1 = np.arange(0,17,1) 57 | pl.hist(s,bins=x1,fc='k') 58 | x1 = np.arange(0,17,0.1) 59 | px1 = np.zeros(len(x1)) 60 | for i in range(len(x1)): 61 | px1[i] = f(x1[i]) 62 | pl.plot(x1, px1*N*10/np.sum(px1), color='k',linewidth=3) 63 | 64 | pl.show() 65 | -------------------------------------------------------------------------------- /Ch15/MH.py: -------------------------------------------------------------------------------- 1 | 2 | # Code from Chapter 15 of Machine Learning: An Algorithmic Perspective (2nd Edition) 3 | # by Stephen Marsland (http://stephenmonika.net) 4 | 5 | # You are free to use, change, or redistribute the code in any way you wish for 6 | # non-commercial purposes, but please maintain the name of the original author. 7 | # This code comes with no warranty of any kind. 8 | 9 | # Stephen Marsland, 2008, 2014 10 | 11 | # The Metropolis-Hastings algorithm 12 | import pylab as pl 13 | import numpy as np 14 | 15 | def p(x): 16 | mu1 = 3 17 | mu2 = 10 18 | v1 = 10 19 | v2 = 3 20 | return 0.3*np.exp(-(x-mu1)**2/v1) + 0.7* np.exp(-(x-mu2)**2/v2) 21 | 22 | def q(x): 23 | mu = 5 24 | sigma = 10 25 | return np.exp(-(x-mu)**2/(sigma**2)) 26 | 27 | stepsize = 0.5 28 | x = np.arange(-10,20,stepsize) 29 | px = np.zeros(np.shape(x)) 30 | for i in range(len(x)): 31 | px[i] = p(x[i]) 32 | N = 5000 33 | 34 | # independence chain 35 | mu = 5 36 | sigma = 10 37 | u = np.random.rand(N) 38 | y = np.zeros(N) 39 | y[0] = np.random.normal(mu,sigma) 40 | for i in range(N-1): 41 | ynew = np.random.normal(mu,sigma) 42 | alpha = min(1,p(ynew)*q(y[i])/(p(y[i])*q(ynew))) 43 | if u[i] < alpha: 44 | y[i+1] = ynew 45 | else: 46 | y[i+1] = y[i] 47 | 48 | # random walk chain 49 | sigma = 10 50 | u2 = np.random.rand(N) 51 | y2 = np.zeros(N) 52 | y2[0] = np.random.normal(0,sigma) 53 | for i in range(N-1): 54 | y2new = y2[i] + np.random.normal(0,sigma) 55 | alpha = min(1,p(y2new)/p(y2[i])) 56 | if u2[i] < alpha: 57 | y2[i+1] = y2new 58 | else: 59 | y2[i+1] = y2[i] 60 | 61 | pl.figure(1) 62 | nbins = 30 63 | pl.hist(y, bins = x) 64 | pl.plot(x, px*N/sum(px), color='r', linewidth=2) 65 | 66 | pl.figure(2) 67 | nbins = 30 68 | pl.hist(y2, bins = x) 69 | pl.plot(x, px*N/sum(px), color='r', linewidth=2) 70 | 71 | pl.show() 72 | -------------------------------------------------------------------------------- /Ch15/SIR.py: -------------------------------------------------------------------------------- 1 | 2 | # Code from Chapter 15 of Machine Learning: An Algorithmic Perspective (2nd Edition) 3 | # by Stephen Marsland (http://stephenmonika.net) 4 | 5 | # You are free to use, change, or redistribute the code in any way you wish for 6 | # non-commercial purposes, but please maintain the name of the original author. 7 | # This code comes with no warranty of any kind. 8 | 9 | # Stephen Marsland, 2008, 2014 10 | 11 | # The Sampling-Importance-Resampling algorithm 12 | import pylab as pl 13 | import numpy as np 14 | 15 | def p(x): 16 | return 0.3*np.exp(-(x-0.3)**2) + 0.7* np.exp(-(x-2.)**2/0.3) 17 | 18 | def q(x): 19 | return 4.0 20 | 21 | def sir(n): 22 | 23 | sample1 = np.zeros(n) 24 | w = np.zeros(n) 25 | sample2 = np.zeros(n) 26 | 27 | # Sample from q 28 | sample1 = np.random.rand(n)*4 29 | 30 | # Compute weights 31 | w = p(sample1)/q(sample1) 32 | w /= np.sum(w) 33 | 34 | # Sample from sample1 according to w 35 | cumw = np.zeros(len(w)) 36 | cumw[0] = w[0] 37 | for i in range(1,len(w)): 38 | cumw[i] = cumw[i-1]+w[i] 39 | 40 | u = np.random.rand(n) 41 | 42 | index = 0 43 | for i in range(n): 44 | indices = np.where(u