├── Readme.md ├── maxent.html ├── lbfgs.js ├── LICENSE ├── xregexp-all-min.js └── numeric-1.2.6.min.js /Readme.md: -------------------------------------------------------------------------------- 1 | ### A javascript implementation of L-BFGS 2 | 3 | The script `lbfgs.js` contains an implementation of the limited-memory BFGS optimizer. 4 | L-BFGS is commonly used for training in supervised Machine Learning tasks. 5 | To use it, call 6 | 7 | limitedMemoryBFGS(optimizable, parameters); 8 | 9 | where `optimizable` is an object that contains two methods, `getValue(parameters)` and `getGradient(parameters, gradient)`. 10 | The first function returns the value of an objective function, such as a log likelihood, given the specified parameter values. 11 | The second copies the gradient of that objective function at the specified parameters into the array `gradient`. 12 | I found that limiting memory allocations caused the single biggest gain in performance. 13 | 14 | This code is based on the implementation in Mallet. 15 | For a trivial example (the two-variable quadratic equation), these implementations produce identical results down to the last available decimal point. 16 | 17 | I've included an example of a Maximum Entropy classifier, in `maxent.html`. 18 | Look at the console output (the actual page is not very interesting!) to see the optimizer working. 19 | The tokenization and MaxEnt value/gradient implementations are not the same as Mallet. 20 | Mallet is about two to three times faster, but I suspect that much of this is due to the fact that javascript is not good with int-int maps, so I'm running through document tokens in sequence, rather than as feature-count vectors. 21 | The sample file contains translations of Danish folktales, used with permission by the translator. 22 | 23 | The script depends on numeric.js and d3, but mainly for convenience methods, which could be trivially rewritten. 24 | I initially used more convenience functions from numeric.js, but found that it was often substantially faster to write them as for-loops. 25 | The XRegExp library is useful for parsing Unicode regular expressions, and has nothing particular to do with L-BFGS. -------------------------------------------------------------------------------- /maxent.html: -------------------------------------------------------------------------------- 1 | 2 |
3 | 4 | 5 | 6 | 7 | 8 | 20 | 21 | 22 |p)g[L+O]=P;else{g[L+O]=-Math.abs(P);if(P>0){for(w=1;w<=o;w+=1)f[w][O]=-f[w][O];l[O]=-l[O]}}return 700}v+=1,d[v]=O,E=k+(v-1)*v/2+1;for(b=1;b<=v-1;b+=1)g[E]=g[b],E+=1;if(v===o)g[E]=g[o];else{for(b=o;b>=v+1;b-=1){if(g[b]===0)break;j=Math.max(Math.abs(g[b-1]),Math.abs(g[b])),F=Math.min(Math.abs(g[b-1]),Math.abs(g[b])),g[b-1]>=0?D=Math.abs(j*Math.sqrt(1+F*F/(j*j))):D=-Math.abs(j*Math.sqrt(1+F*F/(j*j))),j=g[b-1]/D,F=g[b]/D;if(j===1)break;if(j===0){g[b-1]=F*D;for(w=1;w<=o;w+=1)D=e[w][b-1],e[w][b-1]=e[w][b],e[w][b]=D}else{g[b-1]=D,I=F/(1+j);for(w=1;w<=o;w+=1)D=j*e[w][b-1]+F*e[w][b],e[w][b]=I*(e[w][b-1]+D)-e[w][b],e[w][b-1]=D}}g[E]=g[v]}return 0}function J(){E=k+T*(T+1)/2+1,S=E+T;if(g[S]===0)return 798;j=Math.max(Math.abs(g[S-1]),Math.abs(g[S])),F=Math.min(Math.abs(g[S-1]),Math.abs(g[S])),g[S-1]>=0?D=Math.abs(j*Math.sqrt(1+F*F/(j*j))):D=-Math.abs(j*Math.sqrt(1+F*F/(j*j))),j=g[S-1]/D,F=g[S]/D;if(j===1)return 798;if(j===0){for(b=T+1;b<=v;b+=1)D=g[S-1],g[S-1]=g[S],g[S]=D,S+=b;for(b=1;b<=o;b+=1)D=e[b][T],e[b][T]=e[b][T+1],e[b][T+1]=D}else{I=F/(1+j);for(b=T+1;b<=v;b+=1)D=j*g[S-1]+F*g[S],g[S]=I*(g[S-1]+D)-g[S],g[S-1]=D,S+=b;for(b=1;b<=o;b+=1)D=j*e[b][T]+F*e[b][T+1],e[b][T+1]=I*(e[b][T]+D)-e[b][T+1],e[b][T]=D}return 0}function K(){S=E-T;for(b=1;b<=T;b+=1)g[S]=g[E],E+=1,S+=1;return g[A+T]=g[A+T+1],d[T]=d[T+1],T+=1,T =0&&(b=-b),w=y*b-T,c[u][u]=y-b;for(a=l;a =0&&(b=-b),w=y*b-T,c[u][u+1]=y-b;for(a=l;a E&&(E=S)}for(u=p-1;u!=-1;u+=-1){if(b!=0){w=b*c[u][u+1];for(a=l;a =s-1)throw"Error: no convergence.";E=v[l],S=v[f-1],b=d[f-1],w=d[f],y=((S-x)*(S+x)+(b-w)*(b+w))/(2*w*S),b=g(y,1),y<0?y=((E-x)*(E+x)+w*(S/(y-b)-w))/E:y=((E-x)*(E+x)+w*(S/(y+b)-w))/E,o=1,T=1;for(u=l+1;u