├── CNAME ├── index.md ├── Gemfile ├── favicon.ico ├── weather.q ├── test.cmd ├── funq.q ├── qmlmm.q ├── berkstan.q ├── bible.q ├── binary.q ├── dji.q ├── cloud9.q ├── cossim.q ├── smsspam.q ├── files.txt ├── moby.q ├── pandp.q ├── emma.q ├── sands.q ├── liver.q ├── optdigits.q ├── persuasion.q ├── ionosphere.q ├── mansfield.q ├── northanger.q ├── onp.q ├── iris.q ├── testporter.q ├── pima.q ├── seeds.q ├── pendigits.q ├── winequality.q ├── housing.q ├── uef.q ├── wine.q ├── stopwords.q ├── weather.csv ├── wdbc.q ├── zoo.q ├── etl9b.q ├── mnist.q ├── adult.q ├── supportvectormachine.q ├── bitstamp.q ├── _config.yml ├── sparse.q ├── gemini.q ├── cifar.q ├── markov.q ├── .gitignore ├── binance.q ├── kraken.q ├── svm.q ├── testsvm.q ├── LICENSE ├── randomforest.q ├── bitfinex.q ├── testlinear.q ├── mlens.q ├── onevsall.q ├── errata.adoc ├── linear.q ├── silhouette.q ├── plot.q ├── citibike.q ├── hiragana.q ├── knn.q ├── adaboost.q ├── pagerank.q ├── tfidf.q ├── hac.q ├── GNUmakefile ├── logreg.q ├── README.md ├── linreg.q ├── nb.q ├── em.q ├── kmeans.q ├── porter.q ├── decisiontree.q ├── nn.q ├── fmincg.q ├── recommend.q ├── ut.q ├── svm.c ├── linear.c └── ml.q /CNAME: -------------------------------------------------------------------------------- 1 | www.fun-q.net -------------------------------------------------------------------------------- /index.md: -------------------------------------------------------------------------------- 1 | README.md -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | gem "github-pages", group: :jekyll_plugins 2 | -------------------------------------------------------------------------------- /favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/psaris/funq/HEAD/favicon.ico -------------------------------------------------------------------------------- /weather.q: -------------------------------------------------------------------------------- 1 | weather.f:"weather.csv" 2 | weather.t:`Play xcols (" SSSSS";1#",") 0: `$weather.f 3 | -------------------------------------------------------------------------------- /test.cmd: -------------------------------------------------------------------------------- 1 | set SSL_VERIFY_SERVER=NO 2 | 3 | FOR /F "tokens=*" %%A IN (files.txt) DO q %%A -s 4 > nul < nul 4 | -------------------------------------------------------------------------------- /funq.q: -------------------------------------------------------------------------------- 1 | \l ut.q 2 | \l ml.q 3 | \l fmincg.q 4 | \l porter.q 5 | 6 | / attempt to load c libraries 7 | (.ut.loadf ` sv hsym[`$getenv`QHOME],) each`qml.q`svm.q`linear.q; 8 | if[`qml in key `;system "l qmlmm.q"] / use qml matrix operators 9 | -------------------------------------------------------------------------------- /qmlmm.q: -------------------------------------------------------------------------------- 1 | .ml.mm:.qml.mm 2 | .ml.mmt:.qml.mmx[`rflip] 3 | .ml.mtm:.qml.mmx[`lflip] 4 | .ml.minv:.qml.minv 5 | .ml.mlsq:{.qml.mlsqx[`flip;y;x]} 6 | .ml.dot:.qml.dot 7 | .ml.mdet:.qml.mdet 8 | .ml.mchol:.qml.mchol 9 | 10 | .fmincg.dot:.qml.dot -------------------------------------------------------------------------------- /berkstan.q: -------------------------------------------------------------------------------- 1 | berkstan.f:"web-BerkStan.txt" 2 | berkstan.b:"http://snap.stanford.edu/data/" 3 | -1"[down]loading berkstan network graph"; 4 | .ut.download[berkstan.b;;".gz";.ut.gunzip] berkstan.f; 5 | berkstan.l:("II";"\t") 0: 4_read0 `$berkstan.f 6 | -------------------------------------------------------------------------------- /bible.q: -------------------------------------------------------------------------------- 1 | / the bible 2 | bible.f:"10.txt" 3 | bible.b:"https://www.gutenberg.org/files/10/old/" 4 | -1"[down]loading bible text"; 5 | .ut.download[bible.b;;"";""] bible.f; 6 | bible.txt:read0 `$bible.f 7 | bible.s:1_"\n1:1 " vs "\n" sv 39_-373_ bible.txt 8 | -------------------------------------------------------------------------------- /binary.q: -------------------------------------------------------------------------------- 1 | binary.f:"binary.csv" 2 | binary.b:"https://www.ats.ucla.edu/stat/data/" 3 | -1"[down]loading binary data set"; 4 | .ut.download[binary.b;;"";""] binary.f; 5 | binary.t:("BIFI";1#",") 0: `$binary.f 6 | binary[`Y`X]: 0 1 cut "f"$value flip binary.t 7 | -------------------------------------------------------------------------------- /dji.q: -------------------------------------------------------------------------------- 1 | dji.f:"dow_jones_index" 2 | dji.b:"http://archive.ics.uci.edu/ml/machine-learning-databases/" 3 | dji.b,:"00312/" 4 | -1"[down]loading dji data set"; 5 | .ut.download[dji.b;;".zip";.ut.unzip] dji.f; 6 | dji.t:("HSDEEEEJFFJEEFHF";1#",")0: ssr[;"$";""] each read0 `$dji.f,".data" 7 | -------------------------------------------------------------------------------- /cloud9.q: -------------------------------------------------------------------------------- 1 | cloud9.f:("sample-small.txt";"sample-medium.txt";"sample-large.txt") 2 2 | cloud9.b:"http://lintool.github.io/Cloud9/docs/exercises/" 3 | -1"[down]loading cloud9 network graph"; 4 | .ut.download[cloud9.b;;"";""] cloud9.f; 5 | cloud9.l:flip raze {x[0],/:1_ x} each "J"$"\t" vs/: read0 `$cloud9.f 6 | -------------------------------------------------------------------------------- /cossim.q: -------------------------------------------------------------------------------- 1 | \c 40 100 2 | \l funq.q 3 | \l iris.q 4 | 5 | / cosine similarity (distance) 6 | X:.ml.normalize iris.X 7 | flip C:.ml.skmeans[X] over .ml.forgy[3] X / spherical k-means 8 | show m:.ml.mode each iris.y I:.ml.cgroup[.ml.cosdist;X;C] / classify 9 | avg iris.y=.ut.ugrp m!I / accuracy 10 | -------------------------------------------------------------------------------- /smsspam.q: -------------------------------------------------------------------------------- 1 | smsspam.f:"smsspamcollection" 2 | smsspam.b:"http://archive.ics.uci.edu/ml/machine-learning-databases/" 3 | smsspam.b,:"00228/" 4 | -1"[down]loading sms-spam data set"; 5 | .ut.download[smsspam.b;;".zip";.ut.unzip] smsspam.f; 6 | smsspam.t:flip `class`text!("S*";"\t")0: `:SMSSpamCollection 7 | -------------------------------------------------------------------------------- /files.txt: -------------------------------------------------------------------------------- 1 | testporter.q 2 | plot.q 3 | knn.q 4 | kmeans.q 5 | em.q 6 | pagerank.q 7 | sparse.q 8 | decisiontree.q 9 | randomforest.q 10 | markov.q 11 | hac.q 12 | cossim.q 13 | adaboost.q 14 | linreg.q 15 | logreg.q 16 | recommend.q 17 | nn.q 18 | onevsall.q 19 | nb.q 20 | tfidf.q 21 | silhouette.q 22 | -------------------------------------------------------------------------------- /moby.q: -------------------------------------------------------------------------------- 1 | / moby-dick 2 | moby.f:"2701.txt" 3 | moby.b:"https://www.gutenberg.org/files/2701/old/" 4 | -1"[down]loading moby-dick text"; 5 | .ut.download[moby.b;;"";""] moby.f; 6 | moby.txt:read0 `$moby.f 7 | moby.chapters:"\nCHAPTER" vs "\n" sv 298_-373_ moby.txt 8 | moby.s:{(3+first x ss"\n\n\n")_x} each moby.chapters 9 | -------------------------------------------------------------------------------- /pandp.q: -------------------------------------------------------------------------------- 1 | pandp.f:"1342-0.txt" 2 | pandp.b:"https://www.gutenberg.org/files/1342/" 3 | -1"[down]loading pride and prejudice text"; 4 | .ut.download[pandp.b;;"";""] pandp.f; 5 | pandp.txt:read0 `$pandp.f 6 | pandp.chapters:1_"\nChapter " vs "\n" sv 35_-373_ pandp.txt 7 | pandp.s:{first[x ss"\n\n"]_x} each pandp.chapters 8 | -------------------------------------------------------------------------------- /emma.q: -------------------------------------------------------------------------------- 1 | / emma 2 | emma.f:"158.txt" 3 | emma.b:"https://www.gutenberg.org/files/158/old/" 4 | -1"[down]loading emma text"; 5 | .ut.download[emma.b;;"";""] emma.f; 6 | emma.txt:{x where not x like "VOLUME*"} read0 `$emma.f 7 | emma.chapters:1_"CHAPTER" vs "\n" sv 39_-373_emma.txt 8 | emma.s:{(3+first x ss"\n\n\n")_x} each emma.chapters 9 | -------------------------------------------------------------------------------- /sands.q: -------------------------------------------------------------------------------- 1 | / sense and sensibility 2 | sands.f:"161.txt" 3 | sands.b:"https://www.gutenberg.org/files/161/old/" 4 | -1"[down]loading sense and sensibility text"; 5 | .ut.download[sands.b;;"";""] sands.f; 6 | sands.txt:read0 `$sands.f 7 | sands.chapters:1_"CHAPTER" vs "\n" sv 43_-373_sands.txt 8 | sands.s:{(3+first x ss"\n\n\n")_x} each sands.chapters 9 | -------------------------------------------------------------------------------- /liver.q: -------------------------------------------------------------------------------- 1 | liver.f:"bupa.data" 2 | liver.b:"http://archive.ics.uci.edu/ml/machine-learning-databases/" 3 | liver.b,:"liver-disorders/" 4 | -1"[down]loading liver data set"; 5 | .ut.download[liver.b;;"";""] liver.f; 6 | liver.XY:((6#"E"),"H";",")0:`$liver.f 7 | liver.X:-1_liver.XY 8 | liver.c:`mcv`alkphos`sgpt`sgot`gammagt`drinks`train 9 | liver.t:flip liver.c!liver.XY 10 | -------------------------------------------------------------------------------- /optdigits.q: -------------------------------------------------------------------------------- 1 | optdigits.f:("optdigits.tra";"optdigits.tes") 2 | optdigits.b:"http://archive.ics.uci.edu/ml/machine-learning-databases/" 3 | optdigits.b,:"optdigits/" 4 | -1"[down]loading optdigits data set"; 5 | .ut.download[optdigits.b;;"";""] each optdigits.f; 6 | optdigits[`X`Y`Xt`Yt]:raze (64 cut (65#"H";",") 0: `$) each optdigits.f 7 | optdigits[`y`yt]:optdigits[`Y`Yt][;0] 8 | -------------------------------------------------------------------------------- /persuasion.q: -------------------------------------------------------------------------------- 1 | / persuasion 2 | persuasion.f:"105.txt" 3 | persuasion.b:"https://www.gutenberg.org/files/105/" 4 | -1"[down]loading persuasion text"; 5 | .ut.download[persuasion.b;;"";""] persuasion.f; 6 | persuasion.txt:read0 `$persuasion.f 7 | persuasion.chapters:1_"Chapter" vs "\n" sv 44_-373_persuasion.txt 8 | persuasion.s:{(3+first x ss"\n\n\n")_x} each persuasion.chapters 9 | -------------------------------------------------------------------------------- /ionosphere.q: -------------------------------------------------------------------------------- 1 | ionosphere.f:"ionosphere.data" 2 | ionosphere.b:"http://archive.ics.uci.edu/ml/machine-learning-databases/" 3 | ionosphere.b,:"ionosphere/" 4 | -1"[down]loading ionosphere data set"; 5 | .ut.download[ionosphere.b;;"";""] ionosphere.f; 6 | ionosphere.XY:((34#"E"),"C";",")0:`$ionosphere.f 7 | ionosphere.X:-1_ionosphere.XY 8 | ionosphere.y:first ionosphere.Y:-1#ionosphere.XY 9 | -------------------------------------------------------------------------------- /mansfield.q: -------------------------------------------------------------------------------- 1 | / mansfield park 2 | mansfield.f:"141.txt" 3 | mansfield.b:"https://www.gutenberg.org/files/141/old/" 4 | -1"[down]loading mansfield park text"; 5 | .ut.download[mansfield.b;;"";""] mansfield.f; 6 | mansfield.txt:read0 `$mansfield.f 7 | mansfield.chapters:1_"CHAPTER" vs "\n" sv 35_-373_mansfield.txt 8 | mansfield.s:{(2+first x ss"\n\n")_x} each mansfield.chapters 9 | 10 | -------------------------------------------------------------------------------- /northanger.q: -------------------------------------------------------------------------------- 1 | / northanger abbey 2 | northanger.f:"20080515-121.txt" 3 | northanger.b:"https://www.gutenberg.org/files/121/old/" 4 | -1"[down]loading northanger abbey text"; 5 | .ut.download[northanger.b;;"";""] northanger.f; 6 | northanger.txt:read0 `$northanger.f 7 | northanger.chapters:1_"CHAPTER" vs "\n" sv 57_-373_northanger.txt 8 | northanger.s:{(3+first x ss"\n\n\n")_x} each northanger.chapters 9 | -------------------------------------------------------------------------------- /onp.q: -------------------------------------------------------------------------------- 1 | onp.f:"OnlineNewsPopularity" 2 | onp.b:"http://archive.ics.uci.edu/ml/machine-learning-databases/" 3 | onp.b,:"00332/" 4 | -1"[down]loading online news popularity data set"; 5 | .ut.download[onp.b;;".zip";.ut.unzip] onp.f; 6 | 7 | onp.t:(" efefebebfi" where 2 2 3 4 1 1 6 12 8 21 1;1#",") 0: `$onp.f,"/",onp.f,".csv" 8 | onp.t:`popular xcols delete shares from update popular:shares>=1400 from onp.t; 9 | 10 | -------------------------------------------------------------------------------- /iris.q: -------------------------------------------------------------------------------- 1 | iris.f:("iris.data";"bezdekIris.data") 1 2 | iris.b:"http://archive.ics.uci.edu/ml/machine-learning-databases/" 3 | iris.b,:"iris/" 4 | -1"[down]loading iris data set"; 5 | .ut.download[iris.b;;"";""] iris.f; 6 | iris.XY:150#/:("FFFFS";",") 0: `$iris.f 7 | iris.X:-1_iris.XY 8 | iris.y:first iris.Y:-1#iris.XY 9 | iris.c:`slength`swidth`plength`pwidth`species 10 | iris.t:`species xcols flip iris.c!iris.XY 11 | 12 | -------------------------------------------------------------------------------- /testporter.q: -------------------------------------------------------------------------------- 1 | \l funq.q 2 | 3 | b:"https://tartarus.org/martin/PorterStemmer/" 4 | 5 | -1"[down]loading porter stemmer vocabulary"; 6 | pin:read0 .ut.download[b;;"";""] "voc.txt" 7 | -1"[down]loading stemmed vocabulary"; 8 | pout:read0 .ut.download[b;;"";""] "output.txt" 9 | -1"stemming vocabulary"; 10 | out:.porter.stem peach pin 11 | -1"incorrectly stemmed "; 12 | .ut.assert[0] count flip (pin;pout;out)@\: where not pout ~'out 13 | -------------------------------------------------------------------------------- /pima.q: -------------------------------------------------------------------------------- 1 | pima.f:"pima-indians-diabetes.data" 2 | pima.b:"http://archive.ics.uci.edu/ml/machine-learning-databases/" 3 | pima.b,:"pima-indians-diabetes/" 4 | -1"[down]loading pima-indians-diabetes data set"; 5 | .ut.download[pima.b;;"";""] pima.f; 6 | pima.XY:("EEEEEEEEB";",")0:`$pima.f 7 | pima.X:-1_pima.XY 8 | pima.y:first pima.Y:-1#pima.XY 9 | pima.c:`preg`plas`pres`skin`test`mass`pedi`age`class 10 | pima.t:`class xcols flip pima.c!pima.XY 11 | -------------------------------------------------------------------------------- /seeds.q: -------------------------------------------------------------------------------- 1 | seeds.f:"seeds_dataset.txt" 2 | seeds.b:"http://archive.ics.uci.edu/ml/machine-learning-databases/" 3 | seeds.b,:"00236/" 4 | -1"[down]loading seeds data set"; 5 | .ut.download[seeds.b;;"";""] seeds.f; 6 | seeds.XY:("FFFFFFFH";"\t") 0: ssr[;"\t\t";"\t"] each read0 `$seeds.f 7 | seeds.X:-1_seeds.XY 8 | seeds.y:first seeds.Y:-1#seeds.XY 9 | seeds.c:`area`perimeter`compactness`length`width`asymmetry`groove`variety 10 | seeds.t:`variety xcols flip seeds.c!seeds.XY 11 | 12 | -------------------------------------------------------------------------------- /pendigits.q: -------------------------------------------------------------------------------- 1 | pendigits.f:("pendigits.tra";"pendigits.tes") 2 | pendigits.b:"http://archive.ics.uci.edu/ml/machine-learning-databases/" 3 | pendigits.b,:"pendigits/" 4 | -1"download the pendigits training and test data set"; 5 | .ut.download[pendigits.b;;"";""] each pendigits.f; 6 | pendigits.y:last pendigits.X:(17#"h";",") 0: `$pendigits.f 0 7 | pendigits.X:-1_pendigits.X 8 | 9 | pendigits.yt:last pendigits.Xt:(17#"h";",") 0: `$pendigits.f 1 10 | pendigits.Xt:-1_pendigits.Xt 11 | 12 | 13 | -------------------------------------------------------------------------------- /winequality.q: -------------------------------------------------------------------------------- 1 | winequality.f:`red`white!("winequality-red.csv";"winequality-white.csv") 2 | winequality.b:"http://archive.ics.uci.edu/ml/machine-learning-databases/" 3 | winequality.b,:"wine-quality/" 4 | -1"[down]loading wine-quality data set"; 5 | .ut.download[winequality.b;;"";""] each winequality.f; 6 | .winequality.load:{[f] 7 | YX:value flip t:`quality xcols .Q.id (12#"F";1#";")0:f; 8 | d:`X`Y`y`t!(1_YX;1#YX;YX 0;t); 9 | d} 10 | winequality,:.winequality.load each `$winequality.f 11 | -------------------------------------------------------------------------------- /housing.q: -------------------------------------------------------------------------------- 1 | housing.f:"housing.data" 2 | housing.b:"http://archive.ics.uci.edu/ml/machine-learning-databases/" 3 | housing.b,:"housing/" 4 | -1"[down]loading housing data set"; 5 | .ut.download[housing.b;;"";""] housing.f; 6 | housing.c:`crim`zn`indus`chas`nox`rm`age`dis`rad`tax`ptratio`b`lstat`medv 7 | housing.tw:("FFFBFFFFHFFFFF";8 7 8 3 8 8 7 8 4 7 7 7 7 7) 8 | housing.t:`medv xcols flip housing.c!housing.tw 0: `$housing.f 9 | housing[`Y`X]:0 1 cut value flip housing.t 10 | housing.y:first housing.Y 11 | -------------------------------------------------------------------------------- /uef.q: -------------------------------------------------------------------------------- 1 | uef.s:("s1.txt";"s2.txt";"s3.txt";"s4.txt") 2 | uef.a:("a1.txt";"a2.txt";"a3.txt") 3 | uef.d:("dim032.txt";"dim064.txt";"dim128.txt") 4 | uef.d,:("dim256.txt";"dim512.txt";"dim1024.txt"); 5 | uef.b:"http://www.cs.uef.fi/sipu/datasets/" 6 | -1"[down]loading uef data sets"; 7 | .ut.download[uef.b;;"";""] uef.s,uef.a,uef.d; 8 | 9 | uef,:`s1`s2`s3`s4!("JJ";10 10) 0:/: `$uef.s 10 | uef,:`a1`a2`a3!("JJ";8 8) 0:/: `$uef.a 11 | uef,:(!) . flip {(`$"d",string x;(x#"J";x#6) 0: y)}'[16*\6#2;`$uef.d] 12 | 13 | -------------------------------------------------------------------------------- /wine.q: -------------------------------------------------------------------------------- 1 | wine.f:"wine.data" 2 | wine.b:"http://archive.ics.uci.edu/ml/machine-learning-databases/" 3 | wine.b,:"wine/" 4 | -1"[down]loading wine data set"; 5 | .ut.download[wine.b;;"";""] wine.f; 6 | wine.XY:("H",13#"E";",")0:`$wine.f 7 | wine.X:1_wine.XY 8 | wine.y:first wine.Y:1#wine.XY 9 | wine.c:`class`alcohol`malic_acid`ash`alcalinity_of_ash`magnesium 10 | wine.c,:`total_phenols`flavanoids`nonflavanoid_phenols`proanthocyanins 11 | wine.c,:`color_intensity`hue`OD280_OD315`proline 12 | wine.t:flip wine.c!wine.XY 13 | -------------------------------------------------------------------------------- /stopwords.q: -------------------------------------------------------------------------------- 1 | stopwords.f:"stop-word-list.txt" 2 | stopwords.b:"http://xpo6.com/wp-content/uploads/2015/01/" 3 | -1"[down]loading xpo6 stop words"; 4 | .ut.download[stopwords.b;;"";""] stopwords.f; 5 | stopwords.xpo6:asc enlist[""],read0 `$":",stopwords.f 6 | 7 | stopwords.f:"stop.txt" 8 | stopwords.b:"http://snowball.tartarus.org/algorithms/english/" 9 | -1"[down]loading snowball stop words"; 10 | .ut.download[stopwords.b;;"";""] stopwords.f; 11 | stopwords.snowball:asc distinct trim {(x?"|")#x} each read0 `$":",stopwords.f 12 | -------------------------------------------------------------------------------- /weather.csv: -------------------------------------------------------------------------------- 1 | Day,Outlook,Temperature,Humidity,Wind,Play 2 | D1,Sunny,Hot,High,Weak,No 3 | D2,Sunny,Hot,High,Strong,No 4 | D3,Overcast,Hot,High,Weak,Yes 5 | D4,Rain,Mild,High,Weak,Yes 6 | D5,Rain,Cool,Normal,Weak,Yes 7 | D6,Rain,Cool,Normal,Strong,No 8 | D7,Overcast,Cool,Normal,Strong,Yes 9 | D8,Sunny,Mild,High,Weak,No 10 | D9,Sunny,Cool,Normal,Weak,Yes 11 | D10,Rain,Mild,Normal,Weak,Yes 12 | D11,Sunny,Mild,Normal,Strong,Yes 13 | D12,Overcast,Mild,High,Strong,Yes 14 | D13,Overcast,Hot,Normal,Weak,Yes 15 | D14,Rain,Mild,High,Strong,No 16 | -------------------------------------------------------------------------------- /wdbc.q: -------------------------------------------------------------------------------- 1 | wdbc.f:"wdbc.data" 2 | wdbc.b:"http://archive.ics.uci.edu/ml/machine-learning-databases/" 3 | wdbc.b,:"breast-cancer-wisconsin/" 4 | -1"[down]loading wisconsin-diagnostic-breast-cancer data set"; 5 | .ut.download[wdbc.b;;"";""] wdbc.f; 6 | wdbc.XY:(" C",30#"E";",") 0: `$wdbc.f 7 | wdbc.X:1_wdbc.XY 8 | wdbc.y:first wdbc.Y:1#wdbc.XY 9 | wdbc.c:`radius`texture`perimeter`area`smoothness`compactness`concavity 10 | wdbc.c,:`concave_points`symmetry`fractal_dimension 11 | wdbc.c:raze `$"_" sv'string raze wdbc.c,\:/: `mean`se`worst 12 | wdbc.t:flip (`diagnosis,wdbc.c)!wdbc.XY 13 | -------------------------------------------------------------------------------- /zoo.q: -------------------------------------------------------------------------------- 1 | zoo.f:"zoo.data" 2 | zoo.b:"http://archive.ics.uci.edu/ml/machine-learning-databases/" 3 | zoo.b,:"zoo/" 4 | -1"[down]loading zoo data set"; 5 | .ut.download[zoo.b;;"";""] zoo.f; 6 | zoo.c:`animal`hair`feathers`eggs`milk`airborne`aquatic`predator`toothed 7 | zoo.c,:`backbone`breathes`venomous`fins`legs`tail`domestic`catsize`typ 8 | zoo.typ:``mamal`bird`reptile`fish`amphibian`insect`invertebrate 9 | zoo.t:`typ xcols flip zoo.c!("SBBBBBBBBBBBBHBBBJ";",") 0: `$zoo.f 10 | update `zoo.typ!typ from `zoo.t; 11 | zoo.y:first first zoo[`Y`X]: 0 1 cut value flip[zoo.t] _ `animal 12 | zoo.X:"f"$zoo.X 13 | -------------------------------------------------------------------------------- /etl9b.q: -------------------------------------------------------------------------------- 1 | etl9b.f:"ETL9B" 2 | etl9b.b:"http://etlcdb.db.aist.go.jp/etlcdb/data/" 3 | -1"[down]loading handwritten-kanji data set"; 4 | .ut.download[etl9b.b;;".zip";.ut.unzip] etl9b.f; 5 | -1"loading etl9b ('binalized' dataset)"; 6 | etl9b.x:.ut.etl9b read1 `:ETL9B/ETL9B_1 7 | -1"extracting the X matrix and y vector"; 8 | etl9b.h:0x24,/:"x"$0x21+0x01*til 83 / hiragana 9 | / etl9b.h:0x25,/:"x"$0x21+0x01*til 83 / katakana (missing) 10 | etl9b.y:flip etl9b.x 1 2 11 | etl9b.w:where etl9b.y in etl9b.h / find hiragana 12 | etl9b.y:etl9b.h?etl9b.y etl9b.w 13 | / extract 0 1 from bytes 14 | etl9b.X:"f"$flip (raze $[3.5>.z.K;-8#';::] 0b vs/:) each (1_etl9b.x 4) etl9b.w 15 | -------------------------------------------------------------------------------- /mnist.q: -------------------------------------------------------------------------------- 1 | mnist.zf:( 2 | "train-labels.idx1-ubyte"; 3 | "train-images.idx3-ubyte"; 4 | "t10k-labels.idx1-ubyte"; 5 | "t10k-images.idx3-ubyte") 6 | mnist.f:ssr[;".";"-"] each mnist.zf 7 | mnist.b:"http://yann.lecun.com/exdb/mnist/" 8 | -1"[down]loading handwritten-digit data set"; 9 | .ut.download[mnist.b;;".gz";.ut.gunzip] mnist.f; 10 | / rename unzipped file to match zipped file 11 | mnist.zf {[zf;f]if[zfs~key zfs:`$":",zf;system "r ",zf," ",f]}' mnist.f; 12 | 13 | mnist.Y:enlist mnist.y:"i"$.ut.mnist read1 `$mnist.f 0 14 | mnist.X:flip "f"$raze each .ut.mnist read1 `$mnist.f 1 15 | 16 | mnist.Yt:enlist mnist.yt:"i"$.ut.mnist read1 `$mnist.f 2 17 | mnist.Xt:flip "f"$raze each .ut.mnist read1 `$mnist.f 3 18 | -------------------------------------------------------------------------------- /adult.q: -------------------------------------------------------------------------------- 1 | adult.f:("adult.data";"adult.test") 2 | adult.b:"http://archive.ics.uci.edu/ml/machine-learning-databases/" 3 | adult.b,:"adult/" 4 | -1"[down]loading adult data set"; 5 | .ut.download[adult.b;;"";""] each adult.f; 6 | adult.c:`age`workclass`fnlwgt`education`educationnum`maritalstatus 7 | adult.c,:`occupation`relationship`race`sex`capitalgain`capitalloss 8 | adult.c,:`hoursperweek`nativecountry`gt50 9 | adult[`t`tt]:(-1_flip adult.c!("HSISHSSSSSIIISS";",") 0:) each `$adult.f 10 | adult.tt _:0 / drop comment 11 | {update like[;">50K*"] each gt50 from x} each `adult.t`adult.tt; 12 | adult[`X`Y`Xt`Yt]:raze (0 14 cut value flip@) each adult`t`tt 13 | adult[`y`yt]:first each adult`Y`Yt 14 | 15 | -------------------------------------------------------------------------------- /supportvectormachine.q: -------------------------------------------------------------------------------- 1 | \l funq.q 2 | \l iris.q 3 | 4 | stdout:1@ 5 | .svm.set_print_string_function`stdout 6 | -1"enumerate species so we can use the integer value for svm"; 7 | y:`species?iris.y 8 | -1"svm parameter x is a sparse matrix: - list of dictionaries"; 9 | prob:`x`y!(0 1 2 3i!/:flip iris.X;"f"$"i"$y) 10 | -1"define and check svm parameters"; 11 | .svm.check_parameter[prob] param:.svm.defparam[prob] .svm.param 12 | -1"build model by training svm on full dataset"; 13 | m:.svm.train[prob;param] 14 | -1"cross validate"; 15 | .svm.cross_validation[prob;param;2i]; 16 | -1"how well did we learn"; 17 | avg prob.y=p:.svm.predict[m] each prob.x 18 | -1"lets view the confusion matrix"; 19 | show .ut.totals[`TOTAL] .ml.cm[`species!"i"$prob.y] `species!"i"$p 20 | -------------------------------------------------------------------------------- /bitstamp.q: -------------------------------------------------------------------------------- 1 | bitstamp.p:string `daily`hourly`minutely!`d`1h`minute 2 | bitstamp.c:string `BTCUSD`ETHUSD`LTCUSD`XRPUSD`BCHUSD 3 | bitstamp.c,:string `BTCEUR`ETHEUR`LTCEUR`BCHEUR 4 | bitstamp.c,:string `ETHBTC`LTCBTC`BCHBTC 5 | bitstamp.f:bitstamp.p {"_" sv ("Bitstamp";y;x,".csv")}/:\: asc bitstamp.c 6 | bitstamp.b:"http://www.cryptodatadownload.com/cdd/" 7 | -1"[down]loading bitstamp data set"; 8 | .ut.download[bitstamp.b;;"";""] each raze bitstamp.f; 9 | .bitstamp.load:{[f] 10 | if[not count t:("P *FFFFF";1#",") 0: 1_read0 f;:()]; 11 | t:`time`sym`open`high`low`close`qty xcol t; 12 | t:update sym:`$sym except\: "/" from t; 13 | t:`sym xcols 0!select by time from t; / remove duplicates 14 | t} 15 | bitstamp,:({update `p#sym from x} raze .bitstamp.load peach::)'[`$bitstamp.f] 16 | -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | title: Fun Q 2 | description: "Source files for \"Fun Q: A Functional Introduction to Machine Learning in Q\"" 3 | google_analytics: 4 | show_downloads: false 5 | theme: jekyll-theme-hacker 6 | 7 | plugins: 8 | - jekyll-mentions 9 | 10 | exclude: 11 | - xgboost 12 | - liblinear 13 | - libsvm 14 | - ml-latest-small 15 | - readme 16 | - SMSSpamCollection 17 | - dow_jones_index.names 18 | - test.cmd 19 | - GNUmakefile 20 | - "*.dot" 21 | - "*.txt" 22 | - "*.ppm" 23 | - "*.pbm" 24 | - "*.pgm" 25 | - "*.data" 26 | - "*.test" 27 | - "*.tes" 28 | - "*.tra" 29 | - "*.png" 30 | - "*.png" 31 | - "*.zip" 32 | - "*-ubyte" 33 | - "*.c" 34 | - "*.h" 35 | - "*.csv" 36 | - "*.q" 37 | - Gemfile* 38 | - CNAME 39 | - LICENSE 40 | - ref.csv 41 | - errata.adoc -------------------------------------------------------------------------------- /sparse.q: -------------------------------------------------------------------------------- 1 | \c 20 100 2 | \l funq.q 3 | 4 | -1 "given a matrix with many missing values,"; 5 | show X:100 100#"f"$10000?0b 6 | -1 "we can record the non-zero values to create a sparse matrix"; 7 | show S:.ml.sparse X 8 | -1 "the representation includes the number of rows and columns"; 9 | -1 "followed by the x and y coordinates and finally the matrix valus"; 10 | .ut.assert[X] .ml.full S / matrix -> sparse -> matrix == matrix 11 | / sparse matrix multiplication == mmu 12 | -1 "we can perform sparse matrix transposition"; 13 | .ut.assert[flip X] .ml.full .ml.smt S 14 | -1 "sparse matrix multiplication"; 15 | .ut.assert[X$X] .ml.full .ml.smm[S;S] 16 | -1 "sparse matrix addition"; 17 | .ut.assert[X+X] .ml.full .ml.sma[S;S] 18 | -1 "sparse tensors"; 19 | .ut.assert[T] .ml.full .ml.sparse T:2 3 4#"f"$24?0b 20 | 21 | -------------------------------------------------------------------------------- /gemini.q: -------------------------------------------------------------------------------- 1 | gemini.p:string `daily`hourly!`day`1hr 2 | gemini.c:string `BTCUSD`ETHUSD`LTCUSD`ETHBTC`ZECUSD`ZECBTC`ZECETH 3 | gemini.f:gemini.p {"_" sv ("gemini";y;x,".csv")}/:\: asc gemini.c 4 | gemini.y:string (`year$.z.D-1) + reverse neg til 3 5 | gemini.f[`minutely]:raze gemini.y {"_" sv ("gemini";y;x;"1min.csv")}\:/: asc gemini.c 6 | gemini.b:"http://www.cryptodatadownload.com/cdd/" 7 | -1"[down]loading gemini data set"; 8 | .ut.download[gemini.b;;"";""] each raze gemini.f; 9 | .gemini.load:{[f] 10 | if[not count t:("* SFFFFF";1#",") 0: 1_read0 f;:()]; 11 | t:`time`sym`open`high`low`close`qty xcol t; 12 | t:update time:"P"$?[12>count each time;time;-3_/:time] from t; 13 | t:`sym xcols 0!select by time from t; / remove duplicates 14 | t} 15 | gemini,:({update `p#sym from x} raze .gemini.load peach::)'[`$gemini.f] 16 | -------------------------------------------------------------------------------- /cifar.q: -------------------------------------------------------------------------------- 1 | / http://www.cs.toronto.edu/~kriz/cifar.html 2 | cifar.f:"cifar-10-binary" 3 | cifar.b:"http://www.cs.toronto.edu/~kriz/" 4 | -1"[down]loading CFAR-10 data set"; 5 | .ut.download[cifar.b;;".tar.gz";.ut.untar] cifar.f; 6 | cifar.d:"cifar-10-batches-bin/" 7 | -1"reading labels"; 8 | cifar.labels:`$10#read0`$cifar.d,"batches.meta.txt" 9 | cifar.n:5 / how many batches of pictures to load (1-5) 10 | cifar.c:"data_batch_",/:(cifar.n#1_.Q.n),\:".bin" 11 | cifar.parse:(3073#"x";3073#1) 1: 12 | -1"reading images for ",string[cifar.n], " dataset(s)"; 13 | cifar.YX:(,'/) cifar.parse each `$cifar.d,/:cifar.c 14 | cifar.y:first cifar.Y:"h"$1#cifar.YX 15 | cifar.X:"f"$1_cifar.YX 16 | cifar_:`YX 17 | -1"reading test image dataset"; 18 | cifar.YXt:cifar.parse `$cifar.d,"test_batch.bin" 19 | cifar.yt:first cifar.Yt:"h"$1#cifar.YXt 20 | cifar.Xt:"f"$1_cifar.YXt 21 | cifar_:`YXt 22 | -------------------------------------------------------------------------------- /markov.q: -------------------------------------------------------------------------------- 1 | \l funq.q 2 | \l iris.q 3 | 4 | / markov clustering 5 | / https://www.cs.ucsb.edu/~xyan/classes/CS595D-2009winter/MCL_Presentation2.pdf 6 | 7 | / example from mcl man page 8 | / http://micans.org/mcl/man/mcl.html 9 | t:flip `k1`k2`v!"ssf"$\:() 10 | t,:`cat`hat,0.2 11 | t,:`hat`bat,0.16 12 | t,:`bat`cat,1.0 13 | t,:`bat`bit,0.125 14 | t,:`bit`fit,0.25 15 | t,:`fit`hit,0.5 16 | t,:`hit`bit,0.16 17 | 18 | / take max of bidirectional links, enumerate keys 19 | k:() 20 | m:.ml.inflate[1;0f] .ml.addloop m|:flip m:.ml.full enlist[2#count k],exec (v;`k?k1;`k?k2) from t 21 | .ut.assert[(`hat`bat`cat;`bit`fit`hit)] (get`k!) each .ml.interpret .ml.mcl[2;1.5;0f] over m 22 | 23 | / cluster the iris data 24 | sm:.5<.ml.gaussk[iris.X;.5] each flip iris.X / similarity matrix based on Gaussian kernel 25 | show .ml.interpret .ml.mcl[2;1.5;10] over .ml.inflate[1;0f] sm 26 | / are there 4 species: http://www.siam.org/students/siuro/vol4/S01075.pdf 27 | 28 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.gz 2 | *.zip 3 | *.data 4 | /ml-latest 5 | /ml-latest-small 6 | /t10k-images-idx3-ubyte 7 | /t10k-labels-idx1-ubyte 8 | /train-images-idx3-ubyte 9 | /train-labels-idx1-ubyte 10 | /cifar-10-batches-bin 11 | /ETL9B 12 | *.so 13 | /k.h 14 | *.o 15 | /pendigits.tes 16 | /pendigits.tra 17 | liblinear 18 | libsvm 19 | xgboost 20 | /OnlineNewsPopularity 21 | *.p[bgp]m 22 | /web-BerkStan.txt 23 | /*-citibike-tripdata.csv 24 | /citibike 25 | /sample-*.txt 26 | /stop-word-list.txt 27 | /1342-0.txt 28 | /2701.txt 29 | /10.txt 30 | /stop.txt 31 | /SMSSpamCollection 32 | /141.txt 33 | /158.txt 34 | /161.txt 35 | /121.txt 36 | /winequality-*.csv 37 | /voc.txt 38 | /output.txt 39 | /binary.csv 40 | /readme 41 | /tree.dot 42 | /tree.png 43 | /seeds_dataset.txt 44 | /105.txt 45 | /dim*.txt 46 | /s[1-4].txt 47 | /a[1-3].txt 48 | /dow_jones_index.* 49 | /optdigits.tes 50 | /optdigits.tra 51 | /Bitfinex_* 52 | /gemini_* 53 | /Kraken_* 54 | /Bitstamp_* 55 | /Binance_* 56 | /adult.test 57 | Gemfile.lock 58 | _site 59 | -------------------------------------------------------------------------------- /binance.q: -------------------------------------------------------------------------------- 1 | binance.p:string `daily`hourly`minutely!`d`1h`minute 2 | binance.c:string `BTCUSDT`ETHUSDT`LTCUSDT`NEOUSDT`BNBUSDT`XRPUSDT 3 | binance.c,:string `LINKUSDT`TRXUSDT`ETCUSDT`XLMUSDT`ZECUSDT 4 | binance.c,:string `ADAUSDT`QTUMUSDT`DASHUSDT`XMRUSDT`BTTUSDT 5 | binance.c,:string `ADABTC`ASTBTC`BTGBTC`DASHBTC`EOSBTC`ETCBTC`ETHBTC 6 | binance.c,:string `IOTABTC`LTCBTC`NEOBTC`XMRBTC`XLMBTC`XRPBTC 7 | binance.f:binance.p {"_" sv ("Binance";y;x,".csv")}/:\: asc binance.c 8 | binance.b:"http://www.cryptodatadownload.com/cdd/" 9 | -1"[down]loading binance data set"; 10 | .ut.download[binance.b;;"";""] each raze binance.f; 11 | .binance.load:{[f] 12 | if[not count t:("* *FFFFF I";1#",") 0: 1_read0 f;:()]; 13 | t:`time`sym`open`high`low`close`qty`n xcol t; 14 | t:update time:"P"$?[12>count each time;time;-3_/:time] from t; 15 | t:update sym:`$sym except\: "/" from t; 16 | t:`sym xcols 0!select by time from t; / remove duplicates 17 | t} 18 | binance,:({update `p#sym from x} raze .binance.load peach::)'[`$binance.f] 19 | -------------------------------------------------------------------------------- /kraken.q: -------------------------------------------------------------------------------- 1 | kraken.p:string `daily`hourly`minutely!`d`1h`minute 2 | kraken.c:string `BTCUSD`ETHUSD`LTCUSD`XRPUSD`LINKUSD`BCHUSD 3 | kraken.c,:string `DOTUSD`EOSUSD`ADAUSD`XMRUSD`DASHUSD`ETCUSD 4 | kraken.c,:string `ZECUSD`XTZUSD`TRXUSD`PAXGUSD`COMPUSD 5 | kraken.c,:string `BTCEUR`ETHEUR`LTCEUR`XRPEUR`LINKEUR`BCHEUR 6 | kraken.c,:string `DOTEUR`EOSEUR`ADAEUR`XMREUR`DASHEUR`ETCEUR 7 | kraken.c,:string `ZECEUR`XTZEUR`TRXEUR`PAXGEUR`COMPEUR 8 | kraken.c,:string `ETHBTC`LTCBTC 9 | kraken.f:kraken.p {"_" sv ("Kraken";y;x,".csv")}/:\: asc kraken.c 10 | kraken.b:"http://www.cryptodatadownload.com/cdd/" 11 | -1"[down]loading kraken data set"; 12 | .ut.download[kraken.b;;"";""] each raze kraken.f; 13 | .kraken.load:{[f] 14 | if[not count t:("P *FFFFFF I";1#",") 0: 1_read0 f;:()]; 15 | t:`time`sym`open`high`low`close`vwap`qty`n xcol t; 16 | t:update sym:`$sym except\: "/" from t; 17 | t:`sym xcols 0!select by time from t; / remove duplicates 18 | t} 19 | kraken,:({update `p#sym from x} raze .kraken.load peach::)'[`$kraken.f] 20 | -------------------------------------------------------------------------------- /svm.q: -------------------------------------------------------------------------------- 1 | .svm.dll:`libsvm^.svm.dll^:`; / optional override 2 | 3 | .svm,:(.svm.dll 2: (`lib;1))` 4 | .svm,:`C_SVC`NU_SVC`ONE_CLASS`EPSILON_SVR`NU_SVR!"i"$til 5 5 | .svm,:`LINEAR`POLY`RBF`SIGMOID`PRECOMPUTED!"i"$til 5 6 | 7 | \d .svm 8 | 9 | param:(!) . flip ( 10 | (`svm_type;C_SVC); 11 | (`kernel_type;RBF); 12 | (`degree;3i); 13 | (`gamma;-1f); / use defaults 14 | (`coef0;0f); 15 | (`cache_size;100f); 16 | (`eps;.001); 17 | (`C;1f); 18 | (`weight_label;::); 19 | (`weight;::); 20 | (`nu;.5); 21 | (`p;.1); 22 | (`shrinking;1i); 23 | (`probability;0i)) 24 | 25 | defparam:{[prob;param] 26 | if[0f>param`gamma;param[`gamma]:1f%max(last key@)each prob`x]; 27 | param} 28 | 29 | sparse:{{("i"$1+i)!x i:where not 0f=x} each flip x} 30 | prob:{`x`y!(sparse x;y)} 31 | 32 | read_problem:{[s] 33 | i:s?\:" "; 34 | y:i#'s; 35 | x:{(!/)"I: "0:x _y}'[1+i;s]; 36 | if[3.5>.z.K;x:("i"$key x)!value x]; 37 | `x`y!"F"$(x;y)} 38 | write_problem:{ 39 | s:(("+";"")0>x`y),'string x`y; 40 | s:s,'" ",/:{" " sv ":" sv' string flip(key x;value x)} each x`x; 41 | s:s,\:" "; 42 | s} 43 | -------------------------------------------------------------------------------- /testsvm.q: -------------------------------------------------------------------------------- 1 | \l svm.q 2 | \l ut.q 3 | .svm.set_print_string_function ` 4 | .ut.assert[323i] .svm.version 5 | .ut.assert[s] .svm.write_problem prob:.svm.read_problem s:read0 `:libsvm/heart_scale 6 | .ut.assert[::] .svm.check_parameter[prob] param:.svm.defparam[prob] .svm.param 7 | .ut.assert[prob] .svm.prob_inout prob 8 | m1:.svm.train[prob;param] 9 | m2:.svm.load_model `:libsvm/heart_scale.model 10 | do[1000;m:.svm.load_model `:libsvm/heart_scale.model] 11 | m3:{.svm.save_model[`model] x;m:.svm.load_model[`model];hdel`:model;m} m 12 | mp:`svm_type`kernel_type`gamma 13 | .ut.assert[@[m;`param;{y#x};mp]] @[m;`param;{y#x};mp] 14 | do[1000;param ~ .svm.param_inout param] 15 | .ut.assert[m] .svm.model_inout m 16 | do[1000;.svm.model_inout m] 17 | .ut.assert[1b] .8count each time;time;-3_/:time] from t; 18 | t:update sym:`$sym except\: "/" from t; 19 | t:`sym xcols 0!select by time from t; / remove duplicates 20 | t} 21 | bitfinex,:({update `p#sym from x} raze .bitfinex.load peach::)'[`$bitfinex.f] 22 | -------------------------------------------------------------------------------- /testlinear.q: -------------------------------------------------------------------------------- 1 | \l linear.q 2 | \l ut.q 3 | .linear.set_print_string_function ` 4 | .ut.assert[230i] .linear.version 5 | .ut.assert[s] .linear.write_problem prob:.linear.read_problem s:read0 `:liblinear/heart_scale 6 | .ut.assert[::] .linear.check_parameter[prob] param:.linear.defparam[prob] .linear.param 7 | .ut.assert[prob] .linear.prob_inout prob 8 | m1:.linear.train[prob;param] 9 | m2:.linear.load_model `:liblinear/heart_scale.model 10 | do[1000;m:.linear.load_model `:liblinear/heart_scale.model] 11 | m3:{.linear.save_model[`model] x;m:.linear.load_model[`model];hdel`:model;m} m 12 | mp:1#`solver_type 13 | .ut.assert[@[m;`param;{y#x};mp]] @[m;`param;{y#x};mp] 14 | do[1000;param ~ b:.linear.param_inout param] 15 | .ut.assert[m] .linear.model_inout m 16 | do[1000;.linear.model_inout m] 17 | .ut.assert[1b].75=param`eps;param[`eps]:defeps param`solver_type]; 35 | param} 36 | 37 | sparse:{{("i"$1+i)!x i:where not 0f=x} each flip x} 38 | prob:{`x`y!(sparse x;y)} 39 | 40 | read_problem:{[s] 41 | i:s?\:" "; 42 | y:i#'s; 43 | x:{(!/)"I: "0:x _y}'[1+i;s]; 44 | if[3.5>.z.K;x:("i"$key x)!value x]; 45 | `bias`x`y!-1f,"F"$(x;y)} 46 | write_problem:{ 47 | s:(("+";"")0>x`y),'string x`y; 48 | s:s,'" ",/:{" " sv ":" sv' string flip(key x;value x)} each x`x; 49 | s:s,\:" "; 50 | s} 51 | -------------------------------------------------------------------------------- /silhouette.q: -------------------------------------------------------------------------------- 1 | \c 20 77 2 | \l funq.q 3 | \l zoo.q 4 | \l iris.q 5 | 6 | -1"computing the silhouette demonstrates cluster quality"; 7 | -1"by generating a statistic that varies between -1 and 1"; 8 | -1"where 1 indicates a point is very close to all the items"; 9 | -1"within its own cluster and very far from all the items"; 10 | -1"in the next-best cluster while -1 indicates the reverse"; 11 | -1"a negative value indicates a point is closer to the next-best cluster"; 12 | -1""; 13 | -1"we now apply silhouette analysis to the zoo data set"; 14 | df:`.ml.edist 15 | -1"using distance metric: ", string df; 16 | t:(2#/:zoo.t),'([]silhouette:.ml.silhouette[df;zoo.X;zoo.y]) 17 | -1"sorting by avg silhouette within each cluster"; 18 | -1"then by actual data point silhouette value"; 19 | -1"provides good intuition on cluster quality"; 20 | show select[>([](avg;silhouette) fby typ;silhouette)] from t 21 | -1"assert average silhouette"; 22 | .ut.assert[.3] .ut.rnd[.1] exec avg silhouette from t 23 | -1"we see that mammals platypus, seal, dolphin and porpoise"; 24 | -1"as well as all the reptiles are better classified"; 25 | -1"as another type"; 26 | show 75_select[>([](avg;silhouette) fby typ;silhouette)] from t 27 | 28 | -1"we can run the same analysis on the iris data set"; 29 | t:iris.t,'([]silhouette:.ml.silhouette[df;iris.X;iris.y]) 30 | -1"we see that iris-setosa is the best cluster"; 31 | -1"and iris-versicolor and iris-virginica are worse"; 32 | show select avg silhouette by species from t 33 | -1"assert average silhouette"; 34 | .ut.assert[.5] .ut.rnd[.1] exec avg silhouette from t 35 | -------------------------------------------------------------------------------- /plot.q: -------------------------------------------------------------------------------- 1 | \c 20 100 2 | \l funq.q 3 | \l dji.q 4 | 5 | / define a plotting function using 10 characters of gradation 6 | plt:.ut.plot[w:40;h:20;c:.ut.c10;sum] 7 | 8 | -1"plotting 1-dimensional dataset (sin x): x"; 9 | -1 value plt X:sin .01*til 1000; 10 | 11 | -1"plotting 2-dimensional dataset (uniform variates): (x;y)"; 12 | -1 value plt X:10000?/:2#1f; 13 | 14 | -1"plotting 2-dimensional dataset (normal variates): (x;y)"; 15 | -1 value plt (.ml.bm 10000?) each 2#1f; 16 | 17 | -1"plotting 3-dimensional dataset: (x;y;z)"; 18 | -1 value plt {(x;{x*x*x}x-.5;x:til[x]%x)} 1000; 19 | 20 | -1"plotting 3-dimensional grid as a heatmap: X (matrix)"; 21 | -1 value plt .ut.hmap {x*/:(x:til x)*(x;x)#1f} 1000; 22 | 23 | b:1b / use binary encoding for portable (bit|pix)map 24 | 25 | -1"plotting black/white Mandelbrot series"; 26 | c:.ut.tcross . (.ut.nseq .) each flip (-1+w:1000;-2 -1.25;.5 1.25) 27 | x:w cut .ml.mbrotp 20 .ml.mbrotf[c]/0f 28 | -1 value plt .ut.hmap x; 29 | -1"saving PBM image"; 30 | `mandel.pbm 0: .ut.pbm[b] x 31 | 32 | -1"plotting gray scale Mandelbrot series"; 33 | x:w cut last 20 .ml.mbrota[c]// (0f;0) 34 | -1 value plt .ut.hmap x; 35 | -1"saving PGM image"; 36 | `mandel.pgm 0: .ut.pgm[b;20] x 37 | 38 | -1"saving PPM image"; 39 | `mandel.ppm 0: .ut.ppm[b;20] flip[(rand 1+20;til 1+20;rand 1+20)] x 40 | 41 | -1"plotting sparkline of the dow jones index components"; 42 | exec -1 ((4$string first stock),": ",.ut.spark close) by stock from dji.t; 43 | 44 | / tests 45 | .ut.assert[1b] last[x]