├── .gitignore ├── .vscode └── extensions.json ├── Dojo ├── GuidedScript.fsx ├── Helper.fsx ├── trainingsample.csv └── validationsample.csv ├── LICENSE ├── Organizer ├── Digits-Recognizer.pptx ├── Organizer-Notes.md └── PossibleSolution.fsx └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | # Build Folders (you can keep bin if you'd like, to store dlls and pdbs) 2 | [Bb]in/ 3 | [Oo]bj/ 4 | 5 | # mstest test results 6 | TestResults 7 | 8 | ## Ignore Visual Studio temporary files, build results, and 9 | ## files generated by popular Visual Studio add-ons. 10 | 11 | # User-specific files 12 | *.suo 13 | *.user 14 | *.sln.docstates 15 | 16 | # Build results 17 | [Dd]ebug/ 18 | [Rr]elease/ 19 | x64/ 20 | *_i.c 21 | *_p.c 22 | *.ilk 23 | *.meta 24 | *.obj 25 | *.pch 26 | *.pdb 27 | *.pgc 28 | *.pgd 29 | *.rsp 30 | *.sbr 31 | *.tlb 32 | *.tli 33 | *.tlh 34 | *.tmp 35 | *.log 36 | *.vspscc 37 | *.vssscc 38 | .builds 39 | 40 | # Visual C++ cache files 41 | ipch/ 42 | *.aps 43 | *.ncb 44 | *.opensdf 45 | *.sdf 46 | 47 | # Visual Studio profiler 48 | *.psess 49 | *.vsp 50 | *.vspx 51 | 52 | # Guidance Automation Toolkit 53 | *.gpState 54 | 55 | # ReSharper is a .NET coding add-in 56 | _ReSharper* 57 | 58 | # NCrunch 59 | *.ncrunch* 60 | .*crunch*.local.xml 61 | 62 | # Installshield output folder 63 | [Ee]xpress 64 | 65 | # DocProject is a documentation generator add-in 66 | DocProject/buildhelp/ 67 | DocProject/Help/*.HxT 68 | DocProject/Help/*.HxC 69 | DocProject/Help/*.hhc 70 | DocProject/Help/*.hhk 71 | DocProject/Help/*.hhp 72 | DocProject/Help/Html2 73 | DocProject/Help/html 74 | 75 | # Click-Once directory 76 | publish 77 | 78 | # Publish Web Output 79 | *.Publish.xml 80 | 81 | # NuGet Packages Directory 82 | packages 83 | 84 | # Windows Azure Build Output 85 | csx 86 | *.build.csdef 87 | 88 | # Windows Store app package directory 89 | AppPackages/ 90 | 91 | # Others 92 | [Bb]in 93 | [Oo]bj 94 | sql 95 | TestResults 96 | [Tt]est[Rr]esult* 97 | *.Cache 98 | ClientBin 99 | [Ss]tyle[Cc]op.* 100 | ~$* 101 | *.dbmdl 102 | Generated_Code #added for RIA/Silverlight projects 103 | 104 | # Backup & report files from converting an old project file to a newer 105 | # Visual Studio version. Backup files are not needed, because we have git ;-) 106 | _UpgradeReport_Files/ 107 | Backup*/ 108 | UpgradeLog*.XML 109 | .ionide/ 110 | -------------------------------------------------------------------------------- /.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | "recommendations": [ 3 | "ionide.ionide-fsharp" 4 | ], 5 | "unwantedRecommendations": [] 6 | } -------------------------------------------------------------------------------- /Dojo/GuidedScript.fsx: -------------------------------------------------------------------------------- 1 | // This F# dojo is directly inspired by the 2 | // Digit Recognizer competition from Kaggle.com: 3 | // http://www.kaggle.com/c/digit-recognizer 4 | // The datasets below are simply shorter versions of 5 | // the training dataset from Kaggle. 6 | 7 | // The goal of the dojo will be to 8 | // create a classifier that uses training data 9 | // to recognize hand-written digits, and 10 | // evaluate the quality of our classifier 11 | // by looking at predictions on the validation data. 12 | 13 | // This file provides some guidance through the problem: 14 | // each section is numbered, and 15 | // solves one piece you will need. Sections contain 16 | // general instructions, 17 | // [ YOUR CODE GOES HERE! ] tags where you should 18 | // make the magic happen, and 19 | // blocks. These are small 20 | // F# tutorials illustrating aspects of the 21 | // syntax which could come in handy. Run them, 22 | // see what happens, and tweak them to fit your goals! 23 | 24 | 25 | // 0. GETTING READY 26 | 27 | // 28 | // With F# Script files (.fsx) and F# Interactive, 29 | // you can "live code" and see what happens. 30 | 31 | // Try typing let x = 42 in the script file, 32 | // right-click and select "Execute in interactive". 33 | // You can also hit ALT + ENTER on a line to send it to 34 | // F# interactive (FSI). This also works in VS Code. 35 | 36 | // let "binds" the value on the right to a name. 37 | 38 | // Now execute the following lines in FSI (highlight both 39 | // lines and execute them "together"): 40 | let greet name = 41 | printfn "Hello, %s" name 42 | 43 | // let also binds a name to a function. 44 | // greet is a function with one argument, name. 45 | // You should be able to call this function by entering 46 | // the following and sending it to FSI: 47 | // greet "World" 48 | // 49 | 50 | // Two data files are included in the same place you 51 | // found this script: 52 | // trainingsample.csv, a file that contains 5,000 examples, and 53 | // validationsample.csv, a file that contains 500 examples. 54 | // The first file will be used to train your model, and the 55 | // second one to validate the quality of the model. 56 | 57 | // 1. GETTING SOME DATA 58 | 59 | // First let's read the contents of "trainingsample.csv" 60 | 61 | // We will need System and System.IO to work with files, 62 | // let's right-click / run in interactive, 63 | // to have these namespaces loaded: 64 | 65 | open System 66 | open System.IO 67 | 68 | // the following might come in handy: 69 | //File.ReadAllLines(path) 70 | // returns an array of strings for each line 71 | 72 | // [ YOUR CODE GOES HERE! ] 73 | 74 | 75 | // 2. EXTRACTING COLUMNS 76 | 77 | // Break each line of the file into an array of string, 78 | // separating by commas, using Array.map 79 | 80 | // 81 | // Array.map quick-starter: 82 | // Array.map takes an array, and transforms it 83 | // into another array by applying a function to it. 84 | // Example: starting from an array of strings: 85 | let strings = [| "Machine"; "Learning"; "with"; "F#"; "is"; "fun" |] 86 | 87 | // We can transform it into a new array, 88 | // containing the length of each string: 89 | let lengths = Array.map (fun (s:string) -> s.Length) strings 90 | // The exact same operation above can be 91 | // done using the forward pipe operator, 92 | // which makes it look nicer: 93 | let lengths2 = strings |> Array.map (fun s -> s.Length) 94 | // 95 | 96 | // The following function might help 97 | let csvToSplit = "1,2,3,4,5" 98 | let splitResult = csvToSplit.Split(',') 99 | 100 | 101 | // [ YOUR CODE GOES HERE! ] 102 | 103 | 104 | // 3. CLEANING UP HEADERS 105 | 106 | // Did you note that the file has headers? We want to get rid of it. 107 | 108 | // 109 | // Array slicing quick starter: 110 | // Let's start with an Array of ints: 111 | let someNumbers = [| 0 .. 10 |] // create an array from 0 to 10 112 | // You can access Array elements by index: 113 | let first = someNumbers.[0] 114 | // You can also slice the array: 115 | let twoToFive = someNumbers.[ 1 .. 4 ] // grab a slice 116 | let upToThree = someNumbers.[ .. 2 ] 117 | // 118 | 119 | 120 | // [ YOUR CODE GOES HERE! ] 121 | 122 | 123 | // 4. CONVERTING FROM STRINGS TO INTS 124 | 125 | // Now that we have an array containing arrays of strings, 126 | // and the headers are gone, we need to transform it 127 | // into an array of arrays of integers. 128 | // Array.map seems like a good idea again :) 129 | 130 | // The following might help: 131 | let castedInt = (int)"42" 132 | // or, alternatively: 133 | let convertedInt = Convert.ToInt32("42") 134 | 135 | 136 | // [ YOUR CODE GOES HERE! ] 137 | 138 | 139 | // 5. CONVERTING ARRAYS TO RECORDS 140 | 141 | // Rather than dealing with a raw array of ints, 142 | // for convenience let's store these into an array of Records 143 | 144 | // 145 | // Record quick starter: we can declare a 146 | // Record (a lightweight, immutable class) type that way: 147 | type Example = { Label:int; Pixels:int[] } 148 | // and instantiate one this way: 149 | let example = { Label = 1; Pixels = [| 1; 2; 3; |] } 150 | // 151 | 152 | 153 | // [ YOUR CODE GOES HERE! ] 154 | 155 | // 5.1 VISUALISING THE DATA 156 | 157 | // You can visualise the data for an observation using 158 | // a helper function in this repository. First, you can 159 | // "load" the helper module using the #load FSI command: 160 | //#load "Helper.fsx" 161 | // This module contains a function, prettyPrint, which 162 | // prints an ASCII art representation of the number 163 | // directly into the REPL e.g. 164 | // prettyPrint example.Pixels 165 | 166 | // [ YOUR CODE GOES HERE! ] 167 | 168 | // 6. COMPUTING DISTANCES 169 | 170 | // We need to compute the "distance" between images 171 | // Math reminder: the euclidean distance is 172 | // distance [ x1; y1; z1 ] [ x2; y2; z2 ] = 173 | // sqrt((x1-x2)*(x1-x2) + (y1-y2)*(y1-y2) + (z1-z2)*(z1-z2)) 174 | 175 | // 176 | // Array.map2 could come in handy here. 177 | // Suppose we have 2 arrays: 178 | let point1 = [| 0; 1; 2 |] 179 | let point2 = [| 3; 4; 5 |] 180 | // Array.map2 takes 2 arrays at a time 181 | // and maps pairs of elements, for instance: 182 | let map2Example = 183 | Array.map2 (fun p1 p2 -> p1 + p2) point1 point2 184 | // This simply computes the sums for point1 and point2, 185 | // but we can easily turn this into a function now: 186 | let map2PointsExample (P1: int[]) (P2: int[]) = 187 | Array.map2 (fun p1 p2 -> p1 + p2) P1 P2 188 | // 189 | 190 | // Having a function like 191 | let distance (p1: int[]) (p2: int[]) = 42 192 | // would come in very handy right now, 193 | // except that in this case, 194 | // 42 is likely not the right answer 195 | 196 | // [ YOUR CODE GOES HERE! ] 197 | 198 | 199 | // 7. WRITING THE CLASSIFIER FUNCTION 200 | 201 | // We are now ready to write a classifier function! 202 | // The classifier should take a set of pixels 203 | // (an array of ints) as an input, search for the 204 | // closest example in our sample, and use 205 | // that value as the prediction. 206 | 207 | // 208 | // Array.minBy can be handy here, to find 209 | // the closest element in the Array of examples. 210 | // Suppose we have an Array of Example: 211 | let someData = 212 | [| { Label = 0; Pixels = [| 0; 1 |] }; 213 | { Label = 1; Pixels = [| 9; 2 |] }; 214 | { Label = 2; Pixels = [| 3; 4 |] }; |] 215 | // We can find for instance 216 | // the element with largest first pixel 217 | let findThatGuy = 218 | someData 219 | |> Array.maxBy (fun x -> x.Pixels.[0]) 220 | // 221 | 222 | 223 | // 224 | // F# and closures work very well together 225 | let immutableValue = 42 226 | let functionWithClosure (x: int) = 227 | if x > immutableValue // using outside value 228 | then true 229 | else false 230 | // 231 | 232 | 233 | // The classifier function should probably 234 | // look like this - except that this one will 235 | // classify everything as a 0: 236 | let classify (unknown:int[]) = 237 | // do something smart here 238 | // like find the Example with 239 | // the shortest distance to 240 | // the unknown element... 241 | // and use the training examples 242 | // in a closure... 243 | 0 244 | 245 | // [ YOUR CODE GOES HERE! ] 246 | 247 | 248 | // 8. EVALUATING THE MODEL AGAINST VALIDATION DATA 249 | 250 | // Now that we have a classifier, we need to check 251 | // how good it is. 252 | // This is where the 2nd file, validationsample.csv, 253 | // comes in handy. 254 | // For each Example in the 2nd file, 255 | // we know what the true Label is, so we can compare 256 | // that value with what the classifier says. 257 | // You could now check for each 500 example in that file 258 | // whether your classifier returns the correct answer, 259 | // and compute the % correctly predicted. 260 | 261 | 262 | // [ YOUR CODE GOES HERE! ] -------------------------------------------------------------------------------- /Dojo/Helper.fsx: -------------------------------------------------------------------------------- 1 | [] 2 | module Helper 3 | 4 | /// Pretty prints the array representation of a number to the console using ASCII art. 5 | let prettyPrint (observation:int []) = 6 | let printValue = 7 | let mapping = "$@B%8&WM#*oahkbdpqwmZO0QLCJUYXzcvunxrjft/\|()1{}[]?-_+~<>i!lI;:,\"^`'. ".ToCharArray() |> Array.rev 8 | let multiplier = float (mapping.Length - 1) 9 | fun (n:int) -> float n / 256. * multiplier |> int |> Array.get mapping 10 | observation 11 | |> Array.map printValue 12 | |> Array.chunkBySize 28 13 | |> Array.map System.String 14 | |> Array.iter (printfn "%s") 15 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Community for F# 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /Organizer/Digits-Recognizer.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/c4fsharp/Dojo-Digits-Recognizer/1971a1ee37dc978cf5c5bb8b1dbc621e0c6f0931/Organizer/Digits-Recognizer.pptx -------------------------------------------------------------------------------- /Organizer/Organizer-Notes.md: -------------------------------------------------------------------------------- 1 | #Script: Comments & Stumbling Blocks 2 | 3 | Regularly, people (especially if they came in late and missed the introduction...) don't realize that the first number in each row is the actual digit, i.e. the number itself represented by the following digits, its representation in pixels. 4 | 5 | ##Section 1. 6 | 7 | People often forget to run the open statements. 8 | 9 | ##Section 4. 10 | 11 | People often get stuck on the idea to have a double map, like this: 12 | data |> Array.map (fun row -> row |> Array.map (fun x -> (int)x)) 13 | They also often forget to pipe row into the inside map, resulting in an array of unapplied functions. 14 | A possibly better way to convert to ints is to avoid casting, and simply use int: 15 | data |> Array.map (fun row -> row |> Array.map int) 16 | 17 | ##Section 5. 18 | 19 | People often get stuck on the fact that instantiating a record is as simple as { Label = 1; Pixels = [| 1; 2; 3; |] } 20 | 21 | ##Section 6. 22 | 23 | People sometimes get confused by the example, which shows a distance on 3-elements. The idea is to generalize to arbitrary length arrays. 24 | People often stumble on Array.map2; arguably, Array.map2 is not great, in that it breaks the pattern established before, namely data |> Array.map. The pipe forward operator doesn't work here. Rather than introducing the ||> operator or the concept of tuples, I opted for Array.map2, but this tends to be one of the places where people struggle. 25 | 26 | ##Section 7 and 8 27 | 28 | People often get confused on how to use the training and validation sets, and start doing complicated computations which lead nowhere. The idea here is simple: the only way to evaluate the quality of the classifier is to give it data it has not seen before, and see how much it gets right. So take each example in the validation set, pass it to the classifier and see what it predicts, and compare that to the known, correct value. 29 | Also, people sometimes don't realize that the validation set contains known data as well, i.e. observations where we know from the start what the correct value is. 30 | 31 | Typically, when people get a classifier which gets 100% correct, it indicates that they are classifying the training set itself, and not the validation set :) 32 | 33 | The expected correct rate is 94.4% 34 | 35 | #Possible Conclusion / Opening 36 | 37 | Obviously, getting 94.4% correct in 25-ish lines of code is not bad :) 38 | For F# beginners, it's worth pointing out that managing to write a classifier in a language they have not used before is no small feat. 39 | 40 | While 94.4% is nice, it's not "done". At that point, 2 directions are possible: 41 | 1) the real dataset contains 50,000 elements and not 5,000, which will go 10x slower. One direction is speeding up the algorithm, using maybe Array.parallel.map, or specialized data structures, or other ideas. 42 | 2) the model that was built is "the simplest thing that could possibly works", and works pretty well. How can we squeeze more accuracy out of it? There are numerous directions possible, and the only way to know is to just try it, which is why having a validation set (cross-validation) is hugely important: it provides a benchmark for "is the model better or not?". Possible explorations, following the initial model, are: trying 1, 2, 3, ... neighbors (which doesn't work well), trying different distances (which works well but causes technical issues like integer overflow with certain distances), blurring (if you take an image and move it one pixel to the right, the distance might degrade a lot - blurring can compensate for that). 43 | -------------------------------------------------------------------------------- /Organizer/PossibleSolution.fsx: -------------------------------------------------------------------------------- 1 | #load @"..\Dojo\Helper.fsx" 2 | open System.IO 3 | 4 | type Image = int [] 5 | type Observation = { Label:int; Pixels:Image } 6 | type Model = Image -> int 7 | 8 | let euclDistance (img1:Image) (img2:Image) = 9 | (img1, img2) 10 | ||> Seq.map2 (fun x y -> (x-y) * (x-y)) 11 | |> Seq.sum 12 | 13 | let train trainingSet (img:Image) = 14 | let obs = 15 | trainingSet 16 | |> Array.minBy (fun observation -> 17 | euclDistance observation.Pixels img) 18 | obs.Label 19 | 20 | let dropHeader (x:_[]) = x.[1..] 21 | 22 | let read path = 23 | File.ReadAllLines(path) 24 | |> dropHeader 25 | |> Array.map (fun line -> 26 | let numbers = line.Split ',' |> Array.map int 27 | { Label = numbers.[0] 28 | Pixels = numbers.[1..] }) 29 | 30 | let trainingPath = @"Dojo\trainingsample.csv" 31 | let trainingData = read trainingPath 32 | let basicModel = train trainingData 33 | let validationPath = @"Dojo\validationsample.csv" 34 | let validationData = read validationPath 35 | 36 | let evaluate (model:Model) = 37 | validationData 38 | |> Array.Parallel.map (fun observation -> 39 | if (model (observation.Pixels) = observation.Label) then 1. 40 | else 0.) 41 | |> Array.average 42 | 43 | evaluate basicModel 44 | 45 | // If you want to go further... 46 | // * Try out 1, 2, .. n neighbors? 47 | // * Try out different distances: 48 | // Manhattan distance: abs(x1-y1 + x2-y2 + ...), 49 | // "Generalized" distances: abs ((x1-y1)^k + (x2-y2)^k + ...) 50 | // * Try out "blurring" images 51 | // * Try out rescaling pixels to less values, eg. map 0.. 255 to 0..7, 0..15 52 | // Go nuts! 53 | 54 | // Blurring: instead of a single pixel, 55 | // we take n x n pixels and average then out. 56 | 57 | let size = 28 58 | // compute array index of pixel at (row,col) 59 | let offset row col = (row * size) + col 60 | // compute average over square tile; 61 | // note the array comprehension syntax. 62 | let blur (img:Image) row col rad = 63 | [| for x in 0 .. (rad-1) do 64 | for y in 0 .. (rad-1) do 65 | yield img.[offset (row+x) (col+y)] |] 66 | |> Array.sum 67 | 68 | let blurred n img = 69 | [| for row in 0 .. (size - n) do 70 | for col in 0 .. (size - n) do 71 | yield blur img row col n |] 72 | 73 | 74 | // Experiment: try out (hopefully) "better" models! -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Dojo "Digits Recognizer" 2 | ====================== 3 | 4 | This Dojo is inspired by the [Kaggle Digit Recognizer contest](http://www.kaggle.com/c/digit-recognizer). 5 | The goal is to write a **Machine Learning** classifier from scratch, a program that will recognize hand-written digits automatically. 6 | It is a guided Dojo, **suitable for beginners**: the Dojo comes with a Script file, with specific tasks to complete, introducing along the way numerous F# concepts and syntax examples. 7 | The Dojo is on the longer side; for a group with limited F# experience, it is recommended to **schedule it for 2 hours and a half**. 8 | --------------------------------------------------------------------------------