├── .gitignore ├── AzureML-workshopv4.pptx ├── MicroStrategy ├── 2017MSTRWorldAMLWorkshop.docx ├── 2017MSTRWorldAMLWorkshop.pdf ├── FlightData.xlsx ├── FlightML.R ├── Microsoft Azure Workshop.mstr ├── R Metrics Expressions for Microsoft ML workshop.txt ├── R Metrics for Microsoft ML workshop.pdf └── README.md ├── README.md ├── hold ├── 4a.png └── 4c.png └── images ├── 10a.png ├── 11a.png ├── 12 algotry.png ├── 12graph.png ├── 12true.png ├── 12truepos.png ├── 2ANewExperiment.png ├── 2bBlankExperiment.png ├── 2dTitle.png ├── 3A.png ├── 3b.png ├── 3cdataset.png ├── 3cvis.png ├── 4a.png ├── 4b.png ├── 4c.png ├── 4e.png ├── 5a.png ├── 5b.png ├── 6a.png ├── 6b.png ├── 7a.png ├── 8a.png ├── 9a.png └── new4d.png /.gitignore: -------------------------------------------------------------------------------- 1 | ~*.* -------------------------------------------------------------------------------- /AzureML-workshopv4.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bethz/AzureML-FlightPrediction/d27301e8c47770991bd6a3695795a9e69d6929e4/AzureML-workshopv4.pptx -------------------------------------------------------------------------------- /MicroStrategy/2017MSTRWorldAMLWorkshop.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bethz/AzureML-FlightPrediction/d27301e8c47770991bd6a3695795a9e69d6929e4/MicroStrategy/2017MSTRWorldAMLWorkshop.docx -------------------------------------------------------------------------------- /MicroStrategy/2017MSTRWorldAMLWorkshop.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bethz/AzureML-FlightPrediction/d27301e8c47770991bd6a3695795a9e69d6929e4/MicroStrategy/2017MSTRWorldAMLWorkshop.pdf -------------------------------------------------------------------------------- /MicroStrategy/FlightData.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bethz/AzureML-FlightPrediction/d27301e8c47770991bd6a3695795a9e69d6929e4/MicroStrategy/FlightData.xlsx -------------------------------------------------------------------------------- /MicroStrategy/FlightML.R: -------------------------------------------------------------------------------- 1 | #MICROSTRATEGY_BEGIN 2 | # 3 | #RVAR year -input -numeric -vector 4 | #RVAR quarter -input -numeric -vector 5 | #RVAR month -input -numeric -vector 6 | #RVAR dayofmonth -input -numeric -vector 7 | #RVAR dayofweek -input -numeric -vector 8 | #RVAR carrier -input -string -vector 9 | #RVAR originairportid -input -vector 10 | #RVAR destairportid -input -numeric -vector 11 | #RVAR crsdeptime -input -numeric -vector 12 | #RVAR deptimeblk -input -string -vector 13 | #RVAR depdelay -input -numeric -vector 14 | #RVAR depdel15 -input -numeric -vector 15 | #RVAR crsarrtime -input -numeric -vector 16 | #RVAR arrtimeblk -input -string -vector 17 | #RVAR arrdelay -input -numeric -vector 18 | #RVAR arrdel15 -input -numeric -vector 19 | #RVAR cancelled -input -numeric -vector 20 | #RVAR diverted -input -numeric -vector 21 | # 22 | #RVAR outputParam -parameter StringParam1 23 | # 24 | #RVAR predict_result -output -string -vector #Metric Expression: RScript<_RScriptFile="C:\MSTRUsher.R", _InputNames="year, quarter, month, dayofmonth, dayofweek, carrier, originairportid, destairportid, crsdeptime, deptimeblk, depdelay, depdel15, crsarrtime, arrtimeblk, arrdelay, arrdel15, cancelled, diverted">(year, quarter, month, dayofmonth, dayofweek, carrier, originairportid, destairportid, crsdeptime, deptimeblk, depdelay, depdel15, crsarrtime, arrtimeblk, arrdelay, arrdel15, cancelled, diverted) 25 | if(exists("mstr.WorkingDir")) setwd(mstr.WorkingDir) #Working Directory if executed by MicroStrategy 26 | # 27 | #MICROSTRATEGY_END 28 | 29 | 30 | #Check to see if package(s) are installed, install if not and then load 31 | CheckInstallPackages <- function(pkgs) { #pkgs is a vector of strings with length >= 1 32 | x <- lapply(pkgs, function(pkg){ #For each pkg in pkgs (attempt to load each package one at a time): 33 | if(!do.call("require", list(pkg))) { # Load the package if available, 34 | try(install.packages(pkg, lib=.Library, 35 | repos="http://cran.rstudio.com")) # Silently attempt to install into the default library 36 | tryCatch(do.call("library", list(pkg)), # Now attempt to load the package, catch error if it wasn't installed 37 | error = function(err) { # Catch if we're unable to install into the default library 38 | if(!interactive()) { # If non-interactive, install into this user's personal library 39 | personalLibPath <- Sys.getenv("R_LIBS_USER") # Get the path to this user's personal library 40 | if(is.na(match(personalLibPath, .libPaths()))) { # If the personal library is not in the list of libraries 41 | dir.create(personalLibPath, recursive = TRUE) # Then create the personal library 42 | .libPaths(personalLibPath) # And add the personal library to the list of libraries 43 | } 44 | install.packages(pkg, lib=personalLibPath, # Attempt to install the package into the personal library 45 | repos="http://cran.rstudio.com") # if this fails, raise the error back to the report 46 | do.call("library", list(pkg)) # Finally, attempt to load the package 47 | } 48 | } 49 | ) 50 | } 51 | }) 52 | } 53 | CheckInstallPackages(c("RCurl", 'rjson')) 54 | # Accept SSL certificates issued by public Certificate Authorities 55 | options(RCurlOptions = list(cainfo = system.file("CurlSSL", "cacert.pem", package = "RCurl"))) 56 | h = basicTextGatherer() 57 | hdr = basicHeaderGatherer() 58 | #Covert data types for several input parameters 59 | year <- as.character(year) 60 | quarter <- as.character(quarter) 61 | month <- as.character(month) 62 | dayofmonth <- as.character(dayofmonth) 63 | dayofweek <- as.character(dayofweek) 64 | originairportid <- as.character(originairportid) 65 | destairportid <- as.character(destairportid) 66 | crsdeptime <- as.character(crsdeptime) 67 | depdelay <- as.character(depdelay) 68 | depdel15 <- as.character(depdel15) 69 | crsarrtime <- as.character(crsarrtime) 70 | arrdelay <- as.character(arrdelay) 71 | arrdel15 <- as.character(arrdel15) 72 | cancelled <- as.character(cancelled) 73 | diverted <- as.character(diverted) 74 | # Define the request 75 | input <- list() 76 | for(i in (1: length(year))) 77 | input[[i]] <- list( 78 | 'Year' = year[i], 79 | 'Quarter' = quarter[i], 80 | 'Month' = month[i], 81 | 'DayofMonth' = dayofmonth[i], 82 | 'DayOfWeek' = dayofweek[i], 83 | 'Carrier' = carrier[i], 84 | 'OriginAirportID' = originairportid[i], 85 | 'DestAirportID' = destairportid[i], 86 | 'CRSDepTime' = crsdeptime[i], 87 | 'DepTimeBlk' = deptimeblk[i], 88 | 'DepDelay' = depdelay[i], 89 | 'DepDel15' = depdel15[i], 90 | 'CRSArrTime' = crsarrtime[i], 91 | 'ArrTimeBlk' = arrtimeblk[i], 92 | 'ArrDelay' = arrdelay[i], 93 | 'ArrDel15' = arrdel15[i], 94 | 'Cancelled' = cancelled[i], 95 | 'Diverted' = diverted[i] 96 | ) 97 | req = list( 98 | Inputs = list( 99 | "input1"= input 100 | ), 101 | GlobalParameters = setNames(fromJSON('{}'), character(0)) 102 | ) 103 | body = enc2utf8(toJSON(req)) 104 | api_key = "u78j7hyQbp1MuzSvaIdJ58eGaHvxOoby3X233VTAYGExbT9Fy3mpd9wlWK5KhxNMSiFDmLFcX4pyg1h0gRl56g==" # Replace this with the API key for the web service 105 | authz_hdr = paste('Bearer', api_key, sep=' ') 106 | h$reset() 107 | curlPerform(url = "https://ussouthcentral.services.azureml.net/workspaces/fee0bd5c127e471c88ab563ee215101c/services/999b501513db4859b49750eda2aab3e8/execute?api-version=2.0&details=true&format=swagger", 108 | httpheader=c('Content-Type' = "application/json", 'Authorization' = authz_hdr), 109 | postfields=body, 110 | writefunction = h$update, 111 | headerfunction = hdr$update, 112 | verbose = TRUE 113 | ) 114 | headers = hdr$value() 115 | httpStatus = headers["status"] 116 | if (httpStatus >= 400) 117 | { 118 | print(paste("The request failed with status code:", httpStatus, sep=" ")) 119 | # Print the headers - they include the requert ID and the timestamp, which are useful for debugging the failure 120 | print(headers) 121 | } 122 | print("Result:") 123 | result = h$value() 124 | #Collect the result 125 | output <- fromJSON(result)$Results$output1 126 | predict_result <- character(length(year)) 127 | for(i in (1 : length(year))) 128 | predict_result[i] <- output[[i]][[outputParam]] 129 | -------------------------------------------------------------------------------- /MicroStrategy/Microsoft Azure Workshop.mstr: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bethz/AzureML-FlightPrediction/d27301e8c47770991bd6a3695795a9e69d6929e4/MicroStrategy/Microsoft Azure Workshop.mstr -------------------------------------------------------------------------------- /MicroStrategy/R Metrics Expressions for Microsoft ML workshop.txt: -------------------------------------------------------------------------------- 1 | Metric 1: 2 | 3 | Predicted: Scored Labels 4 | 5 | RScript(Year, Quarter, Month, [Day of Month], [Day of Week], Carrier, [Origin Airport ID], [Destination Airport ID], [CRS Departure Time], [Departure Time Block], [Departure Delay (min)], [Departure Delay Indicator], [CRS Arrival Time], [Arrival Time Block], [Arrival Delay (min)], [Arrival Delay Indicator], [Cancelled Flight Indicator], [Diverted Flight Indicator]) 6 | 7 | 8 | 9 | Metric 2: 10 | 11 | Predicted Flight Status 12 | 13 | CaseV([Predicted: Scored Labels], "0", "On‐Time", "Delay") 14 | 15 | -------------------------------------------------------------------------------- /MicroStrategy/R Metrics for Microsoft ML workshop.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bethz/AzureML-FlightPrediction/d27301e8c47770991bd6a3695795a9e69d6929e4/MicroStrategy/R Metrics for Microsoft ML workshop.pdf -------------------------------------------------------------------------------- /MicroStrategy/README.md: -------------------------------------------------------------------------------- 1 | # How to Build & Visualize Machine Learning Predictions with Microsoft Azure & MicroStrategy 2 | 3 | Haven’t you always wondered where machine learning could actually be used by you? 4 | Imagine a powerful cloud-based machine learning application that’s created by a simple drag-n-drop interface along with powerful self-service analytics to provide useful insights. 5 | What about using publicly available flight information to learn how to make a prediction using machine learning if a flight is likely to be delayed? 6 | 7 | And then get this to work in about an hour? 8 | 9 | Microsoft and MicroStrategy provide a step-by-step workshop to show how easy it is to create and use your own Azure ML Model with MicroStrategy Desktop to visualize your prediction as part of a hands-on workshop at MicroStrategy World in Washington, DC from April 18-20. 10 | The three main tools used are Microsoft R Open, Azure Machine Learning and MicroStrategy Desktop. 11 | 12 | # Microsoft R Open 13 | 14 | Microsoft R Open, formerly known as Revolution R Open (RRO), is the enhanced distribution of R from Microsoft Corporation. It is a complete open source platform for statistical analysis and data science. The current version, Microsoft R Open 3.3.2, is based on (and 100% compatible with) R-3.3.2, the most widely used statistics software in the world, and is therefore fully compatible with all packages, scripts, and applications that work with that version of R. It includes additional capabilities for improved performance, reproducibility, as well as support for Windows and Linux-based platforms. 15 | Like R, Microsoft R Open is open source and free to download, use, and share. It is available from https://mran.microsoft.com/open/ . 16 | 17 | # Azure Machine Learning 18 | 19 | Data can hold secrets, especially if you have lots of it. With lots of data about something, you can examine that data in intelligent ways to find patterns. And those patterns, which are typically too complex for you to detect yourself, can tell you how to solve a problem. 20 | This is exactly what machine learning does: It examines large amounts of data looking for patterns, then generates code that lets you recognize those patterns in new data. Your applications can use this generated code to make better predictions. In other words, machine learning can help you create smarter applications. Azure Machine Learning enables you to build powerful, cloud-based machine learning applications. 21 | 22 | # MicroStrategy Desktop 23 | 24 | MicroStrategy Desktop provides powerful self-service analytics that work seamlessly with Microsoft R Open and Azure Machine Learning to uncover insights. People answer their toughest questions with MicroStrategy. 25 | A small R script is provided to demonstrate how to connect to MicroStrategy Desktop and make an insightful dashboard to visualize your prediction. 26 | 27 | Come join us at MicroStrategy World or try the workshop out for yourself. 28 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # AzureML-FlightPrediction 2 | This is detailed documentation for the Azure Machine Learning, ML, tutorial/workshop on flight prediction. 3 | 4 | # Azure Machine Learning Challenge for Flight Prediction 5 | 6 | This Learning Challenge uses Flight data to predict when a flight will be late. We begin by accessing Azure ML. Here is an overview of what we will do: 7 | 8 | [1. Access Azure ML](https://github.com/bethz/AzureML-FlightPrediction#1-access-azure-ml) 9 | [2. Create an Experiment](https://github.com/bethz/AzureML-FlightPrediction/blob/master/README.md#10-run-experiment) 10 | [3. Import, Review and Clean Data](https://github.com/bethz/AzureML-FlightPrediction/blob/master/README.md#3-import-review-and-clean-data) 11 | [4. Specify Columns to Use](https://github.com/bethz/AzureML-FlightPrediction/blob/master/README.md#4-specify-columns-to-use) 12 | [5. Split The Data Into A Training And Test Set](https://github.com/bethz/AzureML-FlightPrediction/blob/master/README.md#5-split-the-data-into-a-training-and-test-set) 13 | [6. Train the model](https://github.com/bethz/AzureML-FlightPrediction/blob/master/README.md#6-train-the-model) 14 | [7. Select Algorithm](https://github.com/bethz/AzureML-FlightPrediction/blob/master/README.md#7-select-algorithm) 15 | [8. Score the Model](https://github.com/bethz/AzureML-FlightPrediction/blob/master/README.md#8-score-the-model) 16 | [9. Evaluate Model](https://github.com/bethz/AzureML-FlightPrediction/blob/master/README.md#9-evaluate-model) 17 | [10. Run Experiment!](https://github.com/bethz/AzureML-FlightPrediction/blob/master/README.md#10-run-experiment) 18 | 19 | ## 1. Access Azure ML 20 | 21 | This Azure Machine Learning Challenge can be completed with: 22 | * [Guest Access](http://studio.azureml.net/home/anonymous) - This does not require an Azure subscription or a credit card. It is anonymous access. After 8 hours the workspace gets reset. This is a great option for evaluation and this Challenge. 23 | * [Your Own Account](https://studio.azureml.net/Home) - Sign in and use a work, school or Microsoft account. Please do not take a tour now, explore the tour later. There are two types: 24 | * Free Account ([Microsoft Account](https://signup.live.com/signup) required) - 10 GB of Storage, R and Python Scripts and Web Service access 25 | * Enterprise Grade ([Azure Subscription](https://azure.microsoft.com/en-us/free/) required) - Full SLA, Bring your own Azure Storage, parallel graph execution, Elastic Web Service Endpoints 26 | 27 | ## 2. Create an Experiment 28 | 29 | Let's get started by making a new experiment. 30 | 31 | ### A. Make a new experiment 32 | Select ```+New``` in the lower left corner. 33 | 34 | 35 | 36 | ### B. Select Blank Experiment 37 | 38 | To the right of Experiment, you will see a tile with a plus sign and the words Blank Experiment. 39 | Select ```+ Blank Experiment```. 40 | 41 | 42 | 43 | ### C. Give the experiment a title 44 | By default, a title is created with a name like "**_Experiment created on 9/24/2016_**". 45 | Change the title to "My first Azure ML experiment" by editing the provided title. 46 | 47 | 48 | 49 | ## 3. Import, Review and Clean Data 50 | 51 | ### A. Search for flight data 52 | Type “flight” into the search bar. 53 | 54 | 55 | 56 | ### B. Import data 57 | 58 | Drag the ```Flight on-time performance Dataset``` to the workspace as show in the image. This is one of many sample datasets built into Azure Machine Learning Studio designed to help you learn and explore the tool. 59 | 60 | 61 | 62 | ### C. Review Data 63 | Right click on the dataset on your worksheet and select **dataset | visualize** from the pop-up menu. 64 | 65 | 66 | 67 | Notice the graphs or charts at the top of each data column. Explore the dataset by clicking on different columns. It’s essential in Machine Learning to be familiar with your data and visualizing your dataset is a great first step. This dataset provides a great deal of information about flights and whether or not they arrived on time. We are going to use Machine Learning to use this data to create a model that predicts whether a given flight will be late. 68 | 69 | 70 | 71 | Note: In an actual data science experiment, it is likely going to be necessary to [Data Wrangle](https://en.wikipedia.org/wiki/Data_wrangling) or clean dirty data. For this example, the data set is clean. 72 | 73 | You can find the column definitions for this data on the [US Department of Transportation site](http://www.transtats.bts.gov/DL_SelectFields.asp?Table_ID=236&DB_Short_Name=On-Time). 74 | 75 | ### D. Close the data visualization window 76 | Click on the X in the top right corner of the window to close the data visualization window.. 77 | 78 | ## 4. Specify Columns to Use 79 | You need to review the data in the dataset and decide which columns represent data that you think will affect whether or not a flight is delayed. You also need to select the column that you want to predict. In this case, we are going to predict the value of ```ArrDel15```. This is a binary state, 0/1, that indicates whether a flight arrival was delayed by more than 15 minutes. 80 | 81 | ### A. Add Manipulation to Select Columns in Dataset 82 | First, type "**project**" into the search bar and drag the ```Select Columns in Dataset``` manipulation to the workspace. 83 | This manipulation enables you to specify which columns in the data set you think are significant to the prediction. 84 | 85 | 86 | 87 | ### B. Connect Flight on-time performance task to Select Columns in Dataset task 88 | Connect the output of ```Flight on-time performance``` dataset to the input of the ```Select Columns in Dataset``` by clicking on the lower center dot and dragging to the input, top center dot, of the ```Select Columns in Dataset``` task. 89 | 90 | 91 | 92 | ### C. Launch Column Selector 93 | Click on the ```Select Columns in Dataset``` module, then on the far right, select **Launch column selector**. 94 | 95 | 96 | 97 | ### D. Select Columns 98 | Select the columns you think affect whether or not a flight is delayed as well as the column we want to predict ```ArrDel15```. In the following screenshot, I selected ```Month```, ```Carrier (airline)```, ```OriginAirportID```, ```DestAirportID```, and ```ArrDel15```. You might select more or less columns. You can also add or remove columns later. 99 | 100 | 101 | 102 | ### E. Complete Column Selection 103 | 104 | Select the checkbox in the lower right of the **Select columns** window. 105 | 106 | ## 5. Split The Data Into A Training And Test Set 107 | 108 | The ```Split Data``` task allows us to divide up our data, we need some of the data to try and find patterns so we can make predictions. We need to save some of the data to test if the model we create successfully makes predictions. 109 | 110 | Traditionally, you will split the data 80/20 or 70/30. For today’s challenge everyone will use an 80/20 split. That means 80% of the data will be used to train the model and 20% will be used to test the accuracy of the model we develop. 111 | 112 | ### A. Split Data Task 113 | 114 | Type “**split**” into the search bar and drag the ```Split Data``` task to the workspace. Connect the output of ```Select Columns in Dataset``` task to the input of the ```Split Data``` task (same way we connected the Flight Data to the Select Columns modules). 115 | 116 | 117 | 118 | ### B. Split our input data 119 | 120 | Click on the ```Split Data``` task to bring up the Properties Pane and specify ```.8``` as the Fraction of rows. 121 | 122 | 123 | 124 | ## 6. Train The Model 125 | 126 | Next, we identify which data is to be predicted. In our case, we are predicting the value of the column ```ArrDel15``` which indicates if a flight arrival time was delayed by more than 15 minutes. 127 | 128 | ### A. Connect Data 129 | 130 | Type “**train model**” into the search bar. Drag the ```Train Model``` task to the workspace. 131 | 132 | Hovering over the input and output dots will reveal what each input/output represents. Connect the first output, ```Results Dataset1```, (the circle on the left) of the ```Split Data``` task to the **rightmost** input of the ```Train Model``` task. This will take 80 % of our data and use it to train/teach our model to make predictions. 133 | 134 | 135 | 136 | ### B. Identify Predicted Value 137 | Click on the ```Train Model``` task. In the **Properties** window, select Launch Column Selector. Select the column ```ArrDel15``` by typing "arrdel15" in to the text box (a smart filter of columns will appear). Click the checkbox in the lower right corner to complete the operation. 138 | 139 | 140 | 141 | ## 7. Select Algorithm 142 | If you are a data scientist who creates their own algorithms, you could now import your own R code to analyze the patterns. But, Azure ML provides a number of standard algorithms which are available for use. 143 | 144 | Selecting an algorithm can be overwhelming, to help narrow the process a [Azure ML Cheat Sheet](https://aka.ms/azuremlcheatsheet) has been created. By narrowing the type of problem you are solving can find the algorithms that will be most likely to generate a good model. 145 | 146 | Today we are doing binary classification also known as Two-Class Classification. Using the cheat sheet we can narrow our selection to a standard algorithm called ```Two-Class Neural Network```. 147 | 148 | As you can see there are many Two-Class algorithms that we can choose from so we may want to try different ones out as we refine our model. Swapping out or even comparing two algorithms is made easy with Azure Machine learning as you will see. 149 | 150 | ### A. Connect algorithm 151 | Type “**two-class**” into the search bar. You will see a number of different classification algorithms listed and each has its own advantages and disadvantages. Each of the two-class algorithms is designed to predict a binary outcome. 152 | 153 | Select ```Two-Class Neural Network``` and drag it to the workspace. Connect the output of the ```Two-Class Neural Network``` task to the **leftmost** input of the ```Train Model``` task. 154 | 155 | 156 | 157 | ## 8. Score the Model 158 | After the model is trained, it is evaluated to determine how well it predicts delayed flights, so the model is scored by testing it against the Test Data which is the remaining 20% of the data we split to the second output of the ```Split Data``` task. 159 | 160 | ### A. Connect test data 161 | Type “**score**” into the search bar and drag the ```Score Model``` task to the workspace. 162 | 163 | Connect the output of ```Train Model``` to the **left input** of the ```Score Model``` task. Connect the Test Data, the **right output** of the ```Split Data``` task to the **right input** of the ```Score Model``` task as shown in the following screenshot. The output of this task is a scored dataset. 164 | 165 | 166 | 167 | ## 9. Evaluate Model 168 | Next, the model is evaluated to determine its accuracy. This is done by evaluating the trained model by using the test data. 169 | 170 | ### A. Determine accuracy of model 171 | Type “**evaluate**” into the search bar and drag the ```Evaluate Model``` task to the bottom of the workspace. 172 | 173 | Connect the output of the ```Score model``` task to the **left input** of the ```Evaluate Model``` task. The other input and output of the Evaluate Model task are not connected at this time. 174 | 175 | You are now ready to run your experiment! 176 | 177 | 178 | 179 | ## 10. Run Experiment 180 | 181 | ### A. Select Run 182 | Select Run on the bottom toolbar. You will see green check marks appear on each task as it completes. The data is flowing through your Machine Learning Workflow, starting with data selection, being trained against the model, and finally being evaluated. 183 | 184 | This process can take several minutes. When there is a green check mark on the ```Evaluate Model``` task the process is complete. 185 | 186 | 187 | 188 | ## 11. Post Run: Evaluate Model 189 | 190 | It is usually necessary to evaluate the model, improve it, re-run it and repeat. 191 | 192 | ### A. Evaluate The Model 193 | 194 | When the entire experiment is completed, right click on the ```Evaluate Model``` task and select “**Evaluation results | Visualize**” to see how well the model predicted delayed flights. 195 | 196 | 197 | 198 | ## 12. Interpreting Results 199 | 200 | The first run of a model is a baseline and is considered a first step. 201 | 202 | One useful piece of the evaluation results is the first graph the **True Positive Rate versus False Positive Rate**. This graph is a representation of the Area Under the Curve. A 45 degree flat line on this chart indicates guessing randomly. A slightly more accurate model than random guessing looks like the image below, our current model. 203 | 204 | 205 | 206 | If you scroll down you can see the accuracy – Higher accuracy is good! 207 | You can also see the number of false and true positive and negative predictions. 208 | - **True positives** are how often your model correctly predicted a flight would be late 209 | - **False positives** are how often your model predicted a flight would be late, when the flight was actually on time (your model predicted incorrectly) 210 | - **True negatives** indicate how often your model correctly predicted a flight would be on time (```arrDel15``` is false) 211 | - **False negatives** indicate how often your model predicted a flight would be on time, when in fact it was delayed (your model predicted incorrectly) 212 | 213 | You want higher values for True positives and True negatives, you want low values for False Positives and False negatives. 214 | 215 | 216 | 217 | From the model, there were no False Positives which is good. 218 | But, you can see from the results above my model predicted every single flight would be on time, not very helpful! I think we need to try something else… 219 | 220 | Your challenge is to improve the model and try different experiments. Change the algorithm, change algorithm parameters, change project columns to get the best possible accuracy/true/false positive/negative results! 221 | 222 | Note: As you make changes you will need to re-run the Experiment. 223 | 224 | Good Luck! 225 | 226 | 227 | 228 | 229 | -------------------------------------------------------------------------------- /hold/4a.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bethz/AzureML-FlightPrediction/d27301e8c47770991bd6a3695795a9e69d6929e4/hold/4a.png -------------------------------------------------------------------------------- /hold/4c.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bethz/AzureML-FlightPrediction/d27301e8c47770991bd6a3695795a9e69d6929e4/hold/4c.png -------------------------------------------------------------------------------- /images/10a.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bethz/AzureML-FlightPrediction/d27301e8c47770991bd6a3695795a9e69d6929e4/images/10a.png -------------------------------------------------------------------------------- /images/11a.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bethz/AzureML-FlightPrediction/d27301e8c47770991bd6a3695795a9e69d6929e4/images/11a.png -------------------------------------------------------------------------------- /images/12 algotry.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bethz/AzureML-FlightPrediction/d27301e8c47770991bd6a3695795a9e69d6929e4/images/12 algotry.png -------------------------------------------------------------------------------- /images/12graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bethz/AzureML-FlightPrediction/d27301e8c47770991bd6a3695795a9e69d6929e4/images/12graph.png -------------------------------------------------------------------------------- /images/12true.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bethz/AzureML-FlightPrediction/d27301e8c47770991bd6a3695795a9e69d6929e4/images/12true.png -------------------------------------------------------------------------------- /images/12truepos.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bethz/AzureML-FlightPrediction/d27301e8c47770991bd6a3695795a9e69d6929e4/images/12truepos.png -------------------------------------------------------------------------------- /images/2ANewExperiment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bethz/AzureML-FlightPrediction/d27301e8c47770991bd6a3695795a9e69d6929e4/images/2ANewExperiment.png -------------------------------------------------------------------------------- /images/2bBlankExperiment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bethz/AzureML-FlightPrediction/d27301e8c47770991bd6a3695795a9e69d6929e4/images/2bBlankExperiment.png -------------------------------------------------------------------------------- /images/2dTitle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bethz/AzureML-FlightPrediction/d27301e8c47770991bd6a3695795a9e69d6929e4/images/2dTitle.png -------------------------------------------------------------------------------- /images/3A.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bethz/AzureML-FlightPrediction/d27301e8c47770991bd6a3695795a9e69d6929e4/images/3A.png -------------------------------------------------------------------------------- /images/3b.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bethz/AzureML-FlightPrediction/d27301e8c47770991bd6a3695795a9e69d6929e4/images/3b.png -------------------------------------------------------------------------------- /images/3cdataset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bethz/AzureML-FlightPrediction/d27301e8c47770991bd6a3695795a9e69d6929e4/images/3cdataset.png -------------------------------------------------------------------------------- /images/3cvis.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bethz/AzureML-FlightPrediction/d27301e8c47770991bd6a3695795a9e69d6929e4/images/3cvis.png -------------------------------------------------------------------------------- /images/4a.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bethz/AzureML-FlightPrediction/d27301e8c47770991bd6a3695795a9e69d6929e4/images/4a.png -------------------------------------------------------------------------------- /images/4b.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bethz/AzureML-FlightPrediction/d27301e8c47770991bd6a3695795a9e69d6929e4/images/4b.png -------------------------------------------------------------------------------- /images/4c.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bethz/AzureML-FlightPrediction/d27301e8c47770991bd6a3695795a9e69d6929e4/images/4c.png -------------------------------------------------------------------------------- /images/4e.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bethz/AzureML-FlightPrediction/d27301e8c47770991bd6a3695795a9e69d6929e4/images/4e.png -------------------------------------------------------------------------------- /images/5a.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bethz/AzureML-FlightPrediction/d27301e8c47770991bd6a3695795a9e69d6929e4/images/5a.png -------------------------------------------------------------------------------- /images/5b.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bethz/AzureML-FlightPrediction/d27301e8c47770991bd6a3695795a9e69d6929e4/images/5b.png -------------------------------------------------------------------------------- /images/6a.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bethz/AzureML-FlightPrediction/d27301e8c47770991bd6a3695795a9e69d6929e4/images/6a.png -------------------------------------------------------------------------------- /images/6b.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bethz/AzureML-FlightPrediction/d27301e8c47770991bd6a3695795a9e69d6929e4/images/6b.png -------------------------------------------------------------------------------- /images/7a.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bethz/AzureML-FlightPrediction/d27301e8c47770991bd6a3695795a9e69d6929e4/images/7a.png -------------------------------------------------------------------------------- /images/8a.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bethz/AzureML-FlightPrediction/d27301e8c47770991bd6a3695795a9e69d6929e4/images/8a.png -------------------------------------------------------------------------------- /images/9a.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bethz/AzureML-FlightPrediction/d27301e8c47770991bd6a3695795a9e69d6929e4/images/9a.png -------------------------------------------------------------------------------- /images/new4d.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bethz/AzureML-FlightPrediction/d27301e8c47770991bd6a3695795a9e69d6929e4/images/new4d.png --------------------------------------------------------------------------------