├── .gitignore ├── 0_welcome ├── UsingStantoEstimateHierarchicalBayesModels-Welcomeemail-FINAL.pdf ├── images │ ├── Bayesian_Inference.jpg │ └── Bayesian_Inference.png ├── welcome.Rmd └── welcome.html ├── 1_lm ├── ComputerConjointData.csv ├── advanced │ ├── lm.stan │ └── lm_advanced.R ├── images │ ├── Bayesian_Inference.jpg │ ├── Bayesian_Inference.png │ └── fuzzycaterpillar.jpg ├── lm.R ├── lm.Rmd ├── lm.Rproj ├── lm.html ├── lm_cache │ └── html │ │ ├── __packages │ │ ├── unnamed-chunk-10_e61df3b1df3ea5fb66ef51d781ddbe4e.RData │ │ ├── unnamed-chunk-10_e61df3b1df3ea5fb66ef51d781ddbe4e.rdb │ │ ├── unnamed-chunk-10_e61df3b1df3ea5fb66ef51d781ddbe4e.rdx │ │ ├── unnamed-chunk-11_51c1a9743b40775acf53e9098d7a2f7c.RData │ │ ├── unnamed-chunk-11_51c1a9743b40775acf53e9098d7a2f7c.rdb │ │ ├── unnamed-chunk-11_51c1a9743b40775acf53e9098d7a2f7c.rdx │ │ ├── unnamed-chunk-12_8a974d8395fc1a579c6be1526018f87a.RData │ │ ├── unnamed-chunk-12_8a974d8395fc1a579c6be1526018f87a.rdb │ │ ├── unnamed-chunk-12_8a974d8395fc1a579c6be1526018f87a.rdx │ │ ├── unnamed-chunk-13_cf43ad4a2c8508354026a06f1bafe9dd.RData │ │ ├── unnamed-chunk-13_cf43ad4a2c8508354026a06f1bafe9dd.rdb │ │ ├── unnamed-chunk-13_cf43ad4a2c8508354026a06f1bafe9dd.rdx │ │ ├── unnamed-chunk-14_e55f71d151e35e93462fb8da2c15cda7.RData │ │ ├── unnamed-chunk-14_e55f71d151e35e93462fb8da2c15cda7.rdb │ │ ├── unnamed-chunk-14_e55f71d151e35e93462fb8da2c15cda7.rdx │ │ ├── unnamed-chunk-15_c6858e6b9aec06e9c08fb751d143bbaa.RData │ │ ├── unnamed-chunk-15_c6858e6b9aec06e9c08fb751d143bbaa.rdb │ │ ├── unnamed-chunk-15_c6858e6b9aec06e9c08fb751d143bbaa.rdx │ │ ├── unnamed-chunk-16_dfb489ca671eb30aada911c7e197e59d.RData │ │ ├── unnamed-chunk-16_dfb489ca671eb30aada911c7e197e59d.rdb │ │ ├── unnamed-chunk-16_dfb489ca671eb30aada911c7e197e59d.rdx │ │ ├── unnamed-chunk-17_be6831e96ef363339f71461780d9139b.RData │ │ ├── unnamed-chunk-17_be6831e96ef363339f71461780d9139b.rdb │ │ ├── unnamed-chunk-17_be6831e96ef363339f71461780d9139b.rdx │ │ ├── unnamed-chunk-18_ea55effbb8c4e585143dfc686130a28a.RData │ │ ├── unnamed-chunk-18_ea55effbb8c4e585143dfc686130a28a.rdb │ │ ├── unnamed-chunk-18_ea55effbb8c4e585143dfc686130a28a.rdx │ │ ├── unnamed-chunk-19_61c53c10543cdf242387e06d4a6a325e.RData │ │ ├── unnamed-chunk-19_61c53c10543cdf242387e06d4a6a325e.rdb │ │ ├── unnamed-chunk-19_61c53c10543cdf242387e06d4a6a325e.rdx │ │ ├── unnamed-chunk-1_7d0e97d16d7d093d5b42e50103ccc145.RData │ │ ├── unnamed-chunk-1_7d0e97d16d7d093d5b42e50103ccc145.rdb │ │ ├── unnamed-chunk-1_7d0e97d16d7d093d5b42e50103ccc145.rdx │ │ ├── unnamed-chunk-20_428166563a4d8667af9a4d9a3d38a940.RData │ │ ├── unnamed-chunk-20_428166563a4d8667af9a4d9a3d38a940.rdb │ │ ├── unnamed-chunk-20_428166563a4d8667af9a4d9a3d38a940.rdx │ │ ├── unnamed-chunk-21_ee1eea167b592e7b336e1bf4775ef2d9.RData │ │ ├── unnamed-chunk-21_ee1eea167b592e7b336e1bf4775ef2d9.rdb │ │ ├── unnamed-chunk-21_ee1eea167b592e7b336e1bf4775ef2d9.rdx │ │ ├── unnamed-chunk-22_781b5003085bd756a5e8fc061c28e436.RData │ │ ├── unnamed-chunk-22_781b5003085bd756a5e8fc061c28e436.rdb │ │ ├── unnamed-chunk-22_781b5003085bd756a5e8fc061c28e436.rdx │ │ ├── unnamed-chunk-23_5ea15adae67079d8719c83b99469ce0a.RData │ │ ├── unnamed-chunk-23_5ea15adae67079d8719c83b99469ce0a.rdb │ │ ├── unnamed-chunk-23_5ea15adae67079d8719c83b99469ce0a.rdx │ │ ├── unnamed-chunk-24_033eb3e60d56b8fb5a1474efcfa12d41.RData │ │ ├── unnamed-chunk-24_033eb3e60d56b8fb5a1474efcfa12d41.rdb │ │ ├── unnamed-chunk-24_033eb3e60d56b8fb5a1474efcfa12d41.rdx │ │ ├── unnamed-chunk-25_5c0f3af19510e7bcf7b327e1f8235a03.RData │ │ ├── unnamed-chunk-25_5c0f3af19510e7bcf7b327e1f8235a03.rdb │ │ ├── unnamed-chunk-25_5c0f3af19510e7bcf7b327e1f8235a03.rdx │ │ ├── unnamed-chunk-26_f47c47918a569d26accd8776d65c4d32.RData │ │ ├── unnamed-chunk-26_f47c47918a569d26accd8776d65c4d32.rdb │ │ ├── unnamed-chunk-26_f47c47918a569d26accd8776d65c4d32.rdx │ │ ├── unnamed-chunk-2_7b35662aa612b3282486f013f29b7cbe.RData │ │ ├── unnamed-chunk-2_7b35662aa612b3282486f013f29b7cbe.rdb │ │ ├── unnamed-chunk-2_7b35662aa612b3282486f013f29b7cbe.rdx │ │ ├── unnamed-chunk-3_2b7a7863ff95c6e3b1cbc6994c39689f.RData │ │ ├── unnamed-chunk-3_2b7a7863ff95c6e3b1cbc6994c39689f.rdb │ │ ├── unnamed-chunk-3_2b7a7863ff95c6e3b1cbc6994c39689f.rdx │ │ ├── unnamed-chunk-4_026f1d4e2418ba1b4cc0abd726e6096a.RData │ │ ├── unnamed-chunk-4_026f1d4e2418ba1b4cc0abd726e6096a.rdb │ │ ├── unnamed-chunk-4_026f1d4e2418ba1b4cc0abd726e6096a.rdx │ │ ├── unnamed-chunk-5_6b72ebddcf9904abe48243aa1d15d59d.RData │ │ ├── unnamed-chunk-5_6b72ebddcf9904abe48243aa1d15d59d.rdb │ │ ├── unnamed-chunk-5_6b72ebddcf9904abe48243aa1d15d59d.rdx │ │ ├── unnamed-chunk-6_211aef65edc6632bf170c6d362374acf.RData │ │ ├── unnamed-chunk-6_211aef65edc6632bf170c6d362374acf.rdb │ │ ├── unnamed-chunk-6_211aef65edc6632bf170c6d362374acf.rdx │ │ ├── unnamed-chunk-7_67b0a668bee133c8f0790474c71af4d9.RData │ │ ├── unnamed-chunk-7_67b0a668bee133c8f0790474c71af4d9.rdb │ │ ├── unnamed-chunk-7_67b0a668bee133c8f0790474c71af4d9.rdx │ │ ├── unnamed-chunk-8_2ab1bd328e66b08a35ba98d9e15fbec8.RData │ │ ├── unnamed-chunk-8_2ab1bd328e66b08a35ba98d9e15fbec8.rdb │ │ ├── unnamed-chunk-8_2ab1bd328e66b08a35ba98d9e15fbec8.rdx │ │ ├── unnamed-chunk-9_7c4795a27949f9e9bc4ddda935329ea6.RData │ │ ├── unnamed-chunk-9_7c4795a27949f9e9bc4ddda935329ea6.rdb │ │ └── unnamed-chunk-9_7c4795a27949f9e9bc4ddda935329ea6.rdx └── lm_files │ └── figure-html │ ├── unnamed-chunk-21-1.png │ ├── unnamed-chunk-22-1.png │ ├── unnamed-chunk-23-1.png │ ├── unnamed-chunk-24-1.png │ ├── unnamed-chunk-25-1.png │ └── unnamed-chunk-3-1.png ├── 2_mnl ├── 2_mnl.Rproj ├── cbc_chocolate.csv ├── images │ ├── Bayesian_Inference.png │ ├── beta[2]beta[3].pdf │ └── beta[2]beta[3].png ├── mnl.R ├── mnl.Rmd ├── mnl.html ├── mnl.rds ├── mnl.stan ├── mnl_cache │ └── html │ │ ├── __packages │ │ ├── unnamed-chunk-25_a54a27fec6de7639f00fb24e52fdbb18.RData │ │ ├── unnamed-chunk-25_a54a27fec6de7639f00fb24e52fdbb18.rdb │ │ ├── unnamed-chunk-25_a54a27fec6de7639f00fb24e52fdbb18.rdx │ │ ├── unnamed-chunk-45_a3ea144d44a6ba6745a74c44592f568e.RData │ │ ├── unnamed-chunk-45_a3ea144d44a6ba6745a74c44592f568e.rdb │ │ └── unnamed-chunk-45_a3ea144d44a6ba6745a74c44592f568e.rdx └── mnl_files │ └── figure-html │ ├── unnamed-chunk-28-1.png │ ├── unnamed-chunk-29-1.png │ ├── unnamed-chunk-30-1.png │ ├── unnamed-chunk-31-1.png │ ├── unnamed-chunk-34-1.png │ ├── unnamed-chunk-35-1.png │ ├── unnamed-chunk-36-1.png │ ├── unnamed-chunk-48-1.png │ └── unnamed-chunk-49-1.png ├── 3_hmnl ├── 3_hmnl.Rproj ├── cbc_chocolate.csv ├── hmnl.R ├── hmnl.Rmd ├── hmnl.html ├── hmnl.rds ├── hmnl.stan ├── hmnl_cache │ └── html │ │ ├── __packages │ │ ├── unnamed-chunk-23_085fd1961339d2a88bff9caa4082a229.RData │ │ ├── unnamed-chunk-23_085fd1961339d2a88bff9caa4082a229.rdb │ │ ├── unnamed-chunk-23_085fd1961339d2a88bff9caa4082a229.rdx │ │ ├── unnamed-chunk-24_8019a6f96fd873f7f4c80480657201b3.RData │ │ ├── unnamed-chunk-24_8019a6f96fd873f7f4c80480657201b3.rdb │ │ ├── unnamed-chunk-24_8019a6f96fd873f7f4c80480657201b3.rdx │ │ ├── unnamed-chunk-39_f87fecbb9859a5a7d3d8e77c67577ceb.RData │ │ ├── unnamed-chunk-39_f87fecbb9859a5a7d3d8e77c67577ceb.rdb │ │ ├── unnamed-chunk-39_f87fecbb9859a5a7d3d8e77c67577ceb.rdx │ │ ├── unnamed-chunk-40_83577461465d937bd855b7fb3dbffd7d.RData │ │ ├── unnamed-chunk-40_83577461465d937bd855b7fb3dbffd7d.rdb │ │ └── unnamed-chunk-40_83577461465d937bd855b7fb3dbffd7d.rdx ├── hmnl_files │ └── figure-html │ │ ├── unnamed-chunk-24-1.png │ │ ├── unnamed-chunk-25-1.png │ │ ├── unnamed-chunk-26-1.png │ │ ├── unnamed-chunk-27-1.png │ │ ├── unnamed-chunk-30-1.png │ │ ├── unnamed-chunk-31-1.png │ │ ├── unnamed-chunk-32-1.png │ │ ├── unnamed-chunk-40-1.png │ │ ├── unnamed-chunk-41-1.png │ │ ├── unnamed-chunk-42-1.png │ │ ├── unnamed-chunk-43-1.png │ │ ├── unnamed-chunk-44-1.png │ │ ├── unnamed-chunk-48-1.png │ │ ├── unnamed-chunk-49-1.png │ │ ├── unnamed-chunk-50-1.png │ │ ├── unnamed-chunk-51-1.png │ │ └── unnamed-chunk-52-1.png └── images │ ├── Bayesian_Inference.png │ └── lkj_prior.png ├── 4_hnestedmnl ├── .Rbuildignore ├── .gitignore ├── 4_hnestedmnl.Rproj ├── GPLv2.txt ├── LICENSE ├── hnestedmnl.Rmd ├── hnestedmnl.pdf ├── hnestedmnl.stan └── img │ ├── Adobe_standard_logo_RGB.eps │ ├── auto1.jpg │ ├── auto2.jpg │ ├── auto3.jpg │ ├── barchart1.png │ ├── barchart2.png │ ├── funnel-plot.png │ ├── jensens.png │ ├── lots-of-bag-cases.jpg │ ├── not-buying.jpg │ ├── themodellersnew.png │ ├── vtask1.png │ ├── vtask2.png │ └── vtask3.png ├── 5_data_fusion ├── SplitQuestionaire.R ├── SplitQuestionaire.stan └── SplitQuestionaireMVP.stan ├── 6_about_stan ├── about_stan.Rmd └── about_stan.html ├── README.md └── demo ├── cbc_chocolate.csv ├── demo.Rmd ├── demo.html ├── hmnl.stan └── images ├── Bayesian_Inference.png ├── barchart1.png ├── barchart2.png └── hnestedmnl_predictions.PNG /.gitignore: -------------------------------------------------------------------------------- 1 | .Rproj.user 2 | .Rhistory 3 | .RData 4 | .Ruserdata 5 | -------------------------------------------------------------------------------- /0_welcome/UsingStantoEstimateHierarchicalBayesModels-Welcomeemail-FINAL.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/0_welcome/UsingStantoEstimateHierarchicalBayesModels-Welcomeemail-FINAL.pdf -------------------------------------------------------------------------------- /0_welcome/images/Bayesian_Inference.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/0_welcome/images/Bayesian_Inference.jpg -------------------------------------------------------------------------------- /0_welcome/images/Bayesian_Inference.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/0_welcome/images/Bayesian_Inference.png -------------------------------------------------------------------------------- /0_welcome/welcome.Rmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Stan Tutorial" 3 | subtitle: "Welcome" 4 | author: "Kevin Van Horn and Elea McDonnell Feit" 5 | date: "June 25, 2017" 6 | output: ioslides_presentation 7 | --- 8 | 9 | ```{r setup, include=FALSE} 10 | knitr::opts_chunk$set(echo = FALSE) 11 | ``` 12 | 13 | ## What is Stan? 14 | > Stan is freedom-respecting open-source software for facilitating statistical inference at the frontiers of applied statistics. 15 | 16 |
-- Stan homepage
17 | 18 | ## What is (Bayesian) statistical inference? 19 | ```{r, fig.align="center"} 20 | knitr::include_graphics("images/Bayesian_Inference.png") 21 | ``` 22 | 23 | ## Software options for Bayesian estimation 24 | - Write-your-own sampler 25 | - Purpose built routines for individual models 26 | + MCMCPack R package 27 | + Stata's Bayesian package 28 | + Sawtooth CBC/HB 29 | - Tools which automatically build samplers for a wide class of models 30 | + WinBugs 31 | + JAGS 32 | + Stan 33 | 34 | ## Why Stan? 35 | - Ideal tool for rapid prototyping of new models 36 | - Modern, thoughful software design 37 | - Increasingly attractive option for fitting even standard Bayesian models 38 | 39 | ## Tutorial modules 40 | - Module 1: Linear regression model 41 | + Get up-and-running with Stan 42 | - Module 2: Multinomial logit model 43 | + Learn basic Stan syntax 44 | + Test a model using synthetic data 45 | - Module 3: Hierarchial multinomial logit model 46 | + Learn more Stan syntax 47 | + Work with hierarchical models 48 | - Module 4: Nested hierarchical multinomial logit 49 | + Illustrate development of new models using Stan 50 | - Bonus: Data fusion for split questionares 51 | 52 | ## A word of warning 53 | Our focus here is on becoming a capable *user* of Stan, so we won't be spending much time talking about how Stan works internally or how it compares to other Bayesian software or classical inference. 54 | 55 | ## Presenters 56 | - **Kevin Van Horn** Senior Data Science Engineer at Adobe 57 | + Develops commercial-quality software for Bayesian estimation and experimental design 58 | 59 | - **Elea McDonnell Feit** Assistant Professor of Marketing at Drexel University and Senior Fellow of the Wharton Customer Analtyics Initiative 60 | + Develops tutorials for practitioners in hierarchical Bayes, R, advertising measurement, and field experiments 61 | 62 | ## To follow you will need to install: 63 | - R statistical language 64 | + Manages data input, inspection and cleaning 65 | + Stores and summarizes posterior draws 66 | + Provides graphics 67 | - R studio (optional) 68 | + Convenient user interface for R 69 | - Rstan package 70 | + Modeling language 71 | + Statistical inference engine 72 | 73 | ## Code files 74 | Within the folder for each module, you will find: 75 | - `.html`: slides for the module 76 | - `.R`: R code for the module, so you can follow along 77 | - `.csv`: data file we will use for estimation 78 | 79 | There are other files in each module folder, which we will explain later. 80 | 81 | -------------------------------------------------------------------------------- /1_lm/advanced/lm.stan: -------------------------------------------------------------------------------- 1 | // Stan model code for linear model 2 | data { 3 | int N; 4 | int K; 5 | vector[N] y; 6 | matrix[N,K] X; 7 | } 8 | parameters { 9 | real beta0; 10 | vector[K] beta; 11 | real sigma; 12 | } 13 | model { 14 | y ~ normal(beta0 + X*beta, sigma); 15 | } -------------------------------------------------------------------------------- /1_lm/advanced/lm_advanced.R: -------------------------------------------------------------------------------- 1 | # Advanced techniques with lm in Stan: 2 | # - Testing with synthetic data 3 | # - Comparing to other inferential engines in R 4 | # Elea McDonnell Feit, eleafeit@gmail.com 5 | # 14 June 2017 6 | 7 | # ======== Test with synthetic data =========== 8 | library(MASS) # multivariate distributions for generating data 9 | 10 | generate_lm_data <- function(N=100, # number of observations 11 | mean.X=c(0, 0), Sigma.X=diag(1,2), # mean and covariance of ivs 12 | beta.0=0, beta=c(1,-1), sigma.e=1) # parameters of linear model 13 | { 14 | epsilon <- rnorm(n=N, mean=0, sd=sigma.e) 15 | X <- mvrnorm(n=N, mu=mean.X, Sigma=Sigma.X) 16 | colnames(X) <- paste("X", 1:(ncol(X)), sep="") 17 | y <- beta.0 + X %*% beta + epsilon 18 | list(data=list(N=N, K=length(beta), y=as.vector(y), X=X), 19 | true=list(beta.0=beta.0, beta=beta, sigma.e=sigma.e)) 20 | } 21 | 22 | set.seed(20130601) 23 | d <- generate_lm_data(N=100)$data 24 | str(d) 25 | 26 | m.stan <- stan(file="lm.stan", data=d, iter=1000) # read model in from file 27 | summary(m.stan)$summary 28 | plot(m.stan) 29 | get_stanmodel(m.stan) 30 | 31 | # fancier visualizations of posteriors 32 | library(bayesplot) 33 | mcmc_combo(As.mcmc.list(m.stan)) 34 | 35 | # ========= Compare to other inferential methods ========== 36 | 37 | # utility function for combining y and X in a data frame 38 | frameup <- function(data) { 39 | data.frame(y=data$y, data$X) 40 | } 41 | 42 | # compare to lm(), which estimates the model by OLS 43 | m.lm <- lm(y ~ ., data=frameup(d)) 44 | summary(m.lm) 45 | 46 | # compare to MCMCregress(), which estimates the model using Bayesian methods 47 | library(MCMCpack) # alternative Bayesian estimation for linear model 48 | m.MCMCpack <- MCMCregress(y ~ ., data=frameup(d)) 49 | summary(m.MCMCpack) 50 | mcmc_combo(as.mcmc.list(m.MCMCpack)) 51 | 52 | # pull out means and posterior std deviations for comparison 53 | data.frame(stan=summary(m.stan)$summary[1:4,c(1,3)], 54 | lm=rbind(coef(summary(m.lm))[1:3,1:2], c(NA, NA)), 55 | MCMCpack=summary(m.MCMCpack)$statistics[1:4,1:2]) 56 | 57 | -------------------------------------------------------------------------------- /1_lm/images/Bayesian_Inference.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/images/Bayesian_Inference.jpg -------------------------------------------------------------------------------- /1_lm/images/Bayesian_Inference.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/images/Bayesian_Inference.png -------------------------------------------------------------------------------- /1_lm/images/fuzzycaterpillar.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/images/fuzzycaterpillar.jpg -------------------------------------------------------------------------------- /1_lm/lm.R: -------------------------------------------------------------------------------- 1 | # Fitting a linear model using Stan 2 | # Elea McDonnell Feit, efeit@drexel.edu 3 | # 22 June 2017 4 | 5 | # Copyright 2017, Kevin Van Horn & Elea McDonnell Feit 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # 10 | # You may obtain a copy of the License at 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | # Make sure you have the following packages installed (only need to to once) 20 | if (FALSE) { 21 | install.packages("MASS") 22 | install.packages("rstan") 23 | install.packages("bayesplot") 24 | install.packages("MCMCpack") 25 | } 26 | 27 | # ========= Load and inspect computer conjoint study ========== 28 | cc.df <- read.csv("ComputerConjointData.csv") 29 | head(cc.df) 30 | 31 | summary(cc.df) 32 | 33 | plot(LIKE ~ Price, data = cc.df, ylab="Rating (0-10)") 34 | 35 | # ========= Specify Stan model ========= 36 | lm.stan <- " 37 | data { 38 | int N; 39 | int K; 40 | vector[N] y; 41 | matrix[N,K] X; 42 | } 43 | 44 | parameters { 45 | real beta0; 46 | vector[K] beta; 47 | real sigma; 48 | } 49 | 50 | model { 51 | y ~ normal(beta0 + X*beta, sigma); 52 | } 53 | " 54 | 55 | # ======== Prepare data for Stan ========= 56 | y <- cc.df[!is.na(cc.df$LIKE),"LIKE"] 57 | str(y) 58 | 59 | # wrong 60 | X <- cc.df[!is.na(cc.df$LIKE), 4:16] # attribute cols 61 | head(X) 62 | 63 | X <- model.matrix(~ HotLine + RAM + Screen + CPU + HardDisk + CD + Cache + Color + 64 | Channel + Warranty + Software + Guarantee + Price, 65 | data=cc.df[!is.na(cc.df$LIKE),]) 66 | X <- X[,2:ncol(X)] # remove intercept 67 | head(X) 68 | 69 | cc.standata <- list(N=length(y), K=ncol(X), y=y, X=X) 70 | str(cc.standata) 71 | rm(y, X) 72 | 73 | # ========= Run Stan Model ========== 74 | library(rstan) # load interface to Stan 75 | m.stan <- stan(model_code = lm.stan, data = cc.standata, iter = 1000, chains=1) 76 | 77 | str(m.stan) 78 | 79 | mydraws <- extract(m.stan, pars=c("beta")) 80 | head(mydraws$beta) 81 | 82 | summary(m.stan)$summary 83 | 84 | cc.lab <- c("Intercept", dimnames(cc.standata$X)[[2]], "sigma", "lp__") 85 | data.frame(label=cc.lab, summary(m.stan)$summary) 86 | 87 | mean(mydraws$beta[,6]>0) 88 | mean(mydraws$beta[,8]>0) 89 | 90 | plot(m.stan, pars=c("beta0", "beta")) 91 | 92 | plot(m.stan, plotfun="trace", pars=c("beta0", "beta", "sigma")) 93 | 94 | plot(m.stan, plotfun="hist", pars=c("beta0", "beta", "sigma")) 95 | 96 | get_stanmodel(m.stan) 97 | 98 | 99 | -------------------------------------------------------------------------------- /1_lm/lm.Rmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Stan Tutorial" 3 | subtitle: "Module 1: Linear Regression Model" 4 | author: "Kevin Van Horn and Elea McDonnell Feit" 5 | date: "June 25, 2017" 6 | output: ioslides_presentation 7 | --- 8 | 9 | ```{r setup, include=FALSE} 10 | knitr::opts_chunk$set(echo = FALSE) 11 | require(jpeg) 12 | require(rstan) 13 | ## See this: http://stackoverflow.com/questions/15625990/how-to-set-size-for-local-image-using-knitr-for-markdown 14 | ``` 15 | 16 | ## Module objectives 17 | In this module, we will run a linear model (standard regression) using Stan. 18 | 19 | This will allow us to: 20 | - Understand how to do Bayesian inference with Stan without getting bogged down in modeling details. 21 | 22 | Running a linear model in Stan isn't something we would normally do in our own work, but it gives us a simple starting point that most people are familar with. We will add details and complexity in subsequent modules. 23 | 24 | ## Let's get started! 25 | - Open the file `lm.R` from the folder `1_lm` folder. 26 | - In that file, you will find the code for this module and can run the code to follow along. 27 | - Nearly all the code in `lm.R` is contained in these slides, as well. 28 | 29 | # Computer conjoint analysis with a linear model 30 | 31 | ## Computer conjoint data 32 | - Our first data set is a *ratings-based* conjoint study reported in Lenk, Desarbo, Green and Young (1996). Thanks, Peter! 33 | - In this study, respondents were shown computer profiles (combinations of features) and rated them on a 0-10 scale. 34 | - Our goal is to analyze this data using a linear model (i.e. standard regression) relating the respondents' ratings to the features of the computers. 35 | - By interpreting the coeficients, we will know how much people value the features of computers (or at least how they valued features back in the early 90's.) 36 | 37 | 38 | ## Read the data into R 39 | ```{r, echo = TRUE, eval=TRUE} 40 | cc.df <- read.csv("ComputerConjointData.csv") 41 | head(cc.df) 42 | ``` 43 | 44 | ## Inspect the data in R 45 | ```{r, echo = TRUE} 46 | summary(cc.df) 47 | ``` 48 | 49 | ## Plot the data in R 50 | ```{r, echo = TRUE} 51 | plot(LIKE ~ Price, data = cc.df, ylab = "Rating (0-10)") 52 | ``` 53 | 54 | ## Have data. Still need model. 55 | ```{r, fig.align="center"} 56 | knitr::include_graphics("images/Bayesian_Inference.png") 57 | ``` 58 | 59 | ## Stan model specification 60 | - In most statistical software (e.g., SPSS, JMP, Stata, SAS, base R, `MCMCpack`), you are provided with a menu of different models that you can run such as standard regression, logistic regression, poisson regression, etc. 61 | - In Stan, you specify the model you want to estimate using a special syntax. 62 | - This gives you the freedom to specify any model you want. 63 | - When we want to estimate a standard regression, we still need to specify the model, which helps us remember to think carefully about the model. 64 | 65 | ## Stan model components 66 | - `data` declares the structure of the data that you observe 67 | - `parameters` declares the parameters of your model 68 | - `model` describes the model you want to estimate 69 | + And, optionally, your priors on the parameters of that model 70 | 71 | ## Stan `data` 72 | The data structures and their dimensions must all be declared in the `data` block. 73 | *Don't type this into R!* 74 | ```{stan, eval=FALSE, echo=TRUE, output.var="dummy"} 75 | data { 76 | int N; 77 | int K; 78 | vector[N] y; 79 | matrix[N,K] X; 80 | } 81 | ``` 82 | For this model, the data is a matrix describing the attributes of the products that were shown to respondents (`X`) and the vector of ratings that the respondents gave those products (`y`). 83 | There are `N` evaluations and `K` product attributes. Note that dimensions are declared explicitly. 84 | 85 | ## Stan `model` 86 | The `model` block we specify says that the ratings `y` are normally distributed with mean `beta0 + X*beta` and standard deviation `sigma`. 87 | *Don't type this into R!* 88 | ```{stan, eval=FALSE, echo=TRUE, output.var="dummy"} 89 | model { 90 | y ~ normal(beta0 + X*beta, sigma); 91 | } 92 | ``` 93 | That's pretty straightforward! 94 | 95 | ## Stan `parameters` 96 | - intercept `beta0` 97 | - a vector of `K` parameters that multiplies the product attributes in `X` which is called `beta` 98 | - a variance parameter `sigma` 99 | *Don't type this into R!* 100 | ```{stan, eval=FALSE, echo=TRUE, output.var="dummy"} 101 | parameters { 102 | real beta0; 103 | vector[K] beta; 104 | real sigma; 105 | } 106 | ``` 107 | The parameters we are really after are those in the `beta` vector, which are sometimes called the part-worths of the product attributes. 108 | 109 | ## Stan specification as a text string in R 110 | ```{r, eval=TRUE, echo=TRUE} 111 | lm.stan <- " 112 | data { 113 | int N; 114 | int K; 115 | vector[N] y; 116 | matrix[N,K] X; 117 | } 118 | 119 | parameters { 120 | real beta0; 121 | vector[K] beta; 122 | real sigma; 123 | } 124 | 125 | model { 126 | y ~ normal(beta0 + X*beta, sigma); 127 | } 128 | " 129 | ``` 130 | 131 | # Preparing the computer data for Stan 132 | 133 | ## Preparing data for Stan: `y` 134 | When you pass data to Stan, it needs to be in the format that we have declared in our Stan code. To create the `y` variable in R: 135 | ```{r, eval=TRUE, echo=TRUE} 136 | y <- cc.df[!is.na(cc.df$LIKE),"LIKE"] 137 | str(y) 138 | ``` 139 | 140 | ## Preparing data for Stan: `X` (wrong!) 141 | For `X`, we want to include all of the product features, which are: 142 | ```{r, eval=TRUE, echo=TRUE} 143 | X <- cc.df[!is.na(cc.df$LIKE), 4:16] 144 | head(X) 145 | ``` 146 | 147 | ## Preparing data for Stan: `X` (right!) 148 | However, in the original data, the values of the attributes are text labels like `8MB` while Stan is expecting a matrix of real numbers that it can use in multiplication. We can code these attributes as numbers using the R function `model.matrix`. 149 | ```{r, eval=TRUE, echo=TRUE} 150 | X <- model.matrix(~ HotLine + RAM + Screen + CPU + HardDisk + 151 | CD + Cache + Color + Channel + Warranty + 152 | Software + Guarantee + Price, 153 | data=cc.df[!is.na(cc.df$LIKE),]) 154 | X <- X[,2:ncol(X)] ## remove intercept 155 | head(X) 156 | ``` 157 | 158 | ## Preparing data for Stan: bundling it up 159 | Finally, to pass the data to Stan, it needs to be bundled up as a `list` object in R. 160 | ```{r, eval=TRUE, echo=TRUE} 161 | cc.standata <- list(N=length(y), K=ncol(X), y=y, X=X) 162 | str(cc.standata) 163 | ``` 164 | ## Tidy up the R workspace by removing `y` and `X` 165 | ```{r, eval=TRUE, echo=TRUE} 166 | rm(y, X) 167 | ``` 168 | 169 | ## Bayesian statistical inference 170 | ```{r, fig.align="center"} 171 | knitr::include_graphics("images/Bayesian_Inference.png") 172 | ``` 173 | 174 | # Running the model 175 | 176 | ## Running the model 177 | Now that we have a Stan model and data in Stan format, we can ask Stan to produce random draws from the posterior for us. 178 | ```{r, eval=TRUE, echo=TRUE, cache=TRUE} 179 | library(rstan) ## load interface to Stan 180 | m.stan <- stan(model_code = lm.stan, data = cc.standata, 181 | iter = 1000, chains=1) 182 | ``` 183 | This part was easy for us, but it may take some time for the computer to produce the posterior draws. 184 | 185 | ## What's `stan()` doing? 186 | - `stan()` is producing a sample of draws from the posterior distribution. 187 | - When you call `stan()`, it first translates the model and data into a program for producing the posterior draws, then runs that program and feeds the output back to R. 188 | - While much Bayesian software uses an MCMC algorithm called Gibbs sampling to produce the posterior draws, Stan defaults to a different MCMC algorithm called Hamiltonian Monte Carlo. 189 | - One of the great things about Stan is you don't need to know too much about the algorithm to use the software (but knowing a little bit helps.) 190 | 191 | ## `stan()` output 192 | The `stanfit` object that is returned to R from `stan()` stores everything about the run: the model text, the posterior draws, the names of the parameters, etc. 193 | ```{r, eval=TRUE, echo=TRUE} 194 | str(m.stan) 195 | ``` 196 | 197 | ## Getting the posterior draws 198 | The core output of Stan is a matrix of posterior draws for all the parameters. We can get the posterior draws using the `extract()` function from the `rstan` package. 199 | ```{r, eval=TRUE, echo=TRUE} 200 | mydraws <- extract(m.stan, pars=c("beta")) 201 | head(mydraws$beta) 202 | ``` 203 | 204 | # Summarizing the draws 205 | 206 | ## Summarizing the posterior draws 207 | - We usually summarize the draws to get a sense of what the parameters might be. 208 | - Some Bayesian software like Sawtooth CBC-HB does this step more automatically for the user, but the this two stage process is typical and allows the analyst to summarize the draws as she likes. 209 | 210 | ## Posterior summaries 211 | ```{r, eval=TRUE, echo=TRUE} 212 | summary(m.stan)$summary 213 | ``` 214 | 215 | ## Adding labels to the posterior summary 216 | ```{r, eval=TRUE, echo=TRUE} 217 | cc.lab <- c("Intercept", dimnames(cc.standata$X)[[2]], "sigma", "lp__") 218 | data.frame(label=cc.lab, summary(m.stan)$summary) 219 | ``` 220 | 221 | ## Bayesian hypothesis testing 222 | We can compute the probability that a parameter is greater than some threshold like zero just by summarizing the draws. 223 | ```{r, eval=TRUE, echo=TRUE} 224 | mean(mydraws$beta[,6]>0) 225 | mean(mydraws$beta[,8]>0) 226 | ``` 227 | If more than 95% of draws are greater than 0, then it is very likley that the parameter is greater than zero. 228 | 229 | ## Plotting the parameters 230 | ```{r, eval=TRUE, echo=TRUE, message=FALSE} 231 | plot(m.stan, pars=c("beta0", "beta")) 232 | ``` 233 | 234 | ## Interpretation 235 | - Having a CD drive increases the rating by almost a point (`beta[6]`). 236 | - Including the faster processor increasing the rating by almost a point (`beta[4]`). 237 | - The higher price reduces the rating by more than 2 points (`beta[13]`). 238 | - The color has almost no effect on the rating (`beta[8]`). 239 | 240 | ## Assessing convergence with traceplots 241 | There is some risk that the Stan sampler did not reach the proper posterior. If this happens, we can usually detect it by ploting the draws in order over time (called a *traceplot*.) 242 | ```{r, eval=TRUE, echo=TRUE} 243 | plot(m.stan, plotfun="trace", pars=c("beta0", "beta", "sigma")) 244 | ``` 245 | 246 | ## Fuzzy caterpillar 247 | Traceplots should look like fuzzy caterpillars. 248 | ```{r, fig.align="center"} 249 | knitr::include_graphics("images/fuzzycaterpillar.jpg") 250 | ``` 251 | 252 | ## Histograms of posterior draws 253 | Histograms of the draws approximate posterior densities. 254 | ```{r, eval=TRUE, echo=TRUE} 255 | plot(m.stan, plotfun="hist", pars=c("beta0", "beta", "sigma")) 256 | ``` 257 | 258 | ## Density plots of posterior draws 259 | Or you could use a density plot, if you prefer. 260 | ```{r, eval=TRUE, echo=TRUE} 261 | plot(m.stan, plotfun="dens", pars=c("beta0", "beta", "sigma")) 262 | ``` 263 | 264 | ## Combining plots 265 | We happen to like this plot from another R package: 266 | ```{r, eval=FALSE, echo=TRUE} 267 | library(bayesplot) 268 | mcmc_combo(As.mcmc.list(m.stan)) 269 | ``` 270 | But it doesn't work very well with 15 parameters. 271 | 272 | ## Other things you can get from the `stanfit` object 273 | ```{r, eval=TRUE, echo=TRUE} 274 | get_stanmodel(m.stan) 275 | ``` 276 | 277 | # Summary of Module 1 278 | 279 | ## What have we accomplished? 280 | - We've reviewed Bayesian inference. 281 | - We've fit linear model to the computer conjoint data using Stan. 282 | + More precisely, we have computed a posterior sample for the parameters of the linear model using the computer conjoint data. 283 | - The model suggests that CD and CPU are the most important attributes with customers prefering computers that have a CD drive and have a faster CPU. 284 | 285 | ## Stan modeling steps 286 | 1. Read data into into R and inspect 287 | 2. Create model specifiction using the Stan language 288 | - `data` 289 | - `parameters` 290 | - `model` 291 | 3. Munge data into Stan format (list of vectors and matricies) 292 | 4. Call `stan()` to create the posterior draws 293 | 5. Assess convergence 294 | 6. Summarize and visualize the draws to make inferences 295 | 296 | ## Advanced techniques 297 | - In the file `lm_advanced.R` you will find code for several advanced topics 298 | - Testing the model using synthetic data 299 | - Comparing inference from `stan()` to `lm()` 300 | + OLS regression in R using `lm()` 301 | + Bayesian regression using `MCMCregress()` from `MCMCPack` 302 | 303 | ## A note on hierarchical models 304 | - Lenk et al. (1996) used this data set to illustrate a *hierarchical* linear model and that is a more approrpriate model for this data. 305 | - However, we are going to switch to choice models before we get to hierarchical models. -------------------------------------------------------------------------------- /1_lm/lm.Rproj: -------------------------------------------------------------------------------- 1 | Version: 1.0 2 | 3 | RestoreWorkspace: Default 4 | SaveWorkspace: Default 5 | AlwaysSaveHistory: Default 6 | 7 | EnableCodeIndexing: Yes 8 | UseSpacesForTab: Yes 9 | NumSpacesForTab: 2 10 | Encoding: UTF-8 11 | 12 | RnwWeave: Sweave 13 | LaTeX: pdfLaTeX 14 | -------------------------------------------------------------------------------- /1_lm/lm_cache/html/__packages: -------------------------------------------------------------------------------- 1 | base 2 | ggplot2 3 | StanHeaders 4 | rstan 5 | -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-10_e61df3b1df3ea5fb66ef51d781ddbe4e.RData: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-10_e61df3b1df3ea5fb66ef51d781ddbe4e.RData -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-10_e61df3b1df3ea5fb66ef51d781ddbe4e.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-10_e61df3b1df3ea5fb66ef51d781ddbe4e.rdb -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-10_e61df3b1df3ea5fb66ef51d781ddbe4e.rdx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-10_e61df3b1df3ea5fb66ef51d781ddbe4e.rdx -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-11_51c1a9743b40775acf53e9098d7a2f7c.RData: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-11_51c1a9743b40775acf53e9098d7a2f7c.RData -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-11_51c1a9743b40775acf53e9098d7a2f7c.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-11_51c1a9743b40775acf53e9098d7a2f7c.rdb -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-11_51c1a9743b40775acf53e9098d7a2f7c.rdx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-11_51c1a9743b40775acf53e9098d7a2f7c.rdx -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-12_8a974d8395fc1a579c6be1526018f87a.RData: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-12_8a974d8395fc1a579c6be1526018f87a.RData -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-12_8a974d8395fc1a579c6be1526018f87a.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-12_8a974d8395fc1a579c6be1526018f87a.rdb -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-12_8a974d8395fc1a579c6be1526018f87a.rdx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-12_8a974d8395fc1a579c6be1526018f87a.rdx -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-13_cf43ad4a2c8508354026a06f1bafe9dd.RData: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-13_cf43ad4a2c8508354026a06f1bafe9dd.RData -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-13_cf43ad4a2c8508354026a06f1bafe9dd.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-13_cf43ad4a2c8508354026a06f1bafe9dd.rdb -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-13_cf43ad4a2c8508354026a06f1bafe9dd.rdx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-13_cf43ad4a2c8508354026a06f1bafe9dd.rdx -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-14_e55f71d151e35e93462fb8da2c15cda7.RData: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-14_e55f71d151e35e93462fb8da2c15cda7.RData -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-14_e55f71d151e35e93462fb8da2c15cda7.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-14_e55f71d151e35e93462fb8da2c15cda7.rdb -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-14_e55f71d151e35e93462fb8da2c15cda7.rdx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-14_e55f71d151e35e93462fb8da2c15cda7.rdx -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-15_c6858e6b9aec06e9c08fb751d143bbaa.RData: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-15_c6858e6b9aec06e9c08fb751d143bbaa.RData -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-15_c6858e6b9aec06e9c08fb751d143bbaa.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-15_c6858e6b9aec06e9c08fb751d143bbaa.rdb -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-15_c6858e6b9aec06e9c08fb751d143bbaa.rdx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-15_c6858e6b9aec06e9c08fb751d143bbaa.rdx -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-16_dfb489ca671eb30aada911c7e197e59d.RData: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-16_dfb489ca671eb30aada911c7e197e59d.RData -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-16_dfb489ca671eb30aada911c7e197e59d.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-16_dfb489ca671eb30aada911c7e197e59d.rdb -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-16_dfb489ca671eb30aada911c7e197e59d.rdx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-16_dfb489ca671eb30aada911c7e197e59d.rdx -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-17_be6831e96ef363339f71461780d9139b.RData: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-17_be6831e96ef363339f71461780d9139b.RData -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-17_be6831e96ef363339f71461780d9139b.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-17_be6831e96ef363339f71461780d9139b.rdb -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-17_be6831e96ef363339f71461780d9139b.rdx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-17_be6831e96ef363339f71461780d9139b.rdx -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-18_ea55effbb8c4e585143dfc686130a28a.RData: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-18_ea55effbb8c4e585143dfc686130a28a.RData -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-18_ea55effbb8c4e585143dfc686130a28a.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-18_ea55effbb8c4e585143dfc686130a28a.rdb -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-18_ea55effbb8c4e585143dfc686130a28a.rdx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-18_ea55effbb8c4e585143dfc686130a28a.rdx -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-19_61c53c10543cdf242387e06d4a6a325e.RData: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-19_61c53c10543cdf242387e06d4a6a325e.RData -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-19_61c53c10543cdf242387e06d4a6a325e.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-19_61c53c10543cdf242387e06d4a6a325e.rdb -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-19_61c53c10543cdf242387e06d4a6a325e.rdx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-19_61c53c10543cdf242387e06d4a6a325e.rdx -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-1_7d0e97d16d7d093d5b42e50103ccc145.RData: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-1_7d0e97d16d7d093d5b42e50103ccc145.RData -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-1_7d0e97d16d7d093d5b42e50103ccc145.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-1_7d0e97d16d7d093d5b42e50103ccc145.rdb -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-1_7d0e97d16d7d093d5b42e50103ccc145.rdx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-1_7d0e97d16d7d093d5b42e50103ccc145.rdx -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-20_428166563a4d8667af9a4d9a3d38a940.RData: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-20_428166563a4d8667af9a4d9a3d38a940.RData -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-20_428166563a4d8667af9a4d9a3d38a940.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-20_428166563a4d8667af9a4d9a3d38a940.rdb -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-20_428166563a4d8667af9a4d9a3d38a940.rdx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-20_428166563a4d8667af9a4d9a3d38a940.rdx -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-21_ee1eea167b592e7b336e1bf4775ef2d9.RData: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-21_ee1eea167b592e7b336e1bf4775ef2d9.RData -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-21_ee1eea167b592e7b336e1bf4775ef2d9.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-21_ee1eea167b592e7b336e1bf4775ef2d9.rdb -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-21_ee1eea167b592e7b336e1bf4775ef2d9.rdx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-21_ee1eea167b592e7b336e1bf4775ef2d9.rdx -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-22_781b5003085bd756a5e8fc061c28e436.RData: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-22_781b5003085bd756a5e8fc061c28e436.RData -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-22_781b5003085bd756a5e8fc061c28e436.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-22_781b5003085bd756a5e8fc061c28e436.rdb -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-22_781b5003085bd756a5e8fc061c28e436.rdx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-22_781b5003085bd756a5e8fc061c28e436.rdx -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-23_5ea15adae67079d8719c83b99469ce0a.RData: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-23_5ea15adae67079d8719c83b99469ce0a.RData -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-23_5ea15adae67079d8719c83b99469ce0a.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-23_5ea15adae67079d8719c83b99469ce0a.rdb -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-23_5ea15adae67079d8719c83b99469ce0a.rdx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-23_5ea15adae67079d8719c83b99469ce0a.rdx -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-24_033eb3e60d56b8fb5a1474efcfa12d41.RData: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-24_033eb3e60d56b8fb5a1474efcfa12d41.RData -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-24_033eb3e60d56b8fb5a1474efcfa12d41.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-24_033eb3e60d56b8fb5a1474efcfa12d41.rdb -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-24_033eb3e60d56b8fb5a1474efcfa12d41.rdx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-24_033eb3e60d56b8fb5a1474efcfa12d41.rdx -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-25_5c0f3af19510e7bcf7b327e1f8235a03.RData: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-25_5c0f3af19510e7bcf7b327e1f8235a03.RData -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-25_5c0f3af19510e7bcf7b327e1f8235a03.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-25_5c0f3af19510e7bcf7b327e1f8235a03.rdb -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-25_5c0f3af19510e7bcf7b327e1f8235a03.rdx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-25_5c0f3af19510e7bcf7b327e1f8235a03.rdx -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-26_f47c47918a569d26accd8776d65c4d32.RData: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-26_f47c47918a569d26accd8776d65c4d32.RData -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-26_f47c47918a569d26accd8776d65c4d32.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-26_f47c47918a569d26accd8776d65c4d32.rdb -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-26_f47c47918a569d26accd8776d65c4d32.rdx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-26_f47c47918a569d26accd8776d65c4d32.rdx -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-2_7b35662aa612b3282486f013f29b7cbe.RData: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-2_7b35662aa612b3282486f013f29b7cbe.RData -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-2_7b35662aa612b3282486f013f29b7cbe.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-2_7b35662aa612b3282486f013f29b7cbe.rdb -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-2_7b35662aa612b3282486f013f29b7cbe.rdx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-2_7b35662aa612b3282486f013f29b7cbe.rdx -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-3_2b7a7863ff95c6e3b1cbc6994c39689f.RData: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-3_2b7a7863ff95c6e3b1cbc6994c39689f.RData -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-3_2b7a7863ff95c6e3b1cbc6994c39689f.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-3_2b7a7863ff95c6e3b1cbc6994c39689f.rdb -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-3_2b7a7863ff95c6e3b1cbc6994c39689f.rdx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-3_2b7a7863ff95c6e3b1cbc6994c39689f.rdx -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-4_026f1d4e2418ba1b4cc0abd726e6096a.RData: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-4_026f1d4e2418ba1b4cc0abd726e6096a.RData -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-4_026f1d4e2418ba1b4cc0abd726e6096a.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-4_026f1d4e2418ba1b4cc0abd726e6096a.rdb -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-4_026f1d4e2418ba1b4cc0abd726e6096a.rdx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-4_026f1d4e2418ba1b4cc0abd726e6096a.rdx -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-5_6b72ebddcf9904abe48243aa1d15d59d.RData: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-5_6b72ebddcf9904abe48243aa1d15d59d.RData -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-5_6b72ebddcf9904abe48243aa1d15d59d.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-5_6b72ebddcf9904abe48243aa1d15d59d.rdb -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-5_6b72ebddcf9904abe48243aa1d15d59d.rdx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-5_6b72ebddcf9904abe48243aa1d15d59d.rdx -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-6_211aef65edc6632bf170c6d362374acf.RData: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-6_211aef65edc6632bf170c6d362374acf.RData -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-6_211aef65edc6632bf170c6d362374acf.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-6_211aef65edc6632bf170c6d362374acf.rdb -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-6_211aef65edc6632bf170c6d362374acf.rdx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-6_211aef65edc6632bf170c6d362374acf.rdx -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-7_67b0a668bee133c8f0790474c71af4d9.RData: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-7_67b0a668bee133c8f0790474c71af4d9.RData -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-7_67b0a668bee133c8f0790474c71af4d9.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-7_67b0a668bee133c8f0790474c71af4d9.rdb -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-7_67b0a668bee133c8f0790474c71af4d9.rdx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-7_67b0a668bee133c8f0790474c71af4d9.rdx -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-8_2ab1bd328e66b08a35ba98d9e15fbec8.RData: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-8_2ab1bd328e66b08a35ba98d9e15fbec8.RData -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-8_2ab1bd328e66b08a35ba98d9e15fbec8.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-8_2ab1bd328e66b08a35ba98d9e15fbec8.rdb -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-8_2ab1bd328e66b08a35ba98d9e15fbec8.rdx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-8_2ab1bd328e66b08a35ba98d9e15fbec8.rdx -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-9_7c4795a27949f9e9bc4ddda935329ea6.RData: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-9_7c4795a27949f9e9bc4ddda935329ea6.RData -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-9_7c4795a27949f9e9bc4ddda935329ea6.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-9_7c4795a27949f9e9bc4ddda935329ea6.rdb -------------------------------------------------------------------------------- /1_lm/lm_cache/html/unnamed-chunk-9_7c4795a27949f9e9bc4ddda935329ea6.rdx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_cache/html/unnamed-chunk-9_7c4795a27949f9e9bc4ddda935329ea6.rdx -------------------------------------------------------------------------------- /1_lm/lm_files/figure-html/unnamed-chunk-21-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_files/figure-html/unnamed-chunk-21-1.png -------------------------------------------------------------------------------- /1_lm/lm_files/figure-html/unnamed-chunk-22-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_files/figure-html/unnamed-chunk-22-1.png -------------------------------------------------------------------------------- /1_lm/lm_files/figure-html/unnamed-chunk-23-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_files/figure-html/unnamed-chunk-23-1.png -------------------------------------------------------------------------------- /1_lm/lm_files/figure-html/unnamed-chunk-24-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_files/figure-html/unnamed-chunk-24-1.png -------------------------------------------------------------------------------- /1_lm/lm_files/figure-html/unnamed-chunk-25-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_files/figure-html/unnamed-chunk-25-1.png -------------------------------------------------------------------------------- /1_lm/lm_files/figure-html/unnamed-chunk-3-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/1_lm/lm_files/figure-html/unnamed-chunk-3-1.png -------------------------------------------------------------------------------- /2_mnl/2_mnl.Rproj: -------------------------------------------------------------------------------- 1 | Version: 1.0 2 | 3 | RestoreWorkspace: Default 4 | SaveWorkspace: Default 5 | AlwaysSaveHistory: Default 6 | 7 | EnableCodeIndexing: Yes 8 | UseSpacesForTab: Yes 9 | NumSpacesForTab: 2 10 | Encoding: UTF-8 11 | 12 | RnwWeave: Sweave 13 | LaTeX: pdfLaTeX 14 | -------------------------------------------------------------------------------- /2_mnl/images/Bayesian_Inference.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/2_mnl/images/Bayesian_Inference.png -------------------------------------------------------------------------------- /2_mnl/images/beta[2]beta[3].pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/2_mnl/images/beta[2]beta[3].pdf -------------------------------------------------------------------------------- /2_mnl/images/beta[2]beta[3].png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/2_mnl/images/beta[2]beta[3].png -------------------------------------------------------------------------------- /2_mnl/mnl.R: -------------------------------------------------------------------------------- 1 | # Code for estimating multinomial logit models in Stan 2 | # Elea McDonnell Feit, eleafeit@gmail.com 3 | # Last updated: 22 June 2017 4 | 5 | # Copyright 2017, Kevin Van Horn & Elea McDonnell Feit 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # 10 | # You may obtain a copy of the License at 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | # ========== Load libraries and set options ========== 20 | library(rstan) 21 | library(shinystan) 22 | 23 | # writes a compiled Stan program to the disk to avoid recompiling 24 | rstan_options(auto_write=TRUE) 25 | # allows Stan chains to run in parallel on multiprocessor machines 26 | options(mc.cores = parallel::detectCores()) 27 | 28 | # ========== Test Stan with synthetic data ============ 29 | 30 | # function to generate mnl data 31 | generate_mnl_data <- function(N=1000, C=3, beta=c(1, -2)){ 32 | K <- length(beta) 33 | Y <- rep(NA, N) 34 | X <- list(NULL) 35 | for (i in 1:N) { 36 | X[[i]] <- matrix(rnorm(C*K), ncol=K) # normal covariates 37 | Y[i] <- sample(x=C, size=1, prob=exp(X[[i]]%*%beta)) # logit formula 38 | } 39 | list(N=N, C=C, K=K, Y=Y, X=X) 40 | } 41 | 42 | d1 <- generate_mnl_data(N=1000, C=3, beta=c(1, -2)) 43 | str(d1) 44 | 45 | test.stan <- stan(file="mnl.stan", data=d1, iter=1000, chains=4) 46 | summary(test.stan)$summary 47 | 48 | summary(test.stan)$c_summary 49 | 50 | plot(test.stan, plotfun="trace") 51 | 52 | plot(test.stan) 53 | 54 | plot(test.stan, plotfun="hist") 55 | 56 | plot(test.stan, plotfun="dens") 57 | 58 | # ========= Read in and inspect Chocolate data ========= 59 | choc.df <- read.csv("cbc_chocolate.csv") 60 | head(choc.df) 61 | 62 | summary(choc.df) 63 | 64 | mosaicplot(~ Brand + Chosen, data=choc.df) 65 | 66 | mosaicplot(~ Type + Chosen, data=choc.df) 67 | 68 | mosaicplot(~ Price + Chosen, data=choc.df) 69 | 70 | # ========== Prep the Chocolate data for Stan ========= 71 | # illustrating alternative coding schemes (not required) 72 | choc.df[1:10,] 73 | model.matrix(~ Brand, data = choc.df)[1:10,] 74 | model.matrix(~ Brand, data = choc.df, contrasts = list(Brand = "contr.sum"))[1:10,] 75 | 76 | # coding the chocolate data 77 | choc.contrasts <- list(Brand = "contr.sum", Type = "contr.sum") 78 | choc.coded <- model.matrix(~ Brand + Type, data = choc.df, contrasts = choc.contrasts) 79 | choc.coded <- choc.coded[,2:ncol(choc.coded)] # remove intercept 80 | # Fix the bad labels from contr.sum 81 | choc.names <- c("BrandDove", "BrandGhirardelli", "BrandGodiva", "BrandHersheys", 82 | "TypeDark", "TypeDarkNuts", "TypeMilk", "TypeMilkNuts") 83 | colnames(choc.coded) <- choc.names 84 | choc.df <- cbind(choc.df, choc.coded) 85 | head(choc.df) 86 | 87 | # munging into stan format 88 | unique(choc.df$Ind) 89 | unique(choc.df$Trial) 90 | 91 | R <- length(unique(choc.df$Ind)) 92 | S <- length(unique(choc.df$Trial)) 93 | Y <- rep(NA, R*S) 94 | X <- vector("list", R*S) 95 | n <- 1 96 | for (r in unique(choc.df$Ind)) { # respondents 97 | for (s in unique(choc.df$Trial)){ # choice scenarios 98 | scenario <- choc.df[choc.df$Ind==r & choc.df$Trial==s,] 99 | X[[n]] <- data.matrix(scenario[,c(7, 9:16)]) # price and coded brand and type 100 | Y[n] <- scenario$Alt[as.logical(scenario$Chosen)] 101 | n <- n + 1 102 | } 103 | } 104 | 105 | str(Y) 106 | str(X) 107 | 108 | choc.standata <- list(N=length(Y), C=3, K=9, Y=Y, X=X) 109 | rm(Y, X, n, R, S, r, s, choc.contrasts, scenario, choc.coded) 110 | 111 | # ========== Estimate mnl model for Chocolate data in Stan ========== 112 | choc.stan <- stan(file="mnl.stan", data=choc.standata) # default: chains=4, iter=2000 113 | 114 | summary(choc.stan)$summary 115 | 116 | plot(choc.stan, plotfun="trace") 117 | 118 | plot(choc.stan) 119 | 120 | data.frame(params=c("Price", choc.names), check.names=FALSE, 121 | summary(choc.stan, pars=c("beta"))$summary) 122 | 123 | # ========= Explore posterior using ShinyStan ========== 124 | launch_shinystan(choc.stan) 125 | 126 | # ========= Share prediction based on model ========== 127 | # computing shares from a point estimate of beta (not good practice) 128 | shares.mnl.point <- function(beta, # vector of parameters (part-worths) 129 | X) { # attribute matrix X for scenario (coded) 130 | if (length(beta) != ncol(X)) 131 | stop("length of beta doesn't match columns in X") 132 | V <- exp(X %*% beta) 133 | data.frame(shares=V/sum(V), X) 134 | } 135 | 136 | 137 | beta.mean <- summary(choc.stan)$summary[1:9,1] 138 | beta.mean 139 | shares.mnl.point(beta.mean, choc.standata$X[[1]]) 140 | 141 | # computing the posterior draws of shares 142 | shares.mnl.post <- function(draws, # matrix of draws (use extract()) 143 | X) { # attribute matrix X for scenario 144 | shares <- matrix(NA, nrow=nrow(draws), ncol=nrow(X)) 145 | for (draw in 1:nrow(draws)) { 146 | shares[draw,] <- shares.mnl.point(draws[draw,], X)[,1] 147 | } 148 | shares 149 | } 150 | 151 | beta.draws <- extract(choc.stan, pars="beta")$beta 152 | head(beta.draws) 153 | shares.draw <- shares.mnl.post(beta.draws, choc.standata$X[[1]]) 154 | 155 | summary(shares.draw) 156 | 157 | apply(shares.draw, 2, quantile, probs=c(0.5, 0.025, 0.975)) 158 | 159 | -------------------------------------------------------------------------------- /2_mnl/mnl.Rmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Stan Tutorial" 3 | subtitle: "Module 2: Multinomial Logit Model" 4 | author: "Kevin Van Horn and Elea McDonnell Feit" 5 | date: "June 25, 2017" 6 | output: ioslides_presentation 7 | --- 8 | 9 | ```{r setup, include=FALSE} 10 | knitr::opts_chunk$set(echo = TRUE, eval=TRUE) 11 | ``` 12 | 13 | ## Module objective 14 | In this module, we will build and estimate a multinomial logit model using Stan. 15 | 16 | This will allow us to: 17 | - Start to learn the Stan model syntax. 18 | - Go through the Bayesian inference process more carefully and completely. 19 | 20 | Think of Module 1 as the "Quick Start Guide," while Module 2 is a more comprehensive "How-To Guide". 21 | 22 | # Stan model syntax basics 23 | 24 | ## Required Stan model components 25 | ```{r, eval=FALSE, echo=FALSE} 26 | # outputvar="stanjunk" is a kludge to make RMarkdown ignore the 27 | # stan code. This code reads directly from mnl.stan. 28 | # See http://rmarkdown.rstudio.com/authoring_knitr_engines.html#stan 29 | # for alternative approach to stan in RMarkdown. 30 | ``` 31 | 32 | ```{stan, output.var="stanjunk", eval=FALSE} 33 | data{ 34 | // declarations of data go here 35 | } 36 | parameters{ 37 | // declarations of parameters go here 38 | } 39 | model{ 40 | // distributions for observables and 41 | //parameters (priors) are specified here 42 | } 43 | ``` 44 | There are a few other types of blocks in a Stan model; the [Stan Language User's Guide and Reference Manual](https://github.com/stan-dev/stan/releases/download/v2.15.0/stan-reference-2.15.0.pdf) is a complete and relatively friendly guide. 45 | 46 | ## Stan types: scalars and arrays 47 | Every variable in a Stan model must have a declared type. Common types are: 48 | 49 | **Scalars** (single variable) 50 | ```{stan, eval=FALSE, output.var="stanjunk",} 51 | real myvar1; 52 | int myvar2; 53 | ``` 54 | **Arrays** 55 | ```{stan, eval=FALSE, output.var="stanjunk",} 56 | real myarray1[2, 3]; # 2 x 3 array of reals 57 | int myarray2[5, 6, myvar1]; # dims may be other declared variables 58 | ``` 59 | 60 | ## Stan types: vectors and matrices 61 | Specific `vector` and `matrix` types must be used when you want to do matrix operations or use multivariate probability distributions. 62 | 63 | **Vectors and Matrices** 64 | ```{stan, eval=FALSE, output.var="stanjunk",} 65 | vector[25] myvector; // column vector of length 25 66 | row_vector[10] myrowvector; // row vector of length 25 67 | matrix[10, 25] mymatrix; 68 | ``` 69 | - You can multiply a `vector` by a `matrix` or a `matrix` by a `matrix` (assuming dimenions match). 70 | - You can not use multiplication with an `array`. 71 | - Elements of a `vector` or `matrix` are always `real`. 72 | 73 | ## Stan types: array of vectors or matrices 74 | You are also allowed to declare an array of vectors or an array of matricies. 75 | 76 | **Combining matrices or vectors with arrays** 77 | ```{stan, eval=FALSE, output.var="stanjunk",} 78 | matrix[25, 3] myarrayofmatrices[1000]; 79 | ``` 80 | - This produces an array of 1000 matrices that each have 25 rows and 3 columns. 81 | 82 | ## Constraints 83 | Any declaration can include a constraint, which applies to all elements in the variable. 84 | ```{stan, eval=FALSE, output.var="stanjunk",} 85 | real myarray2[10, 20]; 86 | ``` 87 | - For data, the constraint provides some data checking. 88 | - For parameters, it constrains the posterior of the parameter. 89 | - Note the model must have support (non-zero density) at every value of the parameters 90 | that meets their declared constraints. If the declared parameter constraints are less strict than the support, the samplers and optimizers may have any of a number of pathologies including just getting stuck, failure to initialize, etc. 91 | 92 | ## Summary of basic Stan types 93 | **Scalars** 94 | ```{stan, eval=FALSE, output.var="stanjunk"} 95 | int varname; 96 | real varname; 97 | ``` 98 | **Arrays** 99 | ```{stan, eval=FALSE, output.var="stanjunk"} 100 | int varname[dim1, dim2, ...]; 101 | real varname[dim1, dim2, ...]; 102 | ``` 103 | **Vectors and Matrices** 104 | ```{stan, eval=FALSE, output.var="stanjunk"} 105 | vector[length] varname[dim1, dim2, ...]; 106 | row_vector[length] varname[dim1, dim2, ...]; 107 | matrix[rows, cols] varname[dim1, dim2, ...]; 108 | ``` 109 | 110 | ## Stan data types: others 111 | - While these are the main data types we use regularly, there are many other data types that Stan provides. 112 | - We'll leave it to you to discover those in the context of specific examples where they are useful. 113 | 114 | ## Specifying probability distributions 115 | The core of any Stan model is the `model` block, where you define probability distributions for the observables. For instance, in our linear model, we defined y as normally distributed. 116 | ```{stan, eval=FALSE, output.var="stanjunk"} 117 | y ~ normal(beta0 + X*beta, sigma); 118 | ``` 119 | This sort of declaration of the distribution of an observable is called a **sampling statement**. 120 | 121 | ## Stan probability distributions 122 | Stan provides many, many distributions so that you can flexibly define the model. 123 | ```{stan, eval=FALSE, output.var="stanjunk"} 124 | y ~ normal(mu, sigma); 125 | y ~ student_t(nu, mu, sigma); 126 | y ~ cauchy(mu, sigma); 127 | ``` 128 | ```{stan, eval=FALSE, output.var="stanjunk"} 129 | y ~ lognormal(mu, sigma); 130 | y ~ exponential(beta); 131 | ``` 132 | ```{stan, eval=FALSE, output.var="stanjunk"} 133 | theta ~ beta(alpha, beta); 134 | ``` 135 | ```{stan, eval=FALSE, output.var="stanjunk"} 136 | n ~ poisson(lambda); 137 | ``` 138 | ```{stan, eval=FALSE, output.var="stanjunk"} 139 | y ~ categorical_logit(beta); 140 | ``` 141 | 142 | ## Referring to data and parameters 143 | To pull out an element of an array or matrix you can use `[ ]`: 144 | ```{stan, eval=FALSE, output.var="stanjunk",} 145 | matrix[25, 3] myarrayofmatrices[1000]; # declared in data 146 | ... 147 | myarrayofmatrices[1]; // first matrix in array of 1000 148 | myarrayofmatrices[1, 2, 3]; // element in second row and third column 149 | // in the first matrix in array of 1000 150 | ``` 151 | - When indexing the array of matrices, array dimensions come first, then the row and column. 152 | + myarrayofmatrices[1] is the first matrix in the array 153 | + myarrayofmatrices[1,2,3] is the element in the second row and third column 154 | 155 | ## Priors 156 | - If you don't specify a distribution for a parameter, Stan will assume a *uniform* prior on that parameter. 157 | + We used uniform priors in our linear model. 158 | - You specify a prior for a parameter by defining a distribution for the parameter in the `model` block. 159 | - Since the NUTS algorithm used by default in Stan is gradient-based, there is no benefit to sticking to conjugate priors (as in Gibbs sampling). 160 | 161 | ## Parameters 162 | - All parameters in a Stan model must be continuous, i.e. you can't have an `int` type parameter. 163 | - This means you can't use models with latent discrete variables. 164 | + For instance you can't estimate membership variables in a mixture model. 165 | - This is due to the fact that the MCMC algorithm that Stan uses to generate draws from the posterior relies on gradients. 166 | - Discrete observables (like we have in the mnl) are fine! 167 | 168 | ## Learning more about Stan syntax 169 | - The best way to learn about the Stan language is to read the [Stan Modeling Language: 170 | User’s Guide and Reference Manual](https://github.com/stan-dev/stan/releases/download/v2.16.0/stan-reference-2.16.0.pdf), which is fairly easy to read and introduces much of the syntax in the context of specific examples. 171 | - If you want to learn about how to use Stan from R, the [rstan vignette](https://cran.r-project.org/web/packages/rstan/vignettes/rstan.html) will walk you through a typical analysis with rstan, very similar to this tutorial. 172 | - There is a very active [Stan forum](http://discourse.mc-stan.org/) board, but since Stan does change over time, make sure the post you are looking at is recent. 173 | 174 | # Stan code for mnl 175 | 176 | ## Stan model files 177 | - As projects get more complicated, it is a good idea to keep the Stan code in separate file with the .stan extension. 178 | - One advantage is that R Studio will color code the syntax in a .stan file. 179 | - Our Stan model is in `mnl.stan`. You can check it out by opening it in RStudio or any text editor. 180 | 181 | ## Data 182 | ```{stan, eval=FALSE, output.var="stanjunk"} 183 | data { 184 | int C; // # of alternatives (choices) in each scenario 185 | int N; // # of observations 186 | int K; // # of covariates 187 | int Y[N]; // # observed choices 188 | matrix[C,K] X[N]; // # matrix of attributes for each obs 189 | } 190 | ``` 191 | 192 | ## Parameters 193 | ```{stan, eval=FALSE, output.var="stanjunk"} 194 | parameters { 195 | vector[K] beta; 196 | } 197 | ``` 198 | 199 | ## Model 200 | ```{stan, eval=FALSE, output.var="stanjunk"} 201 | model { 202 | # priors 203 | // beta ~ normal(0,3); // often used in mcmc 204 | beta ~ cauchy(0, 2.5); // better 205 | # model 206 | for (i in 1:N) 207 | Y[i] ~ categorical_logit(X[i]*beta); 208 | } 209 | ``` 210 | Note that the scale of the prior on `beta` assumes that the predictors are standardized. See [Gelman (2008)](http://www.stat.columbia.edu/~gelman/research/published/standardizing7.pdf) and [Gelman(2009)](http://www.stat.columbia.edu/~gelman/research/published/priors11.pdf) for more details. 211 | 212 | # Testing with synthetic data 213 | 214 | ## Bayesian statistical inference 215 | ```{r, fig.align="center", echo=FALSE} 216 | knitr::include_graphics("images/Bayesian_Inference.png") 217 | ``` 218 | 219 | ## Testing with synthetic data 220 | - When you are building any statistical estimation routine, it is a good idea to test it out with data that you generated synthetically. 221 | - By running the model with data where you know the parameters, you can compare to confirm that the estimation routine is is working properly. 222 | + In Bayesian inference, we are looking for a posterior that "covers" the true parameter values. 223 | - This also allows you to build the analysis before you've got the data. 224 | + We've never been stuck waiting for data, but maybe you have ;) 225 | 226 | ## Libraries and options 227 | Now it is time to start working in R. Open up `mnl.R` and begin by loading a few libraries and setting a few options. 228 | ```{r, message=FALSE, warning=FALSE} 229 | library(rstan) 230 | library(shinystan) 231 | 232 | # writes a compiled Stan program to the disk to avoid recompiling 233 | rstan_options(auto_write=TRUE) 234 | # allows Stan chains to run in parallel on multiprocessor machines 235 | options(mc.cores = parallel::detectCores()) 236 | ``` 237 | 238 | ## Generating synthetic data 239 | R function for generating choice data from the multinomial logit model: 240 | ```{r} 241 | generate_mnl_data <- function(N=1000, C=3, beta=c(1, -2)){ 242 | K <- length(beta) 243 | Y <- rep(NA, N) 244 | X <- list(NULL) 245 | for (i in 1:N) { 246 | X[[i]] <- matrix(rnorm(C*K), ncol=K) # normal covariates 247 | Y[i] <- sample(x=C, size=1, prob=exp(X[[i]]%*%beta)) # logit 248 | } 249 | list(N=N, C=C, K=K, Y=Y, X=X) 250 | } 251 | ``` 252 | Our function returns data as a list of objects of the same names and dimensions as the declared data in `mnl.stan`. This is the format that `stan()` requires. 253 | 254 | ## Synthetic data 255 | Use the function to generate synthetic data. Remember the true value of `beta` is 1, -2. 256 | ```{r} 257 | d1 <- generate_mnl_data(N=1000, C=3, beta=c(1, -2)) 258 | str(d1) 259 | 260 | ``` 261 | 262 | ## Run Stan model on synthetic data 263 | To run the model, we call the `stan()` function. Note that we pass filename `mnl.stan` to the stan function. Make sure that file is in the working directory (or provide a full path)! 264 | ```{r, cache=TRUE} 265 | test.stan <- stan(file="mnl.stan", data=d1, iter=1000, chains=4) 266 | summary(test.stan)$summary 267 | ``` 268 | 269 | ## Running multiple chains 270 | When we called Stan, we asked it to run the NUTS algorithm four times (independantly). This is a good practice as it allows you to assess the convergence of the algorithm! We can can access the summary for indivdual chains with the command: 271 | ```{r} 272 | summary(test.stan)$c_summary 273 | ``` 274 | 275 | ## Rhat and n_eff 276 | - The summary reports two statistics that allow us to evaluate whether the algorithm is working properly. 277 | - **Rhat** compares the variation within chains to the variation between chains and should be close to 1 (also called the potential scale reduction factor) 278 | - **n_eff** is a crude approximation of the effective number of independent draws from the posterior 279 | ```{r} 280 | summary(test.stan)$summary 281 | ``` 282 | 283 | ## Rules of thumb for Rhat and n_eff 284 | - **Rhat** should be less than 1.1 for all parameters (and not too much less than 1) 285 | - **n_eff** should be greater than 400 for all parameters (although we sometimes go with n_eff around 100, for long-running models) 286 | 287 | If these conditions are not met, then 288 | 1. Inspect the traceplots to see what is happening 289 | 2. Run the model longer 290 | 3. (Pro tip) Consider re-parameterizing to improve computation 291 | 292 | ## Traceplots look fine 293 | ```{r} 294 | plot(test.stan, plotfun="trace") 295 | ``` 296 | 297 | ## Plot of the parameter estimates 298 | ```{r, message=FALSE, warning=FALSE} 299 | plot(test.stan) 300 | ``` 301 | 302 | ## Histograms of the posteriors 303 | ```{r, message=FALSE, warning=FALSE} 304 | plot(test.stan, plotfun="hist") 305 | ``` 306 | 307 | ## Density plots of the posteriors 308 | ```{r, message=FALSE, warning=FALSE} 309 | plot(test.stan, plotfun="dens") 310 | ``` 311 | 312 | ## Model confirmed 313 | 1. Convergence is good. 314 | 2. Estimated posteriors of `beta` include support for the true values. 315 | 316 | This synthetic data test confirmes that Stan seems to be working well with this model. Now we can apply it to some real data. 317 | 318 | # Choice-Based Conjoint Data for Chocolate Bars 319 | 320 | ## Choice-based conjoint data for chocolate bars 321 | - Fourteen respondents each answered 25 choice tasks where they selected from among three chocolate bars. 322 | - Three attributes 323 | + Brand: Hersheys, Dove, Lindt, Godiva, Ghirardelli 324 | + Type: Milk, Milk with nuts, Dark, Dark with nuts, White 325 | - Data was generously shared by Betty Kim-Viechnicki and Joe Cable. 326 | + Originally described in Kim-Viechnicki, B.E., Burla, Y., Feit, E., Plassmann, H., & Kable, J.W. (2013). Decisions about chocolate are processed differently than decisions on gambles: Evidence from eye-tracking. 327 | 328 | ## Read in the chocolate data 329 | ```{r} 330 | choc.df <- read.csv("cbc_chocolate.csv") 331 | choc.df 332 | ``` 333 | ## Summary of chocolate data 334 | ```{r} 335 | summary(choc.df) 336 | ``` 337 | 338 | ## Data inspection: **Brand** v. choice 339 | ```{r} 340 | mosaicplot(~ Brand + Chosen, data=choc.df) 341 | ``` 342 | 343 | ## Data inspection: **Type** v. choice 344 | ```{r} 345 | mosaicplot(~ Type + Chosen, data=choc.df) 346 | ``` 347 | 348 | ## Data inspection: **Price** v. choice 349 | ```{r} 350 | mosaicplot(~ Price + Chosen, data=choc.df) 351 | ``` 352 | 353 | # Preparing the Chocolate CBC data for Stan 354 | 355 | ## Coding discrete variables 356 | As with the computer ratings-based conjoint data in Module 1, we have two discrete attributes that need to be coded: `Brand` and `Type`. 357 | ```{r} 358 | choc.df 359 | ``` 360 | 361 | ## Dummy coding 362 | As in, "Just don't do it, dummy." Unfortunately, the default coding in `model.matrix()` is dummy coding. This coding is useful in experiments where there is a control condition, but is less useful for discrete attributes in conjoint where no level is "special". 363 | ```{r} 364 | model.matrix(~ Brand, data = choc.df) 365 | ``` 366 | 367 | ## Effects coding (1) 368 | - A better coding scheme is the sum-to-one coding. 369 | - No level is designated as the base level. 370 | - All parameter values can be interpreted as "relative to the average." 371 | - You can compute a parameter value for the left-out level (negative sum of the parameters for other levels). 372 | - It is easier to put a symetric prior on these parameters. See [Kevin's blog post](http://bayesium.com/a-symmetric-effects-prior/). 373 | 374 | ## Effects coding (2) 375 | ```{r} 376 | model.matrix(~ Brand, data = choc.df, 377 | contrasts = list(Brand = "contr.sum")) 378 | ``` 379 | 380 | ## Coding the chocolate data 381 | ```{r} 382 | choc.contrasts <- list(Brand = "contr.sum", Type = "contr.sum") 383 | choc.coded <- model.matrix(~ Brand + Type, data = choc.df, 384 | contrasts = choc.contrasts) 385 | choc.coded <- choc.coded[,2:ncol(choc.coded)] # remove intercept 386 | # Fix the bad labels from contr.sum 387 | choc.names <- c("BrandDove", "BrandGhirardelli", "BrandGodiva", 388 | "BrandHersheys", "TypeDark", "TypeDarkNuts", 389 | "TypeMilk", "TypeMilkNuts") 390 | colnames(choc.coded) <- choc.names 391 | ``` 392 | 393 | ## Append the coded attributes to `choc.df` 394 | ```{r} 395 | choc.df <- cbind(choc.df, choc.coded) 396 | head(choc.df) 397 | ``` 398 | 399 | ## Respondents and questions 400 | ```{r} 401 | unique(choc.df$Ind) 402 | unique(choc.df$Trial) 403 | ``` 404 | 405 | ## Creating objects like those declared in `mnl.stan` 406 | ```{r} 407 | R <- length(unique(choc.df$Ind)) 408 | S <- length(unique(choc.df$Trial)) 409 | Y <- rep(NA, R*S) 410 | X <- vector("list", R*S) 411 | n <- 1 412 | for (r in unique(choc.df$Ind)) { # respondents 413 | for (s in unique(choc.df$Trial)){ # choice scenarios 414 | scenario <- choc.df[choc.df$Ind==r & choc.df$Trial==s,] 415 | X[[n]] <- data.matrix(scenario[,c(7, 9:16)]) # price and coded brand and type 416 | Y[n] <- scenario$Alt[as.logical(scenario$Chosen)] 417 | n <- n + 1 418 | } 419 | } 420 | ``` 421 | 422 | ## Double check `X` and `Y` 423 | ```{r} 424 | str(Y) 425 | str(X) 426 | ``` 427 | 428 | ## Create the list object for `stan()` 429 | ```{r, cache=TRUE} 430 | choc.standata <- list(N=length(Y), C=3, K=9, Y=Y, X=X) 431 | rm(Y, X, n, R, S, r, s, choc.contrasts, scenario, choc.coded) 432 | ``` 433 | 434 | # Estimating the mnl model using the chocolate data 435 | 436 | ## Call Stan 437 | ```{r} 438 | choc.stan <- stan(file="mnl.stan", data=choc.standata) 439 | # default: chains=4, iter=2000 440 | ``` 441 | 442 | ## Check for convergence (1) 443 | Inspect `Rhat` and `n_eff` in the summary output. 444 | ```{r} 445 | summary(choc.stan)$summary[,c("n_eff", "Rhat")] 446 | ``` 447 | 448 | ## Check for convergence (2) 449 | Eyeball the traceplots. 450 | ```{r} 451 | plot(choc.stan, plotfun="trace") 452 | ``` 453 | 454 | ## Visualize parameter posteriors 455 | ```{r, echo=TRUE, message=FALSE} 456 | plot(choc.stan) 457 | ``` 458 | 459 | ## Summarize the estimates 460 | ```{r} 461 | data.frame(params=c("Price", choc.names), check.names=FALSE, 462 | summary(choc.stan, pars=c("beta"))$summary) 463 | 464 | ``` 465 | 466 | ## More posterior summaries with ShinyStan 467 | ```{r, eval=FALSE} 468 | launch_shinystan(choc.stan) 469 | ``` 470 | 471 | ## The posterior is multidimensional 472 | Don't forget that the posterior is multidimensional. If you find a strong posterior correlation between two parameters, it means those two parameters are partially confounded in the data. 473 | ```{r, fig.align="center", echo=FALSE} 474 | knitr::include_graphics("images/beta[2]beta[3].png") 475 | ``` 476 | 477 | # Simulating choice shares in R using posterior draws 478 | 479 | ## Function for computing shares from a point estimate of beta 480 | ```{r} 481 | shares.mnl.point <- function(beta, # vector of parameters (part-worths) 482 | X) { # attribute matrix X for scenario (coded) 483 | if (length(beta) != ncol(X)) 484 | stop("length of beta doesn't match columns in X") 485 | V <- exp(X %*% beta) 486 | data.frame(shares=V/sum(V), X) 487 | } 488 | ``` 489 | 490 | ## Computing shares from point estimates (1) 491 | Pull out the posterior means of beta. 492 | ```{r} 493 | beta.mean <- summary(choc.stan)$summary[1:9,1] 494 | beta.mean 495 | ``` 496 | 497 | ## Computing shares from point estimates (2) 498 | Compute shares from the point estimates. **This is bad practice as it does not account for our uncertainty in beta!** 499 | ```{r} 500 | shares.mnl.point(beta.mean, choc.standata$X[[1]]) 501 | ``` 502 | 503 | 504 | ## Posterior draws of `beta` 505 | To understand the full posterior of the shares, we should use all of the posterior draws for beta, which represent out uncertainty in `beta`. 506 | ```{r} 507 | beta.draws <- extract(choc.stan, pars="beta")$beta 508 | head(beta.draws) 509 | ``` 510 | 511 | ## Function for computing shares for each of the posterior draws of `beta` 512 | Once you have a set of posterior draws for the model parameters, you can obtain posterior draws for any function of those parameters by computing the function for each draw. 513 | ```{r} 514 | shares.mnl.post <- function(draws, # matrix of draws (use extract()) 515 | X) { # attribute matrix X for scenario 516 | shares <- matrix(NA, nrow=nrow(draws), ncol=nrow(X)) 517 | for (draw in 1:nrow(draws)) { 518 | shares[draw,] <- shares.mnl.point(draws[draw,], X)[,1] 519 | } 520 | shares 521 | } 522 | ``` 523 | 524 | ## Posterior distribution of shares (1) 525 | ```{r} 526 | shares.draw <- shares.mnl.post(beta.draws, choc.standata$X[[1]]) 527 | summary(shares.draw) 528 | ``` 529 | 530 | ## Posterior distribution of shares (2) 531 | ```{r} 532 | apply(shares.draw, 2, quantile, probs=c(0.5, 0.025, 0.975)) 533 | ``` 534 | 535 | # Summary of Module 2 536 | 537 | ## In this module we have 538 | - Built a multinomial logit model in Stan. 539 | + `data`, `parameters` and `model` blocks 540 | + core data types: arrays, vectors, matrices 541 | + sampling statements and distributions 542 | - Tested the model using synthetic data. 543 | - Estimated the model using chocolate bar CBC data. 544 | - Computed the full posterior of shares based on the posterior draws of `beta`. 545 | - Learned about the `Rhat` and `n_eff` measures for assessing convergence. 546 | 547 | Next, we move on to hierarchical models. -------------------------------------------------------------------------------- /2_mnl/mnl.rds: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/2_mnl/mnl.rds -------------------------------------------------------------------------------- /2_mnl/mnl.stan: -------------------------------------------------------------------------------- 1 | # Multinomial logit model 2 | # Elea McDonnell Feit, eleafeit@gmail.com 3 | 4 | data { 5 | int C; // # of alternatives (choices) in each scenario 6 | int N; // # of observations 7 | int K; // # of covariates 8 | int Y[N]; // observed choices 9 | matrix[C,K] X[N]; // matrix of attributes for each obs 10 | } 11 | 12 | parameters { 13 | vector[K] beta; 14 | } 15 | 16 | model { 17 | # priors 18 | // beta ~ normal(0,3); // often used in Gibbs sampling 19 | beta ~ cauchy(0, 2.5); // better 20 | # model 21 | for (i in 1:N) 22 | Y[i] ~ categorical_logit(X[i]*beta); 23 | } 24 | 25 | -------------------------------------------------------------------------------- /2_mnl/mnl_cache/html/__packages: -------------------------------------------------------------------------------- 1 | base 2 | ggplot2 3 | StanHeaders 4 | rstan 5 | shiny 6 | shinystan 7 | -------------------------------------------------------------------------------- /2_mnl/mnl_cache/html/unnamed-chunk-25_a54a27fec6de7639f00fb24e52fdbb18.RData: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/2_mnl/mnl_cache/html/unnamed-chunk-25_a54a27fec6de7639f00fb24e52fdbb18.RData -------------------------------------------------------------------------------- /2_mnl/mnl_cache/html/unnamed-chunk-25_a54a27fec6de7639f00fb24e52fdbb18.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/2_mnl/mnl_cache/html/unnamed-chunk-25_a54a27fec6de7639f00fb24e52fdbb18.rdb -------------------------------------------------------------------------------- /2_mnl/mnl_cache/html/unnamed-chunk-25_a54a27fec6de7639f00fb24e52fdbb18.rdx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/2_mnl/mnl_cache/html/unnamed-chunk-25_a54a27fec6de7639f00fb24e52fdbb18.rdx -------------------------------------------------------------------------------- /2_mnl/mnl_cache/html/unnamed-chunk-45_a3ea144d44a6ba6745a74c44592f568e.RData: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/2_mnl/mnl_cache/html/unnamed-chunk-45_a3ea144d44a6ba6745a74c44592f568e.RData -------------------------------------------------------------------------------- /2_mnl/mnl_cache/html/unnamed-chunk-45_a3ea144d44a6ba6745a74c44592f568e.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/2_mnl/mnl_cache/html/unnamed-chunk-45_a3ea144d44a6ba6745a74c44592f568e.rdb -------------------------------------------------------------------------------- /2_mnl/mnl_cache/html/unnamed-chunk-45_a3ea144d44a6ba6745a74c44592f568e.rdx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/2_mnl/mnl_cache/html/unnamed-chunk-45_a3ea144d44a6ba6745a74c44592f568e.rdx -------------------------------------------------------------------------------- /2_mnl/mnl_files/figure-html/unnamed-chunk-28-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/2_mnl/mnl_files/figure-html/unnamed-chunk-28-1.png -------------------------------------------------------------------------------- /2_mnl/mnl_files/figure-html/unnamed-chunk-29-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/2_mnl/mnl_files/figure-html/unnamed-chunk-29-1.png -------------------------------------------------------------------------------- /2_mnl/mnl_files/figure-html/unnamed-chunk-30-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/2_mnl/mnl_files/figure-html/unnamed-chunk-30-1.png -------------------------------------------------------------------------------- /2_mnl/mnl_files/figure-html/unnamed-chunk-31-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/2_mnl/mnl_files/figure-html/unnamed-chunk-31-1.png -------------------------------------------------------------------------------- /2_mnl/mnl_files/figure-html/unnamed-chunk-34-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/2_mnl/mnl_files/figure-html/unnamed-chunk-34-1.png -------------------------------------------------------------------------------- /2_mnl/mnl_files/figure-html/unnamed-chunk-35-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/2_mnl/mnl_files/figure-html/unnamed-chunk-35-1.png -------------------------------------------------------------------------------- /2_mnl/mnl_files/figure-html/unnamed-chunk-36-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/2_mnl/mnl_files/figure-html/unnamed-chunk-36-1.png -------------------------------------------------------------------------------- /2_mnl/mnl_files/figure-html/unnamed-chunk-48-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/2_mnl/mnl_files/figure-html/unnamed-chunk-48-1.png -------------------------------------------------------------------------------- /2_mnl/mnl_files/figure-html/unnamed-chunk-49-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/2_mnl/mnl_files/figure-html/unnamed-chunk-49-1.png -------------------------------------------------------------------------------- /3_hmnl/3_hmnl.Rproj: -------------------------------------------------------------------------------- 1 | Version: 1.0 2 | 3 | RestoreWorkspace: Default 4 | SaveWorkspace: Default 5 | AlwaysSaveHistory: Default 6 | 7 | EnableCodeIndexing: Yes 8 | UseSpacesForTab: Yes 9 | NumSpacesForTab: 2 10 | Encoding: UTF-8 11 | 12 | RnwWeave: Sweave 13 | LaTeX: pdfLaTeX 14 | -------------------------------------------------------------------------------- /3_hmnl/hmnl.R: -------------------------------------------------------------------------------- 1 | # Code for estimating multinomial logit models in Stan 2 | # Elea McDonnell Feit, eleafeit@gmail.com 3 | # Last updated: 22 June 2017 4 | 5 | # Copyright 2017, Kevin Van Horn & Elea McDonnell Feit 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # 10 | # You may obtain a copy of the License at 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | # ========== Load libraries and set options ========== 20 | library(rstan) 21 | library(MASS) 22 | library(shinystan) 23 | 24 | # writes a compiled Stan program to the disk to avoid recompiling 25 | rstan_options(auto_write=TRUE) 26 | # allows Stan chains to run in parallel on multiprocessor machines 27 | options(mc.cores = parallel::detectCores()) 28 | 29 | # ========== Test Stan with synthetic data ============ 30 | 31 | # function to generate mnl data 32 | generate_hmnl_data <- function(R=100, S=30, C=3, 33 | Theta=matrix(rep(1, 8), nrow=2), 34 | Sigma=diag(0.1, 4)){ 35 | K <- ncol(Theta) 36 | G <- nrow(Theta) 37 | Y <- array(dim=c(R, S)) 38 | X <- array(rnorm(R*S*C*K), dim=c(R, S, C, K)) # normal covariates 39 | Z <- array(dim=c(G, R)) 40 | Z[1,] <- 1 # intercept 41 | if (G > 1) { 42 | Z[2:G,] <- rnorm(R*(G-1)) # normal covariates 43 | } 44 | Beta <- array(dim=c(K, R)) 45 | for (r in 1:R) { 46 | Beta[,r] <- mvrnorm(n=1, mu=Z[,r]%*%Theta, Sigma=Sigma) 47 | for (s in 1:S) 48 | Y[r,s] <- sample(x=C, size=1, prob=exp(X[r,s,,]%*%Beta[,r])) # logit formula 49 | } 50 | list(R=R, S=S, C=C, K=K, G=G, Y=Y, X=X, Z=Z, 51 | beta.true=beta, Theta.true=Theta, Sigma.true=Sigma) 52 | } 53 | 54 | d1 <- generate_hmnl_data() 55 | str(d1) 56 | 57 | test.stan <- stan(file="hmnl.stan", data=d1, iter=1000, chains=4) 58 | 59 | plot(test.stan, plotfun="trace", pars=("Theta")) 60 | 61 | plot(test.stan, plotfun="trace", pars=c("tau")) 62 | 63 | plot(test.stan, plotfun="trace", pars=("Omega")) 64 | 65 | plot(test.stan, pars=c("Theta", "tau", "Omega")) 66 | 67 | plot(test.stan, pars=c("Theta", "Sigma")) 68 | 69 | summary(test.stan, pars=c("Theta"))$summary 70 | 71 | summary(test.stan, pars=c("tau"))$summary 72 | 73 | summary(test.stan, pars=c("Omega"))$summary 74 | 75 | # ========= Read in Chocolate Data and Prep for Stan ========== 76 | rm(list=ls()) # tidy up 77 | choc.df <- read.csv("cbc_chocolate.csv") 78 | 79 | # Coding the chocolate data (this ought to be a function) 80 | choc.contrasts <- list(Brand = "contr.sum", Type = "contr.sum") 81 | choc.coded <- model.matrix(~ Brand + Type, data = choc.df, contrasts = choc.contrasts) 82 | choc.coded <- choc.coded[,2:ncol(choc.coded)] # remove intercept 83 | # Fix the bad labels from contr.sum 84 | choc.names <- c("BrandDove", "BrandGhirardelli", "BrandGodiva", "BrandHersheys", 85 | "TypeDark", "TypeDarkNuts", "TypeMilk", "TypeMilkNuts") 86 | colnames(choc.coded) <- choc.names 87 | choc.df <- cbind(choc.df, choc.coded) 88 | 89 | head(choc.df) 90 | 91 | # Munge into Stan list 92 | R <- length(unique(choc.df$Ind)) 93 | S <- length(unique(choc.df$Trial)) 94 | C <- max(choc.df$Alt) 95 | K <- 9 96 | Y <- array(dim=c(R, S)) 97 | X <- array(rnorm(R*S*C*K), dim=c(R, S, C, K)) 98 | Z <- array(1, dim=c(1, R)) # intercept only 99 | for (r in 1:R) { # respondents 100 | for (s in 1:S){ # choice scenarios 101 | scenario <- choc.df[choc.df$Ind==unique(choc.df$Ind)[r] & 102 | choc.df$Trial==unique(choc.df$Trial)[s], ] 103 | X[r,s,,] <- data.matrix(scenario[,c(7, 9:16)]) # price and coded brand and type 104 | Y[r,s] <- scenario$Alt[as.logical(scenario$Chosen)] 105 | } 106 | } 107 | 108 | choc.standata <- list(C=C, K=K, R=R, S=S, G=1, Y=Y, X=X, Z=Z) 109 | str(choc.standata) 110 | 111 | rm(Y, X, Z, R, S, r, s, C, K, choc.contrasts, scenario, choc.coded) 112 | 113 | # ========== Run Stan and Check Convergence ========= 114 | choc.stan <- stan(file="hmnl.stan", data=choc.standata) 115 | 116 | plot(choc.stan, plotfun="trace", pars=("Theta")) 117 | 118 | plot(choc.stan, plotfun="trace", pars=c("tau")) 119 | 120 | plot(choc.stan, plotfun="trace", pars=c("Omega[1,2]")) 121 | 122 | plot(choc.stan, plotfun="trace", pars=paste("Beta[", 1:9, ",1]", sep="")) # resp 1 123 | 124 | summary(choc.stan)$summary[,c("Rhat", "n_eff")] 125 | 126 | # Convergence checking gets tedious with a lot of parameters, so automate 127 | check_fit <- function(fit) { 128 | summ <- summary(fit)$summary 129 | range_rhat <- range(summ[ , 'Rhat']) 130 | rhat_ok <- 0.99 <= range_rhat[1] && range_rhat[2] <= 1.1 131 | range_neff <- range(summ[ , 'n_eff']) 132 | neff_ok <- range_neff[1] >= 400 133 | sp <- rstan::get_sampler_params(fit, inc_warmup=FALSE) 134 | max_divergent <- max(sapply(sp, function(p){ sum(p[ , 'divergent__']) })) 135 | no_divergent <- max_divergent == 0 136 | 137 | list(ok = rhat_ok && neff_ok && no_divergent, 138 | range_rhat = range_rhat, 139 | range_neff = range_neff, 140 | max_divergent = max_divergent) 141 | } 142 | 143 | check_fit(choc.stan) 144 | 145 | # ========== Summarize Posterior ========== 146 | choc.names 147 | plot(choc.stan, pars=c("Theta", "tau")) 148 | 149 | plot(choc.stan, pars=c("Omega")) 150 | 151 | plot(choc.stan, pars=paste("Beta[", 1:9, ",1]", sep="")) + ggtitle("Respondent 1: Likes Milk Chocolate") 152 | 153 | plot(choc.stan, pars=paste("Beta[", 1:9, ",2]", sep="")) + ggtitle("Respondent 2: Likes Dark Chocolate") 154 | 155 | launch_shinystan(choc.stan) 156 | 157 | # ========== Simulate Shares from Model ========= 158 | # function for computing shares from beta.draws for an hmnl model 159 | shares.hmnl.post <- function(beta.draws, X) # X is attribute matrix for scenario 160 | { 161 | R <- dim(beta.draws)[3] # respondents 162 | D <- dim(beta.draws)[1] # draws 163 | shares <- array(NA, dim=c(nrow(X), R, D)) 164 | for (d in 1:D) { 165 | for (r in 1:R) { 166 | beta <- beta.draws[d,,r] 167 | V <- exp(X %*% beta) 168 | shares[,r,d] <- V/sum(V) 169 | } 170 | } 171 | shares 172 | } 173 | 174 | choc.standata$X[1,1,,] # note option 2 is dark 175 | shares <- shares.hmnl.post(extract(choc.stan, pars=c("Beta"))$Beta, 176 | choc.standata$X[1,1,,]) 177 | str(shares) 178 | 179 | apply(shares, 1, quantile, probs=c(0.5, 0.025, 0.975)) 180 | apply(shares, 1:2, quantile, probs=c(0.5, 0.025, 0.975)) 181 | 182 | -------------------------------------------------------------------------------- /3_hmnl/hmnl.Rmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Stan Tutorial" 3 | subtitle: "Module 3: Hierarchical Multinomial Logit Model" 4 | author: "Kevin Van Horn and Elea McDonnell Feit" 5 | date: "June 25, 2017" 6 | output: ioslides_presentation 7 | --- 8 | 9 | ```{r setup, include=FALSE} 10 | knitr::opts_chunk$set(echo = TRUE, eval=TRUE) 11 | ``` 12 | 13 | ## Module objective 14 | In this module, we will fit a hierarchical multinomial logit model using choice based conjoint data for chocolate bars. 15 | 16 | This will allow us to: 17 | - Learn more Stan syntax 18 | - Work with a hierarchical model 19 | - Fit a model that is commonly fit in marketing and usually requres specialized software 20 | 21 | Many will be familar with estimating this model using Sawtooth CBC/HB or the ChoiceModelR package in R. 22 | 23 | # More Stan syntax 24 | 25 | ## Hierarchical models 26 | There is substantial evidence that different people have different preferences over the features of products. To accomodate this, we alter the multnomial logit model by allowing each person to have her own vector of part-worths for the attributes. 27 | ```{stan, eval=FALSE, echo=TRUE, output.var="dummy"} 28 | Y[r,s] ~ categorical_logit(X[r,s]*Beta[,r]); 29 | ``` 30 | Becuase there is typically insufficient data to estimate each person's `beta[,r]` vector, we then impose a distribution on the individual-level betas. 31 | ```{stan, eval=FALSE, echo=TRUE, output.var="dummy"} 32 | Beta[,r] ~ multi_normal(Theta*Z[,r], Sigma); 33 | ``` 34 | 35 | ## Multivariate normal distribution 36 | This is the first time we have seen a *multivariate* distribution in Stan. This is the *multivariate normal distribution.* 37 | ```{stan, eval=FALSE, echo=TRUE, output.var="dummy"} 38 | y ~ multi_normal(mu, Sigma); 39 | ``` 40 | `y` is a vector (or an array of vectors for multiple observations) 41 | `mu` is the mean vector 42 | `Sigma` is the covariance matrix. 43 | 44 | If this distribution is new to you, check out the [wikipedia page](https://en.wikipedia.org/wiki/Multivariate_normal_distribution). 45 | 46 | ## Covariance matricies 47 | Covariance matrices like `Sigma` are square matrices with special properties. Stan provides special data types just for storing covariance matrices: 48 | ```{stan, eval=FALSE, echo=TRUE, output.var="dummy"} 49 | cov_matrix[K] mycovmatrix; 50 | ``` 51 | And there is a correlation matrix type as well: 52 | ```{stan, stan, eval=FALSE, echo=TRUE, output.var="dummy"} 53 | corr_matrix[K] mycorrmatrix; 54 | ``` 55 | 56 | 57 | ## Other parameterizations 58 | If you prefer other parameterizations of the multivariate normal, they are available. 59 | ```{stan, eval=FALSE, echo=TRUE, output.var="dummy"} 60 | y ~ multi_normal_prec(mu, Omega); 61 | y ~ multi_normal_cholesky(mu, L); 62 | ``` 63 | There are even special types for Cholesky factors, if you are into that. (It can be important for computational efficiency.) 64 | ```{stan, eval=FALSE, echo=TRUE, output.var="dummy"} 65 | cholesky_factor_cov[K] L; 66 | cholesky_factor_cor[K] L; 67 | ``` 68 | 69 | ## Other multivariate distributions 70 | Gaussian process 71 | ```{stan, eval=FALSE, echo=TRUE, output.var="dummy"} 72 | y ~ multi_gp(Sigma, w); 73 | ``` 74 | Multivariate Student t 75 | ```{stan, eval=FALSE, echo=TRUE, output.var="dummy"} 76 | y ~ multi_student_t(nu, mu, Sigma); 77 | ``` 78 | Gaussian DLM 79 | ```{stan, eval=FALSE, echo=TRUE, output.var="dummy"} 80 | y ~ gaussian_dlm_obs(F, G, V, W, m0, C0); 81 | ``` 82 | See the the [Stan Modeling Language Users Guide and Reference Manual](http://mc-stan.org/users/documentation/) for more! 83 | 84 | ## Priors for covariances 85 | One of the difficult things about using the multivariate normal distribution is putting a prior on the covariance `Sigma.` Gibbs samplers often use the *inverted Wishart* prior, which is conjugate. And Stan supports that: 86 | ```{stan, eval=FALSE, echo=TRUE, output.var="dummy"} 87 | W ~ inv_wishart(nu, Sigma); 88 | ``` 89 | However, this prior often puts undue weight near `Sigma` and can influence the posterior in strange ways. 90 | 91 | ## LKJ Prior for correlations 92 | Many analysts have an intuitive sense for correlations; they are always between -1 and 1 with zero indicating no association. The LKJ distribution puts a prior on the correlation matrix. 93 | ```{stan, eval=FALSE, echo=TRUE, output.var="dummy"} 94 | Omega ~ lkj_corr(eta); 95 | ``` 96 | If `eta = 1` then the density is uniform over all correlation matrices of a given order 97 | If `eta > 1` the identity matrix is the modal correlation matrix, with sharper peaks in the density around the identity matrix for larger `eta`. This allows us to reduce correlations, if we have prior beliefs that there are not many correlations between part-worths. 98 | 99 | ## Visualizing the LKJ prior 100 | ![](images/lkj_prior.png) 101 | 102 | This image was borrowed from the [Psychstatistics blog](http://www.psychstatistics.com/2014/12/27/d-lkj-priors/), which illustrates how to use Stan to simulate from the LKJ prior to produce this visualization. 103 | 104 | ## Using LKJ to put a prior on covariance 105 | We can put a prior on a covarince by specifing a variance vector `tau` along with a correlation matrix `Omega` (with the LKJ prior) and then use the `quad_form_diag` function to compute a covariance. 106 | ```{stan, eval=FALSE, echo=TRUE, output.var="dummy"} 107 | parameters{ 108 | corr_matrix[K] Omega; 109 | vector[K] tau; 110 | } 111 | model{ 112 | Omega ~ lkj_corr(2); 113 | Beta[,r] ~ multi_normal(mu, quad_form_diag(Omega, tau)); 114 | } 115 | ``` 116 | 117 | ## Truncated distributions 118 | You can truncate any distribution using this syntax: 119 | ```{stan, eval=FALSE, echo=TRUE, output.var="dummy"} 120 | mu ~ normal(mu, sigma) T[lower, upper]; 121 | ``` 122 | 123 | ## Assignment 124 | Occasionally, you may need to assign one variable (data or parameter) to another, e.g. when you are tranforming data internally in Stan. To do that we use `=`. 125 | 126 | The symbol `~` is for sampling statements that define random variables. 127 | 128 | We will use truncation and assignment in just a few more slides. 129 | 130 | # Hierarchical multnomial logit model in Stan 131 | Enough syntax! Let's get on to the model. You can open the full model specification in `hmnl.stan`. 132 | 133 | ## Data 134 | Putting all of that new syntax together, we can specify a hierarchical multinomial logit model. 135 | ```{stan, eval=FALSE, echo=TRUE, output.var="dummy"} 136 | data { 137 | int C; // # of alternatives (choices) in each scenario 138 | int K; // # of covariates of alternatives 139 | int R; // # of respondents 140 | int S; // # of scenarios per respondent 141 | int G; // # of respondent covariates 142 | int Y[R, S]; // observed choices 143 | matrix[C, K] X[R, S]; // matrix of attributes for each obs 144 | matrix[G, R] Z; // vector of covariates for each respondent 145 | } 146 | ``` 147 | 148 | ## Choice sets of varying size 149 | You may have noticed that we have defined `X` to be a fixed size for each choice question. 150 | ```{stan, eval=FALSE, echo=TRUE, output.var="dummy"} 151 | matrix[C, K] X[R, S]; // matrix of attributes for each obs 152 | ``` 153 | It is possible to write a Stan model that will accomodate a different number of alternatives in each scenario. Look for information on ragged arrays in Stan. 154 | 155 | ## Parameters 156 | ```{stan, eval=FALSE, echo=TRUE, output.var="dummy"} 157 | 158 | parameters { 159 | matrix[K, R] Beta; 160 | matrix[K, G] Theta; 161 | corr_matrix[K] Omega; 162 | vector[K] tau; 163 | } 164 | ``` 165 | 166 | ## Transformed parameters 167 | While it can be nice to look at the output in terms of the variance vector `tau` and the correlation matrix `Omega`, some still like covariances. We can compute the covariance using the `transformed parameters` block in Stan. 168 | ```{stan, eval=FALSE, echo=TRUE, output.var="dummy"} 169 | transformed parameters { 170 | cov_matrix[K] Sigma = quad_form_diag(Omega, tau); 171 | } 172 | ``` 173 | If you aren't using the tranformed parameter in the `model` block, it is more computationally efficient to put it in the `generated quantities` block. 174 | 175 | ## Model 176 | We can use any transformed parameters in our model block. 177 | ```{stan, eval=FALSE, echo=TRUE, output.var="dummy"} 178 | model { 179 | //priors 180 | to_vector(Theta) ~ normal(0, 10); 181 | tau ~ cauchy(0, 2.5); 182 | Omega ~ lkj_corr(2); 183 | //likelihood 184 | for (r in 1:R) { 185 | Beta[,r] ~ multi_normal(Theta*Z[,r], Sigma); 186 | for (s in 1:S) 187 | Y[r,s] ~ categorical_logit(X[r,s]*Beta[,r]); 188 | } 189 | } 190 | ``` 191 | 192 | ## Bayesian inference 193 | ```{r, fig.align="center", echo=FALSE} 194 | knitr::include_graphics("images/Bayesian_Inference.png") 195 | ``` 196 | 197 | ## Get setup in R 198 | It's time to open the `hmnl.R` file and start running R code. 199 | ```{r, message=FALSE, error=FALSE, warning=FALSE} 200 | library(rstan) 201 | library(MASS) 202 | library(shinystan) 203 | # writes a compiled Stan program to the disk to avoid recompiling 204 | rstan_options(auto_write=TRUE) 205 | # allows Stan chains to run in parallel on multiprocessor machines 206 | options(mc.cores = parallel::detectCores()) 207 | ``` 208 | 209 | # Testing the hierarchical multinomial logit model with synthetic data 210 | It's a good idea to test any new model with synthetic data. 211 | 212 | ## Synthetic data generation function (1) 213 | ```{r} 214 | # function to generate mnl data 215 | generate_hmnl_data <- function(R=100, S=30, C=3, 216 | Theta=matrix(rep(1, 8), nrow=2), 217 | Sigma=diag(0.1, 4)){ 218 | K <- ncol(Theta) 219 | G <- nrow(Theta) 220 | Y <- array(dim=c(R, S)) 221 | X <- array(rnorm(R*S*C*K), dim=c(R, S, C, K)) # normal covariates 222 | Z <- array(dim=c(G, R)) 223 | Z[1,] <- 1 # intercept 224 | if (G > 1) { 225 | Z[2:G,] <- rnorm(R*(G-1)) # normal covariates 226 | } 227 | Beta <- array(dim=c(K, R)) 228 | for (r in 1:R) { 229 | Beta[,r] <- mvrnorm(n=1, mu=Z[,r]%*%Theta, Sigma=Sigma) 230 | for (s in 1:S) 231 | Y[r,s] <- sample(x=C, size=1, prob=exp(X[r,s,,]%*%Beta[,r])) 232 | } 233 | list(R=R, S=S, C=C, K=K, G=G, Y=Y, X=X, Z=Z, 234 | beta.true=beta, Theta.true=Theta, Sigma.true=Sigma) 235 | } 236 | ``` 237 | 238 | ## Synthetic data generation function (2) 239 | ```{r, eval=FALSE} 240 | for (r in 1:R) { 241 | Beta[,r] <- mvrnorm(n=1, mu=Z[,r]%*%Theta, Sigma=Sigma) 242 | for (s in 1:S) 243 | Y[r,s] <- sample(x=C, size=1, prob=exp(X[r,s,,]%*%Beta[,r])) 244 | } 245 | list(R=R, S=S, C=C, K=K, G=G, Y=Y, X=X, Z=Z, 246 | beta.true=beta, Theta.true=Theta, Sigma.true=Sigma) 247 | } 248 | ``` 249 | 250 | ## Synthetic data 251 | ```{r} 252 | d1 <- generate_hmnl_data() 253 | str(d1) 254 | ``` 255 | 256 | ## Running the hmnl on synthetic data 257 | ```{r, message=FALSE, warning=FALSE, error=FALSE, cache=TRUE} 258 | test.stan <- stan(file="hmnl.stan", data=d1, iter=1000, chains=4) 259 | ``` 260 | 261 | ## Traceplots: Theta 262 | ```{r} 263 | plot(test.stan, plotfun="trace", pars=("Theta")) 264 | ``` 265 | 266 | ## Traceplots: tau 267 | ```{r} 268 | plot(test.stan, plotfun="trace", pars=c("tau")) 269 | ``` 270 | 271 | ## Traceplots: Omega 272 | ```{r} 273 | plot(test.stan, plotfun="trace", pars=("Omega")) 274 | ``` 275 | 276 | ## Summary: Theta (Population means of part-worths) 277 | ```{r} 278 | summary(test.stan, pars=c("Theta"))$summary 279 | ``` 280 | 281 | ## Summary: tau (Population variances of part-worths) 282 | ```{r} 283 | summary(test.stan, pars=c("tau"))$summary 284 | ``` 285 | 286 | ## Summary: Omega (Populaltion correlations of part-worths) 287 | ```{r} 288 | summary(test.stan, pars=c("Omega"))$summary 289 | ``` 290 | 291 | ## Visualize parameters (1) 292 | ```{r, message=FALSE} 293 | plot(test.stan, pars=c("Theta", "tau", "Omega")) 294 | ``` 295 | 296 | ## Visualize parameters (2) 297 | ```{r, message=FALSE} 298 | plot(test.stan, pars=c("Theta", "Sigma")) 299 | ``` 300 | 301 | # Prepare the chocolate cbc data 302 | 303 | ## Choice-based conjoint data for chocolate bars 304 | - Fourteen respondents each answered 25 choice tasks where they selected from among three chocolate bars. 305 | - Three attributes 306 | + Brand: Hersheys, Dove, Lindt, Godiva, Ghirardelli 307 | + Type: Milk, Milk with nuts, Dark, Dark with nuts, White 308 | 309 | ## Tidy up and read in the chocolate data 310 | ```{r} 311 | rm(list=ls()) # tidy up 312 | choc.df <- read.csv("cbc_chocolate.csv") 313 | ``` 314 | 315 | ## Coding the chocolate data 316 | We've done this twice now, so we ought to package it up as a function (but we haven't). 317 | ```{r} 318 | choc.contrasts <- list(Brand = "contr.sum", Type = "contr.sum") 319 | choc.coded <- model.matrix(~ Brand + Type, data = choc.df, 320 | contrasts = choc.contrasts) 321 | choc.coded <- choc.coded[,2:ncol(choc.coded)] # remove intercept 322 | # Fix the bad labels from contr.sum 323 | choc.names <- c("BrandDove", "BrandGhirardelli", "BrandGodiva", 324 | "BrandHersheys", "TypeDark", "TypeDarkNuts", 325 | "TypeMilk", "TypeMilkNuts") 326 | colnames(choc.coded) <- choc.names 327 | choc.df <- cbind(choc.df, choc.coded) 328 | ``` 329 | 330 | ## Coded chocolate data 331 | ```{r} 332 | head(choc.df) 333 | ``` 334 | 335 | ## Munge the data into Stan list format 336 | ```{r} 337 | R <- length(unique(choc.df$Ind)) 338 | S <- length(unique(choc.df$Trial)) 339 | C <- max(choc.df$Alt) 340 | K <- 9 341 | Y <- array(dim=c(R, S)) 342 | X <- array(rnorm(R*S*C*K), dim=c(R, S, C, K)) 343 | Z <- array(1, dim=c(1, R)) # intercept only 344 | for (r in 1:R) { # respondents 345 | for (s in 1:S){ # choice scenarios 346 | scenario <- choc.df[choc.df$Ind==unique(choc.df$Ind)[r] & 347 | choc.df$Trial==unique(choc.df$Trial)[s], ] 348 | X[r,s,,] <- data.matrix(scenario[,c(7, 9:16)]) 349 | Y[r,s] <- scenario$Alt[as.logical(scenario$Chosen)] 350 | } 351 | } 352 | choc.standata <- list(C=C, K=K, R=R, S=S, G=1, Y=Y, X=X, Z=Z) 353 | ``` 354 | 355 | ## Check the chocolate data 356 | ```{R} 357 | str(choc.standata) 358 | ``` 359 | 360 | ## Tidy up 361 | ```{r} 362 | rm(Y, X, Z, R, S, r, s, C, K, choc.contrasts, scenario, choc.coded) 363 | ``` 364 | 365 | # Run the hmnl model with the chocolate data 366 | 367 | ## Bayesian inference 368 | ```{r, fig.align="center", echo=FALSE} 369 | knitr::include_graphics("images/Bayesian_Inference.png") 370 | ``` 371 | 372 | ## Call the `stan()` function 373 | ```{r, cache=TRUE} 374 | choc.stan <- stan(file="hmnl.stan", data=choc.standata) 375 | ``` 376 | 377 | ## Assess convergence with tracelots (1) 378 | ```{r} 379 | plot(choc.stan, plotfun="trace", pars=("Theta")) 380 | ``` 381 | 382 | ## Assess convergence with tracelots (1) 383 | ```{r} 384 | plot(choc.stan, plotfun="trace", pars=c("tau")) 385 | ``` 386 | 387 | ## Assess convergence with tracelots (1) 388 | ```{r} 389 | plot(choc.stan, plotfun="trace", pars=c("Omega[1,2]")) 390 | ``` 391 | 392 | ## Assess convergence with tracelots (1) 393 | ```{r} 394 | plot(choc.stan, plotfun="trace", pars=paste("Beta[", 1:9, ",1]", sep="")) # resp 1 395 | ``` 396 | 397 | ## Assess convergence with `Rhat` and `n_eff` 398 | ```{r} 399 | summary(choc.stan)$summary[,c("Rhat", "n_eff")] 400 | ``` 401 | 402 | ## Function for convergence checking 403 | In a hierarchical model with many parameters, manually checking for convergence is pretty tedious. So, we can write a function to automate that for us. 404 | ```{r} 405 | check_fit <- function(fit) { 406 | summ <- summary(fit)$summary 407 | range_rhat <- range(summ[ , 'Rhat']) 408 | rhat_ok <- 0.99 <= range_rhat[1] && range_rhat[2] <= 1.1 409 | range_neff <- range(summ[ , 'n_eff']) 410 | neff_ok <- range_neff[1] >= 400 411 | sp <- rstan::get_sampler_params(fit, inc_warmup=FALSE) 412 | max_divergent <- max(sapply(sp, function(p){ sum(p[ , 'divergent__']) })) 413 | no_divergent <- max_divergent == 0 414 | 415 | list(ok = rhat_ok && neff_ok && no_divergent, 416 | range_rhat = range_rhat, 417 | range_neff = range_neff, 418 | max_divergent = max_divergent) 419 | } 420 | ``` 421 | 422 | ## Checking convergence 423 | Our model isn't converging well :( 424 | This is likely due to data limitations; we have just 14 respondents and 9 attributes. 425 | ```{r} 426 | check_fit(choc.stan) 427 | ``` 428 | 429 | ## Options for addressing convergence problems 430 | - Consider whether there are any pathologies in the model specification 431 | + unlikely in this case 432 | - Run the algorithm for longer 433 | - Get more data 434 | - Tighten up the priors 435 | 436 | We are going to move on to use these posterior draws although we should probably not. 437 | 438 | # Summarizing the posterior distribution of the parameters 439 | 440 | ## Attributes 441 | Remember, the attributes are: 442 | ```{r} 443 | c("Price", choc.names) 444 | ``` 445 | 446 | ## Visualize the parameter estimates (1) 447 | ```{r, message=FALSE} 448 | plot(choc.stan, pars=c("Theta", "tau")) 449 | ``` 450 | 451 | It is a shame that you can't give descriptive names to the attributes in Stan. 452 | 453 | ## Visualize the parameter estimates (2) 454 | ```{r, message=FALSE} 455 | plot(choc.stan, pars=c("Omega")) 456 | ``` 457 | 458 | ## Visualize the parameter estimates (3) 459 | ```{r, message=FALSE} 460 | plot(choc.stan, pars=paste("Beta[", 1:9, ",1]", sep="")) + 461 | ggtitle("Respondent 1: Likes Milk Chocolate") 462 | ``` 463 | 464 | ## Visualize the parameter estimates (4) 465 | ```{r, message=FALSE} 466 | plot(choc.stan, pars=paste("Beta[", 1:9, ",2]", sep="")) + 467 | ggtitle("Respondent 2: Likes Dark Chocolate") 468 | ``` 469 | 470 | ## More posterior checking in ShinyStan 471 | ```{r, eval=FALSE} 472 | launch_shinystan(choc.stan) 473 | ``` 474 | 475 | # Simulating shares 476 | 477 | ## Simulating shares from the hmnl 478 | One way to simulate shares from a hierarchical model is to use the draws of the respondent-level parameters (`beta`). 479 | ```{r} 480 | shares.hmnl.post <- function(beta.draws, X) # X is attribute matrix 481 | { 482 | R <- dim(beta.draws)[3] # respondents 483 | D <- dim(beta.draws)[1] # draws 484 | shares <- array(NA, dim=c(nrow(X), R, D)) 485 | for (d in 1:D) { 486 | for (r in 1:R) { 487 | beta <- beta.draws[d,,r] 488 | V <- exp(X %*% beta) 489 | shares[,r,d] <- V/sum(V) 490 | } 491 | } 492 | shares 493 | } 494 | ``` 495 | 496 | 497 | ## New choice set 498 | ```{r} 499 | choc.standata$X[1,1,,] 500 | rbind(c("Price", choc.names), choc.standata$X[1,1,,]) 501 | ``` 502 | 503 | ## Simulated shares for each respondent 504 | ```{r} 505 | shares <- shares.hmnl.post(extract(choc.stan, pars=c("Beta"))$Beta, 506 | choc.standata$X[1,1,,]) 507 | str(shares) 508 | ``` 509 | 510 | ## Summary of simulated shares 511 | Our uncertainty (with only 14 respondents) is reflected in our share predictions 512 | ```{r} 513 | apply(shares, 1, quantile, probs=c(0.5, 0.025, 0.975)) 514 | ``` 515 | 516 | ## Share predictions for indivdiuals 517 | ```{r} 518 | apply(shares, 1:2, quantile, probs=c(0.5, 0.025, 0.975)) 519 | ``` 520 | 521 | # Summary of Module 3 522 | 523 | ## In this module we have: 524 | - Built a hierarchical multnomial model in Stan. 525 | + multivariate normal distribution 526 | + correlation and covariance matrices 527 | + LKJ prior 528 | + transformed parameters 529 | - Tested it with synthetic data 530 | - Estimated it using the chocolate cbc data 531 | - Learned how to automate the process of checking convergence 532 | - Simulated shares from a hierarchical model 533 | 534 | 535 | ## Next in Module 4 536 | The next module will illustrate the real power of Stan, which is to flexibly create new models. 537 | 538 | -------------------------------------------------------------------------------- /3_hmnl/hmnl.rds: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/3_hmnl/hmnl.rds -------------------------------------------------------------------------------- /3_hmnl/hmnl.stan: -------------------------------------------------------------------------------- 1 | # Hierarchical multinomial logit model 2 | # Elea McDonnell Feit, eleafeit@gmail.com 3 | 4 | data { 5 | int C; // # of alternatives (choices) in each scenario 6 | int K; // # of covariates of alternatives 7 | int R; // # of respondents 8 | int S; // # of scenarios per respondent 9 | int G; // # of respondent covariates 10 | int Y[R, S]; // observed choices 11 | matrix[C, K] X[R, S]; // matrix of attributes for each obs 12 | matrix[G, R] Z; // vector of covariates for each respondent 13 | } 14 | 15 | parameters { 16 | matrix[K, R] Beta; 17 | matrix[K, G] Theta; 18 | corr_matrix[K] Omega; 19 | vector[K] tau; 20 | } 21 | transformed parameters { 22 | cov_matrix[K] Sigma = quad_form_diag(Omega, tau); 23 | } 24 | model { 25 | //priors 26 | to_vector(Theta) ~ normal(0, 10); 27 | tau ~ cauchy(0, 2.5); 28 | Omega ~ lkj_corr(2); 29 | //likelihood 30 | for (r in 1:R) { 31 | Beta[,r] ~ multi_normal(Theta*Z[,r], Sigma); 32 | for (s in 1:S) 33 | Y[r,s] ~ categorical_logit(X[r,s]*Beta[,r]); 34 | } 35 | } 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /3_hmnl/hmnl_cache/html/__packages: -------------------------------------------------------------------------------- 1 | base 2 | ggplot2 3 | StanHeaders 4 | rstan 5 | MASS 6 | shiny 7 | shinystan 8 | -------------------------------------------------------------------------------- /3_hmnl/hmnl_cache/html/unnamed-chunk-23_085fd1961339d2a88bff9caa4082a229.RData: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/3_hmnl/hmnl_cache/html/unnamed-chunk-23_085fd1961339d2a88bff9caa4082a229.RData -------------------------------------------------------------------------------- /3_hmnl/hmnl_cache/html/unnamed-chunk-23_085fd1961339d2a88bff9caa4082a229.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/3_hmnl/hmnl_cache/html/unnamed-chunk-23_085fd1961339d2a88bff9caa4082a229.rdb -------------------------------------------------------------------------------- /3_hmnl/hmnl_cache/html/unnamed-chunk-23_085fd1961339d2a88bff9caa4082a229.rdx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/3_hmnl/hmnl_cache/html/unnamed-chunk-23_085fd1961339d2a88bff9caa4082a229.rdx -------------------------------------------------------------------------------- /3_hmnl/hmnl_cache/html/unnamed-chunk-24_8019a6f96fd873f7f4c80480657201b3.RData: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/3_hmnl/hmnl_cache/html/unnamed-chunk-24_8019a6f96fd873f7f4c80480657201b3.RData -------------------------------------------------------------------------------- /3_hmnl/hmnl_cache/html/unnamed-chunk-24_8019a6f96fd873f7f4c80480657201b3.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/3_hmnl/hmnl_cache/html/unnamed-chunk-24_8019a6f96fd873f7f4c80480657201b3.rdb -------------------------------------------------------------------------------- /3_hmnl/hmnl_cache/html/unnamed-chunk-24_8019a6f96fd873f7f4c80480657201b3.rdx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/3_hmnl/hmnl_cache/html/unnamed-chunk-24_8019a6f96fd873f7f4c80480657201b3.rdx -------------------------------------------------------------------------------- /3_hmnl/hmnl_cache/html/unnamed-chunk-39_f87fecbb9859a5a7d3d8e77c67577ceb.RData: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/3_hmnl/hmnl_cache/html/unnamed-chunk-39_f87fecbb9859a5a7d3d8e77c67577ceb.RData -------------------------------------------------------------------------------- /3_hmnl/hmnl_cache/html/unnamed-chunk-39_f87fecbb9859a5a7d3d8e77c67577ceb.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/3_hmnl/hmnl_cache/html/unnamed-chunk-39_f87fecbb9859a5a7d3d8e77c67577ceb.rdb -------------------------------------------------------------------------------- /3_hmnl/hmnl_cache/html/unnamed-chunk-39_f87fecbb9859a5a7d3d8e77c67577ceb.rdx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/3_hmnl/hmnl_cache/html/unnamed-chunk-39_f87fecbb9859a5a7d3d8e77c67577ceb.rdx -------------------------------------------------------------------------------- /3_hmnl/hmnl_cache/html/unnamed-chunk-40_83577461465d937bd855b7fb3dbffd7d.RData: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/3_hmnl/hmnl_cache/html/unnamed-chunk-40_83577461465d937bd855b7fb3dbffd7d.RData -------------------------------------------------------------------------------- /3_hmnl/hmnl_cache/html/unnamed-chunk-40_83577461465d937bd855b7fb3dbffd7d.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/3_hmnl/hmnl_cache/html/unnamed-chunk-40_83577461465d937bd855b7fb3dbffd7d.rdb -------------------------------------------------------------------------------- /3_hmnl/hmnl_cache/html/unnamed-chunk-40_83577461465d937bd855b7fb3dbffd7d.rdx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/3_hmnl/hmnl_cache/html/unnamed-chunk-40_83577461465d937bd855b7fb3dbffd7d.rdx -------------------------------------------------------------------------------- /3_hmnl/hmnl_files/figure-html/unnamed-chunk-24-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/3_hmnl/hmnl_files/figure-html/unnamed-chunk-24-1.png -------------------------------------------------------------------------------- /3_hmnl/hmnl_files/figure-html/unnamed-chunk-25-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/3_hmnl/hmnl_files/figure-html/unnamed-chunk-25-1.png -------------------------------------------------------------------------------- /3_hmnl/hmnl_files/figure-html/unnamed-chunk-26-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/3_hmnl/hmnl_files/figure-html/unnamed-chunk-26-1.png -------------------------------------------------------------------------------- /3_hmnl/hmnl_files/figure-html/unnamed-chunk-27-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/3_hmnl/hmnl_files/figure-html/unnamed-chunk-27-1.png -------------------------------------------------------------------------------- /3_hmnl/hmnl_files/figure-html/unnamed-chunk-30-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/3_hmnl/hmnl_files/figure-html/unnamed-chunk-30-1.png -------------------------------------------------------------------------------- /3_hmnl/hmnl_files/figure-html/unnamed-chunk-31-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/3_hmnl/hmnl_files/figure-html/unnamed-chunk-31-1.png -------------------------------------------------------------------------------- /3_hmnl/hmnl_files/figure-html/unnamed-chunk-32-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/3_hmnl/hmnl_files/figure-html/unnamed-chunk-32-1.png -------------------------------------------------------------------------------- /3_hmnl/hmnl_files/figure-html/unnamed-chunk-40-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/3_hmnl/hmnl_files/figure-html/unnamed-chunk-40-1.png -------------------------------------------------------------------------------- /3_hmnl/hmnl_files/figure-html/unnamed-chunk-41-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/3_hmnl/hmnl_files/figure-html/unnamed-chunk-41-1.png -------------------------------------------------------------------------------- /3_hmnl/hmnl_files/figure-html/unnamed-chunk-42-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/3_hmnl/hmnl_files/figure-html/unnamed-chunk-42-1.png -------------------------------------------------------------------------------- /3_hmnl/hmnl_files/figure-html/unnamed-chunk-43-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/3_hmnl/hmnl_files/figure-html/unnamed-chunk-43-1.png -------------------------------------------------------------------------------- /3_hmnl/hmnl_files/figure-html/unnamed-chunk-44-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/3_hmnl/hmnl_files/figure-html/unnamed-chunk-44-1.png -------------------------------------------------------------------------------- /3_hmnl/hmnl_files/figure-html/unnamed-chunk-48-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/3_hmnl/hmnl_files/figure-html/unnamed-chunk-48-1.png -------------------------------------------------------------------------------- /3_hmnl/hmnl_files/figure-html/unnamed-chunk-49-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/3_hmnl/hmnl_files/figure-html/unnamed-chunk-49-1.png -------------------------------------------------------------------------------- /3_hmnl/hmnl_files/figure-html/unnamed-chunk-50-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/3_hmnl/hmnl_files/figure-html/unnamed-chunk-50-1.png -------------------------------------------------------------------------------- /3_hmnl/hmnl_files/figure-html/unnamed-chunk-51-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/3_hmnl/hmnl_files/figure-html/unnamed-chunk-51-1.png -------------------------------------------------------------------------------- /3_hmnl/hmnl_files/figure-html/unnamed-chunk-52-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/3_hmnl/hmnl_files/figure-html/unnamed-chunk-52-1.png -------------------------------------------------------------------------------- /3_hmnl/images/Bayesian_Inference.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/3_hmnl/images/Bayesian_Inference.png -------------------------------------------------------------------------------- /3_hmnl/images/lkj_prior.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/3_hmnl/images/lkj_prior.png -------------------------------------------------------------------------------- /4_hnestedmnl/.Rbuildignore: -------------------------------------------------------------------------------- 1 | ^.*\.Rproj$ 2 | ^\.Rproj\.user$ 3 | -------------------------------------------------------------------------------- /4_hnestedmnl/.gitignore: -------------------------------------------------------------------------------- 1 | hnestedmnl_cache 2 | hnestedmnl_files 3 | img/Adobe_standard_logo_RGB-eps-converted-to.pdf 4 | -------------------------------------------------------------------------------- /4_hnestedmnl/4_hnestedmnl.Rproj: -------------------------------------------------------------------------------- 1 | Version: 1.0 2 | 3 | RestoreWorkspace: Default 4 | SaveWorkspace: Default 5 | AlwaysSaveHistory: Default 6 | 7 | EnableCodeIndexing: Yes 8 | UseSpacesForTab: Yes 9 | NumSpacesForTab: 2 10 | Encoding: UTF-8 11 | 12 | RnwWeave: Sweave 13 | LaTeX: pdfLaTeX 14 | -------------------------------------------------------------------------------- /4_hnestedmnl/GPLv2.txt: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | 3 | Version 2, June 1991 4 | 5 | Copyright (C) 1989, 1991 Free Software Foundation, Inc. 6 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA 7 | 8 | Everyone is permitted to copy and distribute verbatim copies 9 | of this license document, but changing it is not allowed. 10 | Preamble 11 | 12 | The licenses for most software are designed to take away your freedom to share and change it. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change free software--to make sure the software is free for all its users. This General Public License applies to most of the Free Software Foundation's software and to any other program whose authors commit to using it. (Some other Free Software Foundation software is covered by the GNU Lesser General Public License instead.) You can apply it to your programs, too. 13 | 14 | When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for this service if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs; and that you know you can do these things. 15 | 16 | To protect your rights, we need to make restrictions that forbid anyone to deny you these rights or to ask you to surrender the rights. These restrictions translate to certain responsibilities for you if you distribute copies of the software, or if you modify it. 17 | 18 | For example, if you distribute copies of such a program, whether gratis or for a fee, you must give the recipients all the rights that you have. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights. 19 | 20 | We protect your rights with two steps: (1) copyright the software, and (2) offer you this license which gives you legal permission to copy, distribute and/or modify the software. 21 | 22 | Also, for each author's protection and ours, we want to make certain that everyone understands that there is no warranty for this free software. If the software is modified by someone else and passed on, we want its recipients to know that what they have is not the original, so that any problems introduced by others will not reflect on the original authors' reputations. 23 | 24 | Finally, any free program is threatened constantly by software patents. We wish to avoid the danger that redistributors of a free program will individually obtain patent licenses, in effect making the program proprietary. To prevent this, we have made it clear that any patent must be licensed for everyone's free use or not licensed at all. 25 | 26 | The precise terms and conditions for copying, distribution and modification follow. 27 | 28 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 29 | 30 | 0. This License applies to any program or other work which contains a notice placed by the copyright holder saying it may be distributed under the terms of this General Public License. The "Program", below, refers to any such program or work, and a "work based on the Program" means either the Program or any derivative work under copyright law: that is to say, a work containing the Program or a portion of it, either verbatim or with modifications and/or translated into another language. (Hereinafter, translation is included without limitation in the term "modification".) Each licensee is addressed as "you". 31 | 32 | Activities other than copying, distribution and modification are not covered by this License; they are outside its scope. The act of running the Program is not restricted, and the output from the Program is covered only if its contents constitute a work based on the Program (independent of having been made by running the Program). Whether that is true depends on what the Program does. 33 | 34 | 1. You may copy and distribute verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice and disclaimer of warranty; keep intact all the notices that refer to this License and to the absence of any warranty; and give any other recipients of the Program a copy of this License along with the Program. 35 | 36 | You may charge a fee for the physical act of transferring a copy, and you may at your option offer warranty protection in exchange for a fee. 37 | 38 | 2. You may modify your copy or copies of the Program or any portion of it, thus forming a work based on the Program, and copy and distribute such modifications or work under the terms of Section 1 above, provided that you also meet all of these conditions: 39 | 40 | a) You must cause the modified files to carry prominent notices stating that you changed the files and the date of any change. 41 | b) You must cause any work that you distribute or publish, that in whole or in part contains or is derived from the Program or any part thereof, to be licensed as a whole at no charge to all third parties under the terms of this License. 42 | c) If the modified program normally reads commands interactively when run, you must cause it, when started running for such interactive use in the most ordinary way, to print or display an announcement including an appropriate copyright notice and a notice that there is no warranty (or else, saying that you provide a warranty) and that users may redistribute the program under these conditions, and telling the user how to view a copy of this License. (Exception: if the Program itself is interactive but does not normally print such an announcement, your work based on the Program is not required to print an announcement.) 43 | These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Program, and can be reasonably considered independent and separate works in themselves, then this License, and its terms, do not apply to those sections when you distribute them as separate works. But when you distribute the same sections as part of a whole which is a work based on the Program, the distribution of the whole must be on the terms of this License, whose permissions for other licensees extend to the entire whole, and thus to each and every part regardless of who wrote it. 44 | 45 | Thus, it is not the intent of this section to claim rights or contest your rights to work written entirely by you; rather, the intent is to exercise the right to control the distribution of derivative or collective works based on the Program. 46 | 47 | In addition, mere aggregation of another work not based on the Program with the Program (or with a work based on the Program) on a volume of a storage or distribution medium does not bring the other work under the scope of this License. 48 | 49 | 3. You may copy and distribute the Program (or a work based on it, under Section 2) in object code or executable form under the terms of Sections 1 and 2 above provided that you also do one of the following: 50 | 51 | a) Accompany it with the complete corresponding machine-readable source code, which must be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, 52 | b) Accompany it with a written offer, valid for at least three years, to give any third party, for a charge no more than your cost of physically performing source distribution, a complete machine-readable copy of the corresponding source code, to be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, 53 | c) Accompany it with the information you received as to the offer to distribute corresponding source code. (This alternative is allowed only for noncommercial distribution and only if you received the program in object code or executable form with such an offer, in accord with Subsection b above.) 54 | The source code for a work means the preferred form of the work for making modifications to it. For an executable work, complete source code means all the source code for all modules it contains, plus any associated interface definition files, plus the scripts used to control compilation and installation of the executable. However, as a special exception, the source code distributed need not include anything that is normally distributed (in either source or binary form) with the major components (compiler, kernel, and so on) of the operating system on which the executable runs, unless that component itself accompanies the executable. 55 | 56 | If distribution of executable or object code is made by offering access to copy from a designated place, then offering equivalent access to copy the source code from the same place counts as distribution of the source code, even though third parties are not compelled to copy the source along with the object code. 57 | 58 | 4. You may not copy, modify, sublicense, or distribute the Program except as expressly provided under this License. Any attempt otherwise to copy, modify, sublicense or distribute the Program is void, and will automatically terminate your rights under this License. However, parties who have received copies, or rights, from you under this License will not have their licenses terminated so long as such parties remain in full compliance. 59 | 60 | 5. You are not required to accept this License, since you have not signed it. However, nothing else grants you permission to modify or distribute the Program or its derivative works. These actions are prohibited by law if you do not accept this License. Therefore, by modifying or distributing the Program (or any work based on the Program), you indicate your acceptance of this License to do so, and all its terms and conditions for copying, distributing or modifying the Program or works based on it. 61 | 62 | 6. Each time you redistribute the Program (or any work based on the Program), the recipient automatically receives a license from the original licensor to copy, distribute or modify the Program subject to these terms and conditions. You may not impose any further restrictions on the recipients' exercise of the rights granted herein. You are not responsible for enforcing compliance by third parties to this License. 63 | 64 | 7. If, as a consequence of a court judgment or allegation of patent infringement or for any other reason (not limited to patent issues), conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot distribute so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not distribute the Program at all. For example, if a patent license would not permit royalty-free redistribution of the Program by all those who receive copies directly or indirectly through you, then the only way you could satisfy both it and this License would be to refrain entirely from distribution of the Program. 65 | 66 | If any portion of this section is held invalid or unenforceable under any particular circumstance, the balance of the section is intended to apply and the section as a whole is intended to apply in other circumstances. 67 | 68 | It is not the purpose of this section to induce you to infringe any patents or other property right claims or to contest validity of any such claims; this section has the sole purpose of protecting the integrity of the free software distribution system, which is implemented by public license practices. Many people have made generous contributions to the wide range of software distributed through that system in reliance on consistent application of that system; it is up to the author/donor to decide if he or she is willing to distribute software through any other system and a licensee cannot impose that choice. 69 | 70 | This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License. 71 | 72 | 8. If the distribution and/or use of the Program is restricted in certain countries either by patents or by copyrighted interfaces, the original copyright holder who places the Program under this License may add an explicit geographical distribution limitation excluding those countries, so that distribution is permitted only in or among countries not thus excluded. In such case, this License incorporates the limitation as if written in the body of this License. 73 | 74 | 9. The Free Software Foundation may publish revised and/or new versions of the General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. 75 | 76 | Each version is given a distinguishing version number. If the Program specifies a version number of this License which applies to it and "any later version", you have the option of following the terms and conditions either of that version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of this License, you may choose any version ever published by the Free Software Foundation. 77 | 78 | 10. If you wish to incorporate parts of the Program into other free programs whose distribution conditions are different, write to the author to ask for permission. For software which is copyrighted by the Free Software Foundation, write to the Free Software Foundation; we sometimes make exceptions for this. Our decision will be guided by the two goals of preserving the free status of all derivatives of our free software and of promoting the sharing and reuse of software generally. 79 | 80 | NO WARRANTY 81 | 82 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 83 | 84 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. 85 | 86 | -------------------------------------------------------------------------------- /4_hnestedmnl/LICENSE: -------------------------------------------------------------------------------- 1 | # Copyright 2017, Kevin Van Horn 2 | # 3 | # Except where otherwise noted, licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use these files except in compliance with the License. 5 | # 6 | # You may obtain a copy of the License at 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /4_hnestedmnl/hnestedmnl.Rmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Stan Tutorial" 3 | subtitle: "Module 4: Hierarchical Nested Multinomial Logit" 4 | author: "Kevin Van Horn and Elea McDonnell Feit" 5 | date: "June 25, 2017" 6 | output: beamer_presentation 7 | --- 8 | 9 | ```{r setup, include=FALSE} 10 | knitr::opts_chunk$set(echo = FALSE) 11 | require(gridExtra) 12 | require(grid) 13 | require(jpeg) 14 | require(png) 15 | require(rstan) 16 | ``` 17 | 18 | ## Module Objectives 19 | 20 | In this module we illustrate how Stan can be used to fit a non-standard model. This flexibility to fit almost any model you can imagine is a key advantage of Stan. 21 | 22 | ## Acknowledgments 23 | 24 | ```{r, out.width="30%", fig.align="center" } 25 | knitr::include_graphics("img/themodellersnew.png") 26 | ``` 27 | 28 | Based on work done for the Modellers along with Michael Smith. Stan code © 2015, The Modellers, licensed under GPL v2. 29 | 30 | ```{r, fig.align="center"} 31 | knitr::include_graphics("img/Adobe_standard_logo_RGB.eps") 32 | ``` 33 | 34 | Thanks to Kevin's employer Adobe Systems for their support. 35 | 36 | ## The Problem: Failed Validation Tasks 37 | 38 | Choice-based conjoint survey; dual-response DCM. 39 | 40 | Validation tasks: one product shown, asked if they would buy. 41 | 42 | ![](img/barchart1.png)\ 43 | 44 | ## Posterior Predictive Probabilities 45 | 46 | Weighted average of choice probabilities. 47 | 48 | Weight by posterior probability of parameter vector $\psi$. 49 | $$ \Pr(c\mid s,i,D) = \int \Pr(c\mid s, i, \psi) p(\psi \mid D) \,\mathrm{d}\psi $$ 50 | 51 | Choice $c$, scenario $s$, individual $i$. 52 | 53 | Survey data $D$, model parameters $\psi$. 54 | 55 | Implemented by averaging predictions over posterior draws. 56 | 57 | $$ \Pr(c\mid s,i,D) \approx \frac{1}{n}\sum_{j=1}^n \Pr\left(c\mid s, i, \psi^{(j)}\right) $$ 58 | 59 | 60 | ## Jensen's Inequality 61 | 62 | For any convex function $f$, $E\left[f(\eta)\right] > f\left(E[\eta]\right)$. 63 | 64 | ![](img/jensens.png){ width=70% }\ 65 | 66 | Example: $\frac{1}{2}f(-4.5) + \frac{1}{2}f(-1.5) > f(-3)$ 67 | 68 | ## Jensen's Inequality (continued) 69 | 70 | $\Pr(\mbox{buy}) = f(\eta)$ where $f$ is inverse logit, $\eta=\sum_i\beta_i x_i$ 71 | 72 | $E\left[f(\eta)\right]$ is posterior predictive choice probability. 73 | 74 | $f\left(E[\eta]\right)$ is choice probability using point estimate of $\beta$. 75 | 76 | $f(\eta)$ is convex for $\eta < 0$. 77 | 78 |   79 | 80 | If $\eta < 0$ and using point estimate of $\beta$... 81 | 82 | ...computed choice prob $f\left(E[\eta]\right)$ is *biased downward*. 83 | 84 | ## Results: Using Posterior Predictive Probabilities 85 | 86 | ![](img/barchart2.png)\ 87 | 88 | ## One of These Things Is Not Like the Others 89 | 90 | ```{r, fig.align="center"} 91 | fig1 <- rasterGrob(readJPEG("img/auto1.jpg"), interpolate=TRUE) 92 | fig2 <- rasterGrob(readJPEG("img/auto2.jpg"), interpolate=TRUE) 93 | fig3 <- rasterGrob(readJPEG("img/auto3.jpg"), interpolate=TRUE) 94 | grid.arrange(fig1, fig2, fig3, ncol=3) 95 | #knitr::include_graphics("img/auto1.jpg") 96 | #knitr::include_graphics("img/auto2.jpg") 97 | #knitr::include_graphics("img/auto3.jpg") 98 | ``` 99 | 100 | ```{r, out.width="30%", fig.align="center"} 101 | knitr::include_graphics("img/not-buying.jpg") 102 | ``` 103 | 104 | Clashes with MNL property: Independence of Irrelevant Alternatives 105 | 106 | ## The Problem with Multinomial Logit 107 | 108 | **I Can Make You Buy If I Give You Enough Options** 109 | 110 | Even if they're all similar. 111 | 112 | ```{r, out.width="50%", fig.align="center"} 113 | knitr::include_graphics("img/lots-of-bag-cases.jpg") 114 | ``` 115 | 116 | ## The Math 117 | 118 | Let $\eta_i$ be linear predictor for product $i$. 119 | 120 | Linear predictor for `none` is 0. 121 | 122 | $$\Pr\left(\mbox{none}\right) = \frac{1}{1+\sum_{i=1}^C \exp\left(\eta_i\right)}$$ 123 | 124 | So $\Pr(\mbox{none})\rightarrow 0$ as $C \rightarrow \infty$. 125 | 126 | Tasks used for estimation: 5 product alternatives and `none`. 127 | 128 | Tasks used for validation: 1 product and `none`. 129 | 130 | ## Nested Multinomial Logit ("none" vs. all else) 131 | 132 | $$\Pr(\mbox{choose product $i$}) = \frac{1}{1+\exp(-\alpha-\lambda I)} \cdot \frac{\exp(\boldsymbol{\beta}' \boldsymbol{x}_i)}{\sum_j \exp(\boldsymbol{\beta}' \boldsymbol{x}_j)}$$ 133 | 134 | $$\Pr(\mbox{choose none}) = \frac{1}{1+\exp(\alpha + \lambda I)}$$ 135 | 136 | $$I = \log \sum_j \exp(\boldsymbol{\beta}' \boldsymbol{x}_j)$$ 137 | 138 | $0\leq \lambda \leq 1$; if $\lambda = 1$ then MNL. 139 | 140 | ## Hierarchical Nested MNL 141 | 142 | $r$ indexes respondents. 143 | $$\Pr(\mbox{choose product $i$}) = \frac{1}{1+\exp(-\alpha_r-\lambda I_r)} \cdot \frac{\exp(\boldsymbol{\beta}_r' \boldsymbol{x}_i)}{\sum_j \exp(\boldsymbol{\beta}_r' \boldsymbol{x}_j)}$$ 144 | 145 | $$\Pr(\mbox{choose none}) = \frac{1}{1+\exp(\alpha_r + \lambda I_r)}$$ 146 | 147 | $$I_r = \log \sum_j \exp(\boldsymbol{\beta}_r' \boldsymbol{x}_j)$$ 148 | 149 | $$\begin{array}{rcl}\beta_{rk} & \sim & \mathrm{Normal}\left(\boldsymbol{\theta}_k' \boldsymbol{z}_r,\, \sigma_k\right) \\ 150 | \alpha_r & \sim & \mathrm{Normal}\left(\boldsymbol{\varphi}' \boldsymbol{z}_r,\, \sigma_{\alpha}\right) \end{array}$$ 151 | 152 | 153 | $$\begin{array}{rcl}\lambda & \sim & \mathrm{Uniform}(0,1) \\ 154 | \theta_{kg} & \sim & \mathrm{Normal}(0,10) \\ 155 | \sigma_k & \sim & \mathrm{HalfNormal}(5) \end{array}$$ 156 | 157 | ## Initial Stan Model (1) 158 | 159 | ```{stan, eval=FALSE, echo=TRUE, output.var="dummy"} 160 | data { 161 | int R; // # respondents 162 | int K; // # product covariates; no intercept 163 | int G; // # respondent covariates 164 | int S; // # scenarios per respondent 165 | int C; // # alts (choices) per scenario 166 | matrix[C, K] X[R, S]; 167 | // X[r,s] is cov mat of scen s for resp r. 168 | matrix[G, R] Z; 169 | // Z[,r] is vector of covariates for resp r 170 | int Y1[R,S]; // forced choice 171 | int Y2[R,S]; // dual response 172 | } 173 | ``` 174 | 175 | ## Initial Stan Model (2) 176 | 177 | ```{stan, eval=FALSE, echo=TRUE, output.var="dummy"} 178 | parameters { 179 | real lambda; 180 | vector[K+1] sigma; 181 | matrix[K+1, G] Theta; 182 | matrix[K+1, R] Beta; 183 | } 184 | model { 185 | lambda ~ uniform(0, 1); 186 | sigma ~ normal(0, 5); 187 | to_vector(Theta) ~ normal(0, 10); 188 | to_vector(Beta) ~ 189 | normal(to_vector(Theta * Z), 190 | to_vector(rep_matrix(sigma, R))); 191 | ... 192 | } 193 | ``` 194 | 195 | ## Vectorization 196 | 197 | ```{stan, eval=FALSE, echo=TRUE, output.var="dummy"} 198 | to_vector(Theta) ~ normal(0, 10); 199 | to_vector(Beta) ~ 200 | normal(to_vector(Theta * Z), 201 | to_vector(rep_matrix(sigma, R))); 202 | ``` 203 | 204 | is equivalent to 205 | 206 | ```{stan, eval=FALSE, echo=TRUE, output.var="dummy"} 207 | for (k in 1:(K+1)) 208 | for (g in 1:G) 209 | Theta[k,g] ~ normal(0, 10); 210 | for (k in 1:(K+1)) 211 | for (r in 1:R) 212 | Beta[k, r] ~ normal((Theta * Z)[k, r], sigma[k]); 213 | ``` 214 | 215 | ## Initial Stan Model (3) 216 | 217 | ```{stan, eval=FALSE, echo=TRUE, output.var="dummy"} 218 | model { 219 | ... 220 | for (r in 1:R) { 221 | vector[K] b = Beta[2:(K+1), r]; 222 | real alpha = Beta[1,r]; 223 | for (s in 1:S) { 224 | vector[C] u = X[r,s] * b; 225 | real u_buy = alpha + lambda * log_sum_exp(u); 226 | Y1[r,s] ~ categorical_logit(u); 227 | Y2[r,s] ~ bernoulli_logit(u_buy); 228 | } 229 | } 230 | } 231 | ``` 232 | 233 | ## Some Stan Details 234 | 235 | `log_sum_exp(u)` is numerically robust form of 236 | $$ \log\left(\sum_i \exp\left(u_i\right)\right) $$ 237 | 238 | `y ~ categorical_logit(u)` means 239 | $$ \Pr(y=i) = \frac{\exp\left(u_i\right)}{\sum_j \exp\left(u_j\right)}$$ 240 | `y ~ bernoulli_logit(u)` means 241 | 242 | $$ \Pr(y = 1) = \frac{\exp(u)}{1 + \exp(u)}$$ 243 | 244 | ## And When We Run It... 245 | 246 | ```{r, echo=FALSE} 247 | message('There were 11 divergent transitions after warmup.\nIncreasing adapt_delta above 0.8 may help.\nSee http://mc-stan.org/misc/warnings.html#divergent-transitions-after-warmup') 248 | message('There were 4 chains where the estimated\nBayesian Fraction of Missing Information was low.\nSee 249 | http://mc-stan.org/misc/warnings.html#bfmi-low') 250 | ``` 251 | 252 | ## An Extreme Form of the Problem: Neal's Funnel 253 | 254 | ```{stan, eval=FALSE, echo=TRUE, output.var="dummy"} 255 | parameters { 256 | real y; 257 | vector[9] x; 258 | } 259 | model { 260 | y ~ normal(0,3); 261 | x ~ normal(0,exp(y/2)); 262 | } 263 | ``` 264 | 265 | Variance for $x$ strongly varies with $y$! 266 | 267 | ## Distribution for Neal's Funnel 268 | 269 | ```{r, out.width="100%", fig.align="center"} 270 | knitr::include_graphics("img/funnel-plot.png") 271 | ``` 272 | 273 | ## Why Neal's Funnel Is a Problem 274 | 275 | Hamiltonian MC simulates trajectory of system whose potential energy is 276 | $$V(x) = -\log(\mbox{prob density at }x)$$ 277 | 278 | To efficiently explore distribution, need 279 | 280 | - large step size when curvature of $V(x)$ is small; 281 | - small step size when curvature of $V(x)$ is large. 282 | 283 | But have to use a single step size globally. 284 | 285 | - Normal distribution: constant curvature. 286 | - Neal's Funnel: extreme variation in curvature. 287 | 288 | HMC has trouble entering the narrow neck of the funnel. 289 | 290 | Warning / error message: "divergent transitions after warmup." 291 | 292 | ## Run the Funnel Model 293 | 294 | ```{r, cache=TRUE, echo=TRUE, results='hide', warning=FALSE, error=FALSE, message=FALSE} 295 | fit <- stan_demo('funnel', seed=928374) 296 | pairs(fit, pars=c('y','x[1]'), las=1) 297 | ``` 298 | 299 | ## Reparameterized Model 300 | 301 | Non-Centered Parameterization 302 | 303 | ```{stan, eval=FALSE, echo=TRUE, output.var="dummy"} 304 | parameters { 305 | real y_raw; 306 | vector[9] x_raw; 307 | } 308 | transformed parameters { 309 | real y = 3.0 * y_raw; 310 | vector[9] x = exp(y/2) * x_raw; 311 | } 312 | model { 313 | y_raw ~ normal(0,1); 314 | x_raw ~ normal(0,1); 315 | } 316 | ``` 317 | 318 | ## Run the Reparameterized Model 319 | 320 | ```{r, cache=TRUE, echo=TRUE, results='hide', warning=FALSE, error=FALSE, message=FALSE} 321 | fit <- stan_demo('funnel_reparam', seed=928374) 322 | pairs(fit, pars=c('y_raw','x_raw[1]'), las=1) 323 | ``` 324 | 325 | ## Reparameterized Stan Model (2a) 326 | 327 | ```{stan, eval=FALSE, echo=TRUE, output.var="dummy"} 328 | parameters { 329 | real lambda; 330 | // vector[K+1] sigma; 331 | vector[K+1] sigma_raw; 332 | // matrix[K+1, G] Theta; 333 | matrix[K+1, G] Theta_raw; 334 | // matrix[K+1, R] Beta; 335 | matrix[K+1, R] Epsilon; 336 | } 337 | ``` 338 | 339 | ## Reparameterized Stan Model (2b) 340 | 341 | ```{stan, eval=FALSE, echo=TRUE, output.var="dummy"} 342 | transformed parameters { 343 | vector[K+1] sigma = 5 * sigma_raw; 344 | matrix[K+1, G] Theta = 10 * Theta_raw; 345 | matrix[K+1, R] Beta = 346 | Theta * Z + diag_pre_multiply(sigma, Epsilon); 347 | } 348 | model { 349 | lambda ~ uniform(0, 1); 350 | // sigma ~ normal(0, 5); 351 | sigma_raw ~ normal(0, 1); 352 | // to_vector(Theta) ~ normal(0, 10); 353 | to_vector(Theta_raw) ~ normal(0, 1); 354 | // to_vector(Beta) ~ normal(to_vector(Theta * Z), 355 | // to_vector(rep_matrix(sigma, R))); 356 | to_vector(Epsilon) ~ normal(0, 1); 357 | ... 358 | } 359 | ``` 360 | 361 | ## Results: Hnestedmnl + Posterior Predictive Probs 362 | 363 | Histograms of $p^{(j)}$ with $j$ running over posterior draws. 364 | $$ p^{(j)} = \frac{1}{R}\sum_{r=1}^R \Pr\left(\mbox{buy} \mid r, \psi^{(j)}\right) $$ 365 | 366 | Each of three validation tasks. 367 | 368 | Blue lines are empirical probs on holdout data. 369 | 370 | ```{r, fig.align="center"} 371 | fig1 <- rasterGrob(readPNG("img/vtask1.png"), interpolate=TRUE) 372 | fig2 <- rasterGrob(readPNG("img/vtask2.png"), interpolate=TRUE) 373 | fig3 <- rasterGrob(readPNG("img/vtask3.png"), interpolate=TRUE) 374 | grid.arrange(fig1, fig2, fig3, ncol=3) 375 | ``` 376 | 377 | # Summary of Module 4 378 | 379 | ## In this module we have: 380 | 381 | - Discussed posterior predictive probabilities 382 | + How to compute them 383 | + When predictions from point estimates are biased 384 | 385 | - Created a custom model to solve a specific problem 386 | + A common case where MNL goes wrong: `none` vs actual product 387 | + Solution using restricted form of nested multinomial logit 388 | + Converting equations into Stan code 389 | 390 | - Discussed computational issues 391 | + Vectorization 392 | + Problem of highly variable posterior curvature 393 | + Non-centered parameterization 394 | -------------------------------------------------------------------------------- /4_hnestedmnl/hnestedmnl.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/4_hnestedmnl/hnestedmnl.pdf -------------------------------------------------------------------------------- /4_hnestedmnl/hnestedmnl.stan: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2015, The Modellers / Hall and Partners 2 | # 3 | # This program is free software; you can redistribute it and/or 4 | # modify it under the terms of the GNU General Public License 5 | # version 2, as published by the Free Software Foundation. 6 | # 7 | # This program is distributed in the hope that it will be useful, 8 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | # GNU General Public License for more details. 11 | # 12 | # You should have received a copy of the GNU General Public License 13 | # along with this program. If not, see . 14 | # 15 | data { 16 | int R; // # of respondents 17 | int K; // # of product covariates; no intercept 18 | int G; // # of respondent covariates 19 | int S; // # of scenarios per respondent 20 | int C; // # of alternatives (choices) per scenario 21 | matrix[C, K] X[R, S]; // X[r,s] is covariate matrix of scenario s for respondent r. 22 | matrix[G, R] Z; // Z[,r] is vector of covariates describing respondent r 23 | int Y1[R,S]; // forced-choice responses 24 | int Y2[R,S]; // whether respondent would choose any of the options 25 | } 26 | parameters { 27 | real lambda; 28 | vector[K+1] sigma_raw; 29 | matrix[K+1, G] Theta_raw; 30 | matrix[K+1, R] Epsilon; 31 | } 32 | transformed parameters { 33 | vector[K+1] sigma = 5 * sigma_raw; 34 | matrix[K+1, G] Theta = 10 * Theta_raw; 35 | matrix[K+1, R] Beta = Theta * Z + diag_pre_multiply(sigma, Epsilon); 36 | } 37 | model { 38 | lambda ~ uniform(0, 1); 39 | sigma_raw ~ normal(0, 1); 40 | to_vector(Theta_raw) ~ normal(0, 1); 41 | to_vector(Epsilon) ~ normal(0, 1); 42 | for (r in 1:R) { 43 | vector[K] b = Beta[2:(K+1), r]; 44 | real alpha = Beta[1,r]; 45 | for (s in 1:S) { 46 | vector[C] u = X[r,s] * b; 47 | real u_buy = alpha + lambda * log_sum_exp(u); 48 | Y1[r,s] ~ categorical_logit(u); 49 | Y2[r,s] ~ bernoulli_logit(u_buy); 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /4_hnestedmnl/img/Adobe_standard_logo_RGB.eps: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/4_hnestedmnl/img/Adobe_standard_logo_RGB.eps -------------------------------------------------------------------------------- /4_hnestedmnl/img/auto1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/4_hnestedmnl/img/auto1.jpg -------------------------------------------------------------------------------- /4_hnestedmnl/img/auto2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/4_hnestedmnl/img/auto2.jpg -------------------------------------------------------------------------------- /4_hnestedmnl/img/auto3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/4_hnestedmnl/img/auto3.jpg -------------------------------------------------------------------------------- /4_hnestedmnl/img/barchart1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/4_hnestedmnl/img/barchart1.png -------------------------------------------------------------------------------- /4_hnestedmnl/img/barchart2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/4_hnestedmnl/img/barchart2.png -------------------------------------------------------------------------------- /4_hnestedmnl/img/funnel-plot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/4_hnestedmnl/img/funnel-plot.png -------------------------------------------------------------------------------- /4_hnestedmnl/img/jensens.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/4_hnestedmnl/img/jensens.png -------------------------------------------------------------------------------- /4_hnestedmnl/img/lots-of-bag-cases.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/4_hnestedmnl/img/lots-of-bag-cases.jpg -------------------------------------------------------------------------------- /4_hnestedmnl/img/not-buying.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/4_hnestedmnl/img/not-buying.jpg -------------------------------------------------------------------------------- /4_hnestedmnl/img/themodellersnew.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/4_hnestedmnl/img/themodellersnew.png -------------------------------------------------------------------------------- /4_hnestedmnl/img/vtask1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/4_hnestedmnl/img/vtask1.png -------------------------------------------------------------------------------- /4_hnestedmnl/img/vtask2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/4_hnestedmnl/img/vtask2.png -------------------------------------------------------------------------------- /4_hnestedmnl/img/vtask3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/4_hnestedmnl/img/vtask3.png -------------------------------------------------------------------------------- /5_data_fusion/SplitQuestionaire.R: -------------------------------------------------------------------------------- 1 | # Data Fusion / split quetionaire example in Stan 2 | # Elea McDonnell Feit, eleafeit@gmail.com 3 | # 11 March 2016 4 | 5 | # From Feit, Elea McDonnel and Eric T. Bradlow (2017) Fusion Modeling 6 | # Invited chapter in Handbook of Market Research by 7 | # Christian Homburg, Martin Klarmann and Arnd Vomberg 8 | 9 | # Copyright 2017, Elea McDonnell Feit 10 | # 11 | # Licensed under the Apache License, Version 2.0 (the "License"); 12 | # you may not use this file except in compliance with the License. 13 | # 14 | # You may obtain a copy of the License at 15 | # http://www.apache.org/licenses/LICENSE-2.0 16 | # 17 | # Unless required by applicable law or agreed to in writing, software 18 | # distributed under the License is distributed on an "AS IS" BASIS, 19 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 20 | # See the License for the specific language governing permissions and 21 | # limitations under the License. 22 | 23 | rm(list=ls()) 24 | setwd("~/GitHub/artforum2017/5_data_fusion") 25 | list.files() 26 | 27 | library(MASS) 28 | library(coda) 29 | library(beanplot) 30 | library(rstan) 31 | 32 | # Functions =============================================================== 33 | data.mvn.split <- function(K1=2, K2=2, Kb=3, N1=100, N2=100, 34 | mu=rep(0, K1+K2+Kb), Sigma=diag(1, K1+K2+Kb)) 35 | { 36 | y <- mvrnorm(n=N1+N2, mu=mu, Sigma=Sigma) 37 | list(data=list(K1=K1, K2=K2, Kb=Kb, N1=N1, N2=N2, 38 | y1=as.matrix(y[1:N1, 1:K1], col=K1), 39 | y2=as.matrix(y[N1+1:N2, K1+1:K2], col=K2), 40 | yb=as.matrix(y[,K1+K2+1:Kb], col=Kb)), 41 | true=list(mu=mu, Sigma=Sigma, 42 | y1mis=y[1:N1, K1+1:K2], 43 | y2mis=y[N1+1:N2, 1:K1])) 44 | } 45 | 46 | data.mvp.split <- function(K1=2, K2=2, Kb=3, N1=100, N2=100, 47 | mu=rep(0, K1+K2+Kb), Sigma=diag(1, K1+K2+Kb)) 48 | { 49 | z <- mvrnorm(n=N1+N2, mu=mu, Sigma=Sigma) 50 | y <- z 51 | y[y>0] <- 1 52 | y[y<0] <- 0 53 | y1mis <- y[1:N1, K1+1:K2] 54 | y2mis <- y[N1+1:N2, 1:K1] 55 | y[1:N1, K1+1:K2] <- NA 56 | y[N1+1:N2, 1:K1] <- NA 57 | true=list(mu=mu, Sigma=Sigma, z=z, y=y, y1mis=y1mis, y2mis=y2mis) 58 | y[is.na(y)] <- 0 59 | data=list(K1=K1, K2=K2, Kb=Kb, N1=N1, N2=N2, y=y) 60 | list(data=data, true=true) 61 | } 62 | 63 | plot.post.density <- function(m.stan, pars, true, prefix=NULL){ 64 | for (i in 1:length(pars)) { 65 | draws <- As.mcmc.list(m.stan, pars=pars[i]) 66 | if (!is.null(prefix)) { 67 | filename <- paste(prefix, "Post", pars[i], ".png", sep="") 68 | png(filename=filename, width=600, height=400) 69 | } 70 | beanplot(data.frame(draws[[1]]), 71 | horizontal=TRUE, las=1, what=c(0, 1, 1, 0), side="second", 72 | main=paste("Posterior Density of", pars[[i]])) 73 | if (!is.null(prefix)) dev.off() 74 | } 75 | } 76 | 77 | plot.true.v.est <- function(m.stan, pars, true, prefix=NULL){ 78 | for (i in 1:length(pars)) { 79 | draws <- As.mcmc.list(m.stan, pars=pars[i]) 80 | est <- summary(draws) 81 | if (!is.null(prefix)) { 82 | filename <- paste(prefix, "TrueVEst", pars[i], ".png", sep="") 83 | png(filename=filename, width=600, height=400) 84 | } 85 | plot(true[[i]], est$quantiles[,3], col="blue", 86 | xlab=paste("True", pars[i]), 87 | ylab=paste("Estiamted", pars[i], "(posterior median)")) 88 | abline(a=0, b=1) 89 | arrows(true[[i]], est$quantiles[,3], true[[i]], est$quantiles[,1], 90 | col="gray90", length=0) 91 | arrows(true[[i]], est$quantiles[,3], true[[i]], est$quantiles[,5], 92 | col="gray90", length=0) 93 | points(true[[i]], est$quantiles[,3], col="blue") 94 | if (!is.null(prefix)) dev.off() 95 | } 96 | } 97 | 98 | # Example 1a: MVN ================================================================ 99 | # Generate synthetic data 100 | set.seed(20030601) 101 | Sigma <- matrix(c(1, 0.3, -0.2, 0.7, 0.3, 1, -0.6, 0.4, 102 | -0.2, -0.6, 1, 0.1, 0.7, 0.4, 0.1, 1), nrow=4) 103 | d1 <- data.mvn.split(K1=1, K2=1, Kb=2, N1=100, N2=100, mu=rep(0,4), Sigma=Sigma) 104 | str(d1$data) 105 | # Call to Stan to generate posterior draws 106 | m1 <- stan(file="SplitQuestionaire.stan", data=d1$data, 107 | iter=10000, warmup=2000, chains=1, seed=12) 108 | # Summaries of posterior draws for population-level parameters 109 | summary(m1, par=c("mu")) 110 | summary(m1, par=c("tau")) 111 | summary(m1, par=c("Omega")) 112 | plot.post.density(m1, pars=c("mu", "tau"), 113 | true=list(d1$true$mu, sqrt(diag(d1$true$Sigma)), 114 | cov2cor(d1$true$Sigma))) 115 | draws <- As.mcmc.list(m1, pars=c("Omega")) 116 | beanplot(data.frame(draws[[1]][,c(2:4, 7:8, 12)]), 117 | horizontal=TRUE, las=1, what=c(0, 1, 1, 0), side="second", 118 | main=paste("Posterior Density of Omega (correlations)", log=""), 119 | cex.axis=0.5) 120 | # Summaries of posterior draws for missing data 121 | summary(extract(m1, par=c("y1mis"))$y1mis[,3,]) 122 | plot(density(extract(m1, par=c("y1mis"))$y1mis[,3,]), 123 | main="Posterior of Unobserved y_1", xlab="y_1") 124 | 125 | summary(m1, par=c("y")) # posteriors of observed data place a point mass at the observed value 126 | plot.true.v.est(m1, pars=c("y1mis", "y2mis"), 127 | true=list(d1$true$y1mis, d1$true$y2mis)) 128 | 129 | # Example 1b: MVN with zero correlations ======================================= 130 | # Generate synthetic data 131 | set.seed(20030601) 132 | Sigma <- matrix(0, nrow=4, ncol=4) 133 | diag(Sigma) <- 1 134 | # Call to Stan to generate posterior draws 135 | d2 <- data.mvn.split(K1=1, K2=1, Kb=2, N1=100, N2=100, mu=rep(0,4), Sigma=Sigma) 136 | m2 <- stan(file="SplitQuestionaire.stan", data=d2$data, 137 | iter=10000, warmup=2000, chains=1, seed=12) 138 | # Summarize posteriors of population-level parameters 139 | summary(m2, par=c("mu")) 140 | summary(m2, par=c("tau")) 141 | summary(m2, par=c("Omega")) 142 | plot.post.density(m2, pars=c("mu", "tau"), 143 | true=list(d1$true$mu, sqrt(diag(d1$true$Sigma)), 144 | cov2cor(d1$true$Sigma))) 145 | draws <- As.mcmc.list(m2, pars=c("Omega")) 146 | beanplot(data.frame(draws[[1]][,c(2:4, 7:8, 12)]), 147 | horizontal=TRUE, las=1, what=c(0, 1, 1, 0), side="second", 148 | main=paste("Posterior Density of Omega", log=""), cex.axis=0.5) 149 | 150 | # Summaries of posterior draws for missing data 151 | plot.true.v.est(m2, pars=c("y1mis", "y2mis"), 152 | true=list(d2$true$y1mis, d2$true$y2mis)) 153 | 154 | # Example 1c: MVN with strong positive correlations =========================== 155 | # Generate synthetic data 156 | set.seed(20030601) 157 | Sigma <- matrix(0.9, nrow=4, ncol=4) 158 | diag(Sigma) <- 1 159 | # Call to Stan to generate posterior draws 160 | d3 <- data.mvn.split(K1=1, K2=1, Kb=2, N1=100, N2=100, mu=rep(0,4), Sigma=Sigma) 161 | m3 <- stan(file="SplitQuestionaire.stan", data=d3$data, 162 | iter=10000, warmup=2000, chains=1, seed=12) 163 | # Summaries of population-level parameters 164 | summary(m3, par=c("mu")) 165 | summary(m3, par=c("tau")) 166 | summary(m3, par=c("Omega")) 167 | plot.post.density(m3, pars=c("mu", "tau"), 168 | true=list(d1$true$mu, sqrt(diag(d1$true$Sigma)))) 169 | draws <- As.mcmc.list(m3, pars=c("Omega")) 170 | beanplot(data.frame(draws[[1]][,c(2:4, 7:8, 12)]), 171 | horizontal=TRUE, las=1, what=c(0, 1, 1, 0), side="second", 172 | main=paste("Posterior Density of Omega", log="")) 173 | 174 | # Summaries of posterior draws for missing data 175 | plot.true.v.est(m3, pars=c("y1mis", "y2mis"), 176 | true=list(d3$true$y1mis, d3$true$y2mis)) 177 | 178 | # Example 2: MVP ================================================================= 179 | # Generate synthetic data 180 | set.seed(20030601) 181 | Sigma <- matrix(c(1, 0.3, -0.2, 0.7, 0.3, 1, -0.6, 0.4, 182 | -0.2, -0.6, 1, 0.1, 0.7, 0.4, 0.1, 1), nrow=4) 183 | d1 <- data.mvp.split(K1=1, K2=1, Kb=2, N1=100, N2=100, mu=rep(0,4), Sigma=Sigma) 184 | # Call to Stan to generate posterior draws 185 | m1 <- stan(file="SplitQuestionaireMVP.stan", data=d1$data, 186 | iter=10000, warmup=2000, chains=1, seed=35) 187 | # Check that the z's are consistent with their observed values 188 | summary(m1, par=c("z")) 189 | d1$true$y 190 | # Summaries of posteriors of population-level parameters 191 | summary(m1, par=c("mu", "Omega")) 192 | plot.post.density(m1, pars=c("mu"), true=list(d1$true$mu)) 193 | draws <- As.mcmc.list(m1, pars=c("Omega")) 194 | beanplot(data.frame(draws[[1]][,c(2:4, 7:8, 12)]), 195 | horizontal=TRUE, las=1, what=c(0, 1, 1, 0), side="second", 196 | main=paste("Posterior Density of Omega", log="")) 197 | 198 | # Summarize posterior of missing data 199 | y1mis.est <- summary(m1, par=c("y1mis"))$summary 200 | y1mis.est[1,] 201 | xtabs(~y1mis.est[,"50%"]+d1$true$y1mis) 202 | y2mis.est <- summary(m1, par=c("y1mis"))$summary 203 | xtabs(~y2mis.est[,"50%"]+d1$true$y2mis) 204 | z.est <- data.frame(z.true=as.vector(t(d1$true$z)), y=as.vector(t(d1$true$y)), 205 | z.postmed=summary(m1, pars=c("z"))$summary[,"50%"]) 206 | plot(z.est[,c(1,3)], xlab="True Latent Variable", ylab="Posterior Mean of Latent Variable") 207 | points(z.est[is.na(z.est$y), c(1,3)], col="red") 208 | abline(h=0, v=0) 209 | 210 | 211 | Sigma <- matrix(0, nrow=4, ncol=4) 212 | diag(Sigma) <- 1 213 | d2 <- data.mvp.split(K1=1, K2=1, Kb=2, N1=100, N2=100, mu=rep(0,4), Sigma=Sigma) 214 | m2 <- stan(file="SplitQuestionaireMVP.stan", data=d2$data, 215 | iter=10000, warmup=2000, chains=1, seed=35) 216 | print(m2, par=c("mu", "Omega")) 217 | plot.post.density(m2, pars=c("mu"), true=list(d2$true$mu)) 218 | draws <- As.mcmc.list(m2, pars=c("Omega")) 219 | beanplot(data.frame(draws[[1]][,c(2:4, 7:8, 12)]), 220 | horizontal=TRUE, las=1, what=c(0, 1, 1, 0), side="second", 221 | main=paste("Posterior Density of Omega", log="")) 222 | 223 | y1mis.est <- summary(m2, par=c("y1mis"))$summary 224 | xtabs(~y1mis.est[,"50%"]+d2$true$y1mis) 225 | y2mis.est <- summary(m2, par=c("y1mis"))$summary 226 | xtabs(~y2mis.est[,"50%"]+d2$true$y2mis) 227 | z.est <- data.frame(z.true=as.vector(t(d2$true$z)), y=as.vector(t(d2$true$y)), 228 | z.postmed=summary(m2, pars=c("z"))$summary[,"50%"]) 229 | plot(z.est[,c(1,3)], xlab="True Latent Variable", ylab="Posterior Mean of Latent Variable") 230 | points(z.est[is.na(z.est$y), c(1,3)], col="red") 231 | abline(h=0, v=0) 232 | 233 | 234 | 235 | Sigma <- matrix(0.9, nrow=4, ncol=4) 236 | diag(Sigma) <- 1 237 | d3 <- data.mvp.split(K1=1, K2=1, Kb=2, N1=100, N2=100, mu=rep(0,4), Sigma=Sigma) 238 | m3 <- stan(file="SplitQuestionaireMVP.stan", data=d3$data, 239 | iter=10000, warmup=2000, chains=1, seed=12) 240 | print(m3, par=c("mu", "Omega")) 241 | plot.post.density(m3, pars=c("mu"), true=list(d2$true$mu)) 242 | draws <- As.mcmc.list(m3, pars=c("Omega")) 243 | beanplot(data.frame(draws[[1]][,c(2:4, 7:8, 12)]), 244 | horizontal=TRUE, las=1, what=c(0, 1, 1, 0), side="second", 245 | main=paste("Posterior Density of Omega", log="")) 246 | 247 | y1mis.est <- summary(m3, par=c("y1mis"))$summary 248 | xtabs(~y1mis.est[,"50%"]+d3$true$y1mis) 249 | y2mis.est <- summary(m3, par=c("y1mis"))$summary 250 | xtabs(~y2mis.est[,"50%"]+d3$true$y2mis) 251 | z.est <- data.frame(z.true=as.vector(t(d3$true$z)), y=as.vector(t(d3$true$y)), 252 | z.postmed=summary(m3, pars=c("z"))$summary[,"50%"]) 253 | plot(z.est[,c(1,3)], xlab="True Latent Variable", ylab="Posterior Mean of Latent Variable") 254 | points(z.est[is.na(z.est$y), c(1,3)], col="red") 255 | abline(h=0, v=0) 256 | 257 | 258 | Sigma <- matrix(0.5, nrow=7, ncol=7) 259 | diag(Sigma) <- 1 260 | d4 <- data.mvp.split(K1=2, K2=2, Kb=3, N1=100, N2=100, mu=rep(0,7), Sigma=Sigma) 261 | m4 <- stan(file="SplitQuestionaireMVP20160312.stan", data=d4$data, 262 | iter=10000, warmup=2000, chains=1, seed=12) 263 | print(m4, pars=c("mu", "Omega")) 264 | -------------------------------------------------------------------------------- /5_data_fusion/SplitQuestionaire.stan: -------------------------------------------------------------------------------- 1 | data { 2 | int N1; //obserations in data set 1 3 | int N2; //observations in data set 2 4 | int K1; //number of variables observed in data set 1 only 5 | int K2; //number of variables observed in data set 2 only 6 | int Kb; //number of variables observed in both data sets 7 | vector[K1] y1[N1]; 8 | vector[K2] y2[N2]; 9 | vector[Kb] yb[N1 + N2]; 10 | } 11 | parameters { 12 | vector[K1 + K2 + Kb] mu; 13 | corr_matrix[K1 + K2 + Kb] Omega; 14 | vector[K1 + K2 + Kb] tau; 15 | vector[K2] y1mis[N1]; 16 | vector[K1] y2mis[N2]; 17 | } 18 | transformed parameters{ 19 | vector[K1 + K2 + Kb] y[N1 + N2]; 20 | for (n in 1:N1) { 21 | for (k in 1:K1) y[n][k] = y1[n][k]; 22 | for (k in 1:K2) y[n][K1 + k] = y1mis[n][k]; 23 | for (k in 1:Kb) y[n][K1 + K2 + k] = yb[n][k]; 24 | } 25 | for (n in 1:N2) { 26 | for (k in 1:K1) y[N1+n][k] = y2mis[n][k]; 27 | for (k in 1:K2) y[N1+n][K1+k] = y2[n][k]; 28 | for (k in 1:Kb) y[N1+n][K1+K2+k] = yb[N1+n][k]; 29 | } 30 | } 31 | model { 32 | //priors 33 | mu ~ normal(0, 100); 34 | tau ~ cauchy(0,2.5); 35 | Omega ~ lkj_corr(2); 36 | //likelihood 37 | y ~ multi_normal(mu, quad_form_diag(Omega, tau)); 38 | } 39 | -------------------------------------------------------------------------------- /5_data_fusion/SplitQuestionaireMVP.stan: -------------------------------------------------------------------------------- 1 | functions { 2 | int mysum(int[,] a) { 3 | int s; 4 | s = 0; 5 | for (i in 1:size(a)) 6 | s = s + sum(a[i]); 7 | return s; 8 | } 9 | } 10 | data { 11 | int K1; // number of vars only observed in data set 1 12 | int K2; // number of vars only observed in data set 2 13 | int Kb; // number of vars observed in both data sets 14 | int N1; // number of observations in data set 1 15 | int N2; // number of observations in data set 2 16 | int y[N1+N2, K1+K2+Kb]; // should contain zeros in missing positions 17 | } 18 | transformed data { 19 | int n_pos[mysum(y)]; 20 | int k_pos[size(n_pos)]; 21 | int n_neg[(N1+N2)*(K1+K2+Kb) - K2*N1 - K1*N2 - mysum(y)]; 22 | int k_neg[size(n_neg)]; 23 | int N_pos; 24 | int N_neg; 25 | N_pos = size(n_pos); 26 | N_neg = size(n_neg); 27 | { 28 | int i; 29 | int j; 30 | i = 1; 31 | j = 1; 32 | for (n in 1:N1) { //positions in observed y1 33 | for (k in 1:K1) { 34 | if (y[n,k] == 1) { 35 | n_pos[i] = n; 36 | k_pos[i] = k; 37 | i = i + 1; 38 | } else { 39 | n_neg[j] = n; 40 | k_neg[j] = k; 41 | j = j + 1; 42 | } 43 | } 44 | for (k in (K1+K2+1):(K1+K2+Kb)) { 45 | if (y[n,k] == 1) { 46 | n_pos[i] = n; 47 | k_pos[i] = k; 48 | i = i + 1; 49 | } else { 50 | n_neg[j] = n; 51 | k_neg[j] = k; 52 | j = j + 1; 53 | } 54 | } 55 | } 56 | for (n in (N1+1):(N1+N2)) { //positions in observed y2 57 | for (k in (K1+1):(K1+K2+Kb)) { 58 | if (y[n,k] == 1) { 59 | n_pos[i] = n; 60 | k_pos[i] = k; 61 | i = i + 1; 62 | } else { 63 | n_neg[j] = n; 64 | k_neg[j] = k; 65 | j = j + 1; 66 | } 67 | } 68 | } 69 | } 70 | } 71 | parameters { 72 | vector[K1 + K2 + Kb] mu; 73 | corr_matrix[K1 + K2 + Kb] Omega; 74 | vector[N_pos] z_pos; 75 | vector[N_neg] z_neg; 76 | vector[K2] z1mis[N1]; 77 | vector[K1] z2mis[N2]; 78 | } 79 | transformed parameters{ 80 | vector[K1 + K2 + Kb] z[N1 + N2]; 81 | vector[K2] y1mis[N1]; 82 | vector[K1] y2mis[N2]; 83 | for (i in 1:N_pos) 84 | z[n_pos[i], k_pos[i]] = z_pos[i]; 85 | for (i in 1:N_neg) 86 | z[n_neg[i], k_neg[i]] = z_neg[i]; 87 | for (n in 1:N1) { 88 | for (k in 1:K2) { 89 | z[n, K1 + k] = z1mis[n, k]; 90 | if (z1mis[n, k] > 0) 91 | y1mis[n, k] = 1; 92 | if (z1mis[n, k] < 0) 93 | y1mis[n, k] = 0; 94 | } 95 | } 96 | for (n in 1:N2) { 97 | for (k in 1:K1) { 98 | z[N1 + n, k] = z2mis[n, k]; 99 | if (z2mis[n, k] > 0) 100 | y2mis[n, k] = 1; 101 | if (z2mis[n, k] < 0) 102 | y2mis[n, k] = 0; 103 | } 104 | } 105 | } 106 | model { 107 | mu ~ normal(0, 3); 108 | Omega ~ lkj_corr(1); 109 | z ~ multi_normal(mu, Omega); 110 | } 111 | 112 | -------------------------------------------------------------------------------- /6_about_stan/about_stan.Rmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Stan Tutorial" 3 | subtitle: "About Stan" 4 | author: "Kevin Van Horn and Elea McDonnell Feit" 5 | date: "June 25, 2017" 6 | output: ioslides_presentation 7 | --- 8 | 9 | ```{r setup, include=FALSE} 10 | knitr::opts_chunk$set(echo = FALSE) 11 | ``` 12 | 13 | # Some other things you ought to know about Stan 14 | 15 | ## Hamiltonian Monte Carlo 16 | - To produce draws from the posterior distribution, Stan uses a variation of **Hamiltonian Monte Carlo (HMC)** called the **No U-Turn Sampler (NUTS)**. 17 | - HMC is a variation on the Metropolis-Hastings (M-H) algorithm, but it uses the gradient of the distribution to more quickly explore the bulk of the posterior. 18 | - NUTS further improves on HMC by automating tuning of the HMC algorithm. 19 | - For an overview of HMC see the [MCMC Handbook Chapter by Radford Neal](http://www.mcmchandbook.net/HandbookChapter5.pdf) and for more on NUTS see [Hoffman and Gelman](http://www.stat.columbia.edu/~gelman/research/published/nuts.pdf). 20 | 21 | ## There are other ways to interface with Stan 22 | - RStan (R) 23 | - PyStan (Python) 24 | - CmdStan (shell, command-line terminal) 25 | - MatlabStan (MATLAB) 26 | - Stan.jl (Julia) 27 | - StataStan (Stata) 28 | - MathematicaStan (Mathematica) 29 | 30 | ## Stan supports other approches to inference 31 | - Hamiltonian Monte Carlo / NUTS 32 | + Default algorithm we have used in this tutorial 33 | - Variational Bayes 34 | + A fast approximation to full Bayesian inference 35 | + Somewhat experimental in Stan 36 | - "Optimization" 37 | + Routines that find point estimates of parameters rather than the full posterior, e.g. maximum likelihood (ML) or maximum a posteriori (MAP) 38 | 39 | ## You can specify the likelihood directly in Stan 40 | Advanced Stan users sometimes find it easier to specify the full likelihood of the model directly rather than using the sampling statements we have shown here. 41 | ```{stan, eval=FALSE, echo=TRUE, output.var="dummy"} 42 | y ~ normal(mu, sigma); 43 | ``` 44 | is equivalent to 45 | ```{stan,eval=FALSE, echo=TRUE, output.var="dummy"} 46 | target += normal_lpdf(y | mu, sigma); 47 | ``` 48 | Examples of when this is handy are beyond the scope of this tutorial. 49 | 50 | ## Useful R packages for working with Stan 51 | - rstan 52 | - coda 53 | + tools for summarizing draws 54 | - bayesplot 55 | + tools for summarizing draws 56 | - ShinyStan 57 | + a GUI for exploring posterior draws 58 | - RStanArm 59 | + Provides an "R-like" interface with formula notation for Stan 60 | 61 | # Learning more about Stan 62 | 63 | ## Where can you find help with Stan? 64 | - The [Stan forum](http://discourse.mc-stan.org/) is very active and the Stan development team often provides answers to questions. 65 | + A Google search like "multinomial logit in Stan" will often lead you to the forums. 66 | - Keep in mind that Stan develops over time, so you want to check the date of any post. 67 | 68 | ## Stan examples 69 | - The Stan website maintains a library of Stan examples. 70 | - We often begin developing a new model by finding an example that is close to what we want and then modifying it. 71 | 72 | ## Popular Stan conferences 73 | - StanCon held annually (StanCon 2017 Notebooks)[http://mc-stan.org/events/stancon]) and is a great place to learn new ways to use Stan. 74 | - Most presenters provide code. 75 | 76 | ## Who is responsible for Stan? 77 | - Stan is maintained by the [Stan development team](http://mc-stan.org/about/team/index.html), which consistes of consists of volunteers from around the world. 78 | - Key players include Andrew Gelman and Bob Carpenter, but they have been joined by many others. 79 | 80 | ## Books about Stan 81 | - [Kruschke, Doing Bayesian Data Analysis, Second Edition: A Tutorial with R, JAGS, and Stan](https://sites.google.com/site/doingbayesiandataanalysis/) 82 | - [McElreath, Statistical Rethinking: A Bayesian Course with Examples in R and Stan](http://xcelab.net/rm/statistical-rethinking/) 83 | - [Gelman and Hill, Applied Regression Modeling](http://www.stat.columbia.edu/~gelman/arm/) 84 | + Focuses on the application of Stan models to social science problems 85 | - [Gelman et al., Bayesian Data Analysis](http://www.stat.columbia.edu/~gelman/book/) 86 | + Excellent reference if you want to learn Bayesian theory and how Bayesian algorithms work 87 | 88 | 89 | 90 | 91 | 92 | 93 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ART-Forum-2017-Stan-Tutorial 2 | Materials from tutorial "Using Stan to Estimate Hierarchical Bayes Models," ART Forum 2017 3 | -------------------------------------------------------------------------------- /demo/demo.Rmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Have you met Stan?" 3 | author: "Kevin Van Horn (Adobe) and Elea McDonnell Feit (Drexel)" 4 | date: "June 26, 2017" 5 | output: ioslides_presentation 6 | --- 7 | 8 | ## What is Stan? 9 | > Stan is freedom-respecting open-source software for facilitating statistical inference at the frontiers of applied statistics. 10 | 11 |
-- Stan homepage
12 | 13 | ## Software for Bayesian inference 14 | - Write-your-own sampler 15 | - Purpose built routines for individual models 16 | + MCMCPack or bayesm in R 17 | + Stata's Bayesian package 18 | + Sawtooth CBC/HB 19 | - Tools that automatically build samplers for a wide class of models 20 | + WinBugs 21 | + JAGS 22 | + **Stan** 23 | 24 | # Fitting a hierarchical Bayes choice model with Stan 25 | 26 | ## Defining a model in Stan 27 | - When you fit a model in Stan, you have to define the model using Stan's syntax. 28 | - We don't have time to teach you the syntax, but we'll give you a quick peek at what a hierarchical Bayes multinomail logit model looks like. 29 | - The is model is very similar to that estimated in Sawtooth CBC/HB and bayesm() 30 | 31 | ## Declare `data` 32 | ```{stan, eval=FALSE, echo=TRUE, output.var="dummy"} 33 | data { 34 | int C; // # of alternatives (choices) in each scenario 35 | int K; // # of covariates of alternatives 36 | int R; // # of respondents 37 | int S; // # of scenarios per respondent 38 | int G; // # of respondent covariates 39 | int Y[R, S]; // observed choices 40 | matrix[C, K] X[R, S]; // matrix of attributes for each obs 41 | matrix[G, R] Z; // vector of covariates for each respondent 42 | } 43 | ``` 44 | 45 | ## Declare `parameters` 46 | ```{stan, eval=FALSE, echo=TRUE, output.var="dummy"} 47 | parameters { 48 | matrix[K, R] Beta; 49 | matrix[K, G] Theta; 50 | corr_matrix[K] Omega; 51 | vector[K] tau; 52 | } 53 | ``` 54 | 55 | ## Declare `transformed parameters` 56 | ```{stan, eval=FALSE, echo=TRUE, output.var="dummy"} 57 | transformed parameters { 58 | cov_matrix[K] Sigma = quad_form_diag(Omega, tau); 59 | } 60 | ``` 61 | 62 | ## Declare `model` 63 | ```{stan, eval=FALSE, echo=TRUE, output.var="dummy"} 64 | model { 65 | //priors 66 | to_vector(Theta) ~ normal(0, 10); 67 | tau ~ cauchy(0, 2.5); 68 | Omega ~ lkj_corr(2); 69 | //likelihood 70 | for (r in 1:R) { 71 | Beta[,r] ~ multi_normal(Theta*Z[,r], Sigma); 72 | for (s in 1:S) 73 | Y[r,s] ~ categorical_logit(X[r,s]*Beta[,r]); 74 | } 75 | } 76 | ``` 77 | 78 | ## Choice-based conjoint data for chocolate bars 79 | - Fourteen respondents each answered 25 choice tasks where they selected from among three chocolate bars. 80 | - Three attributes 81 | + Brand: Hersheys, Dove, Lindt, Godiva, Ghirardelli 82 | + Type: Milk, Milk with nuts, Dark, Dark with nuts, White 83 | 84 | ```{r, echo=FALSE, eval=TRUE} 85 | rm(list=ls()) # tidy up 86 | choc.df <- read.csv("cbc_chocolate.csv") 87 | choc.contrasts <- list(Brand = "contr.sum", Type = "contr.sum") 88 | choc.coded <- model.matrix(~ Brand + Type, data = choc.df, 89 | contrasts = choc.contrasts) 90 | choc.coded <- choc.coded[,2:ncol(choc.coded)] # remove intercept 91 | # Fix the bad labels from contr.sum 92 | choc.names <- c("BrandDove", "BrandGhirardelli", "BrandGodiva", 93 | "BrandHersheys", "TypeDark", "TypeDarkNuts", 94 | "TypeMilk", "TypeMilkNuts") 95 | colnames(choc.coded) <- choc.names 96 | choc.df <- cbind(choc.df, choc.coded) 97 | R <- length(unique(choc.df$Ind)) 98 | S <- length(unique(choc.df$Trial)) 99 | C <- max(choc.df$Alt) 100 | K <- 9 101 | Y <- array(dim=c(R, S)) 102 | X <- array(rnorm(R*S*C*K), dim=c(R, S, C, K)) 103 | Z <- array(1, dim=c(1, R)) # intercept only 104 | for (r in 1:R) { # respondents 105 | for (s in 1:S){ # choice scenarios 106 | scenario <- choc.df[choc.df$Ind==unique(choc.df$Ind)[r] & 107 | choc.df$Trial==unique(choc.df$Trial)[s], ] 108 | X[r,s,,] <- data.matrix(scenario[,c(7, 9:16)]) 109 | Y[r,s] <- scenario$Alt[as.logical(scenario$Chosen)] 110 | } 111 | } 112 | choc.standata <- list(C=C, K=K, R=R, S=S, G=1, Y=Y, X=X, Z=Z) 113 | rm(Y, X, Z, R, S, r, s, C, K, choc.contrasts, scenario, choc.coded) 114 | ``` 115 | 116 | ## Chocolate data in R 117 | Data is formatted as a list with elements the same as those declared in the `data` block in the Stan model. 118 | ```{R} 119 | str(choc.standata) 120 | ``` 121 | 122 | ## Bayesian inference 123 | ![](images/Bayesian_Inference.png) 124 | 125 | ## Call the `stan()` function 126 | ```{R} 127 | library(rstan) 128 | choc.stan <- stan(file="hmnl.stan", data=choc.standata) 129 | ``` 130 | 131 | ## Parameter estimates (1) 132 | ```{r, message=FALSE} 133 | plot(choc.stan, pars=c("Theta", "tau")) 134 | ``` 135 | 136 | ## Parameter estimates (2) 137 | ```{r, message=FALSE} 138 | plot(choc.stan, pars=paste("Beta[", 1:9, ",1]", sep="")) + 139 | ggtitle("Respondent 1: Likes Milk Chocolate") 140 | ``` 141 | 142 | ## Parameter estimates (3) 143 | ```{r, message=FALSE} 144 | plot(choc.stan, pars=paste("Beta[", 1:9, ",2]", sep="")) + 145 | ggtitle("Respondent 2: Likes Dark Chocolate") 146 | ``` 147 | 148 | ## Simulating shares 149 | - You can write a simulator in R to simulate shares. 150 | - If you want a user interface for the simulator, try Shiny. 151 | 152 | # Fitting other models in Stan 153 | 154 | ## Fitting other models in Stan 155 | - The real power of Stan is that you can change the model to whatever you want. 156 | - We have used Stan to fit: 157 | + Alternative priors (conjugacy isn't required!) 158 | + Hierarchcal nested logit model for dual choice 159 | + Dynamic linear models 160 | + Missing data models 161 | 162 | ## Problem: Shares in dual choice model 163 | Predicted shares for no-choice option are much less than predicted. 164 | 165 | ![](images/barchart2.png) 166 | 167 | ## HB nested logit Stan model 168 | ```{stan, eval=FALSE, echo=TRUE, output.var="dummy"} 169 | model { 170 | lambda ~ uniform(0, 1); 171 | sigma ~ normal(0, 5); 172 | to_vector(Theta) ~ normal(0, 10); 173 | to_vector(Epsilon) ~ normal(0, 1); 174 | matrix[K+1, R] Beta = 175 | Theta * Z + diag_pre_multiply(sigma, Epsilon); 176 | for (r in 1:R) { 177 | vector[K] b = Beta[2:(K+1), r]; 178 | real alpha = Beta[1,r]; 179 | for (s in 1:S) { 180 | vector[C] u = X[r,s] * b; 181 | real u_buy = alpha + lambda * log_sum_exp(u); 182 | Y1[r,s] ~ categorical_logit(u); 183 | Y2[r,s] ~ bernoulli_logit(u_buy); 184 | } 185 | } 186 | } 187 | ``` 188 | 189 | ## Problem solved 190 | The nested logit model produces good predictions for the no-choice option in holdout tasks. 191 | 192 | ![](images/hnestedmnl_predictions.PNG) 193 | 194 | And building this model in Stan took hours, not days or weeks. 195 | 196 | # A few more things you should know about Stan 197 | 198 | ## Who is responsible for Stan? 199 | - Stan is maintained by the [Stan development team](http://mc-stan.org/about/team/index.html), which consistes of consists of volunteers from around the world. 200 | - Key players include Andrew Gelman, Bob Carpenter, Michael Betancourt, but they have been joined by many others. 201 | 202 | ## There are many ways to interface with Stan 203 | - **RStan (R)** 204 | - PyStan (Python) 205 | - CmdStan (shell, command-line terminal) 206 | - MatlabStan (MATLAB) 207 | - Stan.jl (Julia) 208 | - StataStan (Stata) 209 | - MathematicaStan (Mathematica) 210 | 211 | ## Stan supports multiple approches to inference 212 | - Hamiltonian Monte Carlo / NUTS 213 | + Robust algorithm for Bayesian inference 214 | - Variational Bayes 215 | + A fast approximation to full Bayesian inference 216 | + Somewhat experimental in Stan 217 | - "Optimization" 218 | + Routines that find point estimates of parameters rather than the full posterior, e.g. maximum likelihood (ML) or maximum a posteriori (MAP) 219 | 220 | 221 | ## Why Stan? 222 | - Ideal tool for rapid prototyping of new models 223 | - Robust and fast sampler 224 | - Modern, thoughful software design 225 | - Increasingly attractive option for fitting even standard Bayesian models 226 | 227 | # Thank you! 228 | Thanks to The Modellers for sharing the code for the hierarchical nested logit model. Thanks to Adobe for their support. -------------------------------------------------------------------------------- /demo/hmnl.stan: -------------------------------------------------------------------------------- 1 | # Hierarchical multinomial logit model 2 | # Elea McDonnell Feit, eleafeit@gmail.com 3 | 4 | data { 5 | int C; // # of alternatives (choices) in each scenario 6 | int K; // # of covariates of alternatives 7 | int R; // # of respondents 8 | int S; // # of scenarios per respondent 9 | int G; // # of respondent covariates 10 | int Y[R, S]; // observed choices 11 | matrix[C, K] X[R, S]; // matrix of attributes for each obs 12 | matrix[G, R] Z; // vector of covariates for each respondent 13 | } 14 | 15 | parameters { 16 | matrix[K, R] Beta; 17 | matrix[K, G] Theta; 18 | corr_matrix[K] Omega; 19 | vector[K] tau; 20 | } 21 | transformed parameters { 22 | cov_matrix[K] Sigma = quad_form_diag(Omega, tau); 23 | } 24 | model { 25 | //priors 26 | to_vector(Theta) ~ normal(0, 10); 27 | tau ~ cauchy(0, 2.5); 28 | Omega ~ lkj_corr(2); 29 | //likelihood 30 | for (r in 1:R) { 31 | Beta[,r] ~ multi_normal(Theta*Z[,r], Sigma); 32 | for (s in 1:S) 33 | Y[r,s] ~ categorical_logit(X[r,s]*Beta[,r]); 34 | } 35 | } 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /demo/images/Bayesian_Inference.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/demo/images/Bayesian_Inference.png -------------------------------------------------------------------------------- /demo/images/barchart1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/demo/images/barchart1.png -------------------------------------------------------------------------------- /demo/images/barchart2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/demo/images/barchart2.png -------------------------------------------------------------------------------- /demo/images/hnestedmnl_predictions.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ksvanhorn/ART-Forum-2017-Stan-Tutorial/5d42fc55acd2028eb0f0d0181c8c9c7dce819142/demo/images/hnestedmnl_predictions.PNG --------------------------------------------------------------------------------