├── CatBoost_Starter.ipynb
├── Keras Starter - Binary Classification.ipynb
├── LightGBM_Script.ipynb
├── README.md
├── lgb_starter.R
├── rank1_tinku
└── lgb_script.R
├── rank2_munish
└── boost_final.py
└── stackingClicks.R
/CatBoost_Starter.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "metadata": {},
6 | "source": [
7 | "This script will introduce you to use catboost package in python.
\n",
8 | "It uses less than 10% of the train data for model training.
\n",
9 | "This script scores ~0.67 on public LB."
10 | ]
11 | },
12 | {
13 | "cell_type": "code",
14 | "execution_count": 1,
15 | "metadata": {
16 | "collapsed": true
17 | },
18 | "outputs": [],
19 | "source": [
20 | "import pandas as pd\n",
21 | "import numpy as np\n",
22 | "from catboost import CatBoostClassifier\n",
23 | "from sklearn.model_selection import train_test_split"
24 | ]
25 | },
26 | {
27 | "cell_type": "markdown",
28 | "metadata": {},
29 | "source": [
30 | "You can install this package using: `pip install catboost`"
31 | ]
32 | },
33 | {
34 | "cell_type": "markdown",
35 | "metadata": {},
36 | "source": [
37 | "Catboost package has been recently open-source by Yandex. It can handle categorical variables in the original form. Lets see how good this package performs on this dataset which has mostly categorical variables.\n",
38 | "\n",
39 | "In R, catboost has been integerated with caret package. So, it's quite easier to use in R as well.\n",
40 | "\n",
41 | "You can find more information here: [CatBoost](https://tech.yandex.com/catboost/)"
42 | ]
43 | },
44 | {
45 | "cell_type": "code",
46 | "execution_count": 2,
47 | "metadata": {
48 | "collapsed": true
49 | },
50 | "outputs": [],
51 | "source": [
52 | "train = pd.read_csv(\"train.csv\")\n",
53 | "test = pd.read_csv(\"test.csv\")"
54 | ]
55 | },
56 | {
57 | "cell_type": "code",
58 | "execution_count": 17,
59 | "metadata": {
60 | "collapsed": false
61 | },
62 | "outputs": [
63 | {
64 | "data": {
65 | "text/html": [
66 | "\n",
67 | "
\n",
68 | " \n",
69 | " \n",
70 | " | \n",
71 | " ID | \n",
72 | " datetime | \n",
73 | " siteid | \n",
74 | " offerid | \n",
75 | " category | \n",
76 | " merchant | \n",
77 | " countrycode | \n",
78 | " browserid | \n",
79 | " devid | \n",
80 | " click | \n",
81 | "
\n",
82 | " \n",
83 | " \n",
84 | " \n",
85 | " | 0 | \n",
86 | " IDsrk7SoW | \n",
87 | " 2017-01-14 09:42:09 | \n",
88 | " 4709696.0 | \n",
89 | " 887235 | \n",
90 | " 17714 | \n",
91 | " 20301556 | \n",
92 | " e | \n",
93 | " Firefox | \n",
94 | " NaN | \n",
95 | " 0 | \n",
96 | "
\n",
97 | " \n",
98 | " | 1 | \n",
99 | " IDmMSxHur | \n",
100 | " 2017-01-18 17:50:53 | \n",
101 | " 5189467.0 | \n",
102 | " 178235 | \n",
103 | " 21407 | \n",
104 | " 9434818 | \n",
105 | " b | \n",
106 | " Mozilla Firefox | \n",
107 | " Desktop | \n",
108 | " 0 | \n",
109 | "
\n",
110 | " \n",
111 | " | 2 | \n",
112 | " IDVLNN0Ut | \n",
113 | " 2017-01-11 12:46:49 | \n",
114 | " 98480.0 | \n",
115 | " 518539 | \n",
116 | " 25085 | \n",
117 | " 2050923 | \n",
118 | " a | \n",
119 | " Edge | \n",
120 | " NaN | \n",
121 | " 0 | \n",
122 | "
\n",
123 | " \n",
124 | " | 3 | \n",
125 | " ID32T6wwQ | \n",
126 | " 2017-01-17 10:18:43 | \n",
127 | " 8896401.0 | \n",
128 | " 390352 | \n",
129 | " 40339 | \n",
130 | " 72089744 | \n",
131 | " c | \n",
132 | " Firefox | \n",
133 | " Mobile | \n",
134 | " 0 | \n",
135 | "
\n",
136 | " \n",
137 | " | 4 | \n",
138 | " IDqUShzMg | \n",
139 | " 2017-01-14 16:02:33 | \n",
140 | " 5635120.0 | \n",
141 | " 472937 | \n",
142 | " 12052 | \n",
143 | " 39507200 | \n",
144 | " d | \n",
145 | " Mozilla Firefox | \n",
146 | " Desktop | \n",
147 | " 0 | \n",
148 | "
\n",
149 | " \n",
150 | "
\n",
151 | "
\n",
189 | "
\n",
190 | " \n",
191 | " \n",
192 | " | \n",
193 | " ID | \n",
194 | " datetime | \n",
195 | " siteid | \n",
196 | " offerid | \n",
197 | " category | \n",
198 | " merchant | \n",
199 | " countrycode | \n",
200 | " browserid | \n",
201 | " devid | \n",
202 | "
\n",
203 | " \n",
204 | " \n",
205 | " \n",
206 | " | 0 | \n",
207 | " IDFDJVI | \n",
208 | " 2017-01-22 09:55:48 | \n",
209 | " 755610.0 | \n",
210 | " 808980 | \n",
211 | " 17714 | \n",
212 | " 26391770 | \n",
213 | " b | \n",
214 | " Mozilla Firefox | \n",
215 | " Desktop | \n",
216 | "
\n",
217 | " \n",
218 | " | 1 | \n",
219 | " IDNWkTQ | \n",
220 | " 2017-01-22 03:54:39 | \n",
221 | " 3714899.0 | \n",
222 | " 280355 | \n",
223 | " 12052 | \n",
224 | " 39507200 | \n",
225 | " b | \n",
226 | " Edge | \n",
227 | " Tablet | \n",
228 | "
\n",
229 | " \n",
230 | " | 2 | \n",
231 | " ID9pRmM | \n",
232 | " 2017-01-21 10:25:50 | \n",
233 | " 4378333.0 | \n",
234 | " 930819 | \n",
235 | " 30580 | \n",
236 | " 46148550 | \n",
237 | " e | \n",
238 | " Mozilla Firefox | \n",
239 | " NaN | \n",
240 | "
\n",
241 | " \n",
242 | " | 3 | \n",
243 | " IDHaQaj | \n",
244 | " 2017-01-22 14:45:53 | \n",
245 | " 1754730.0 | \n",
246 | " 612234 | \n",
247 | " 11837 | \n",
248 | " 8837581 | \n",
249 | " b | \n",
250 | " Edge | \n",
251 | " Tablet | \n",
252 | "
\n",
253 | " \n",
254 | " | 4 | \n",
255 | " IDT2CrF | \n",
256 | " 2017-01-22 09:34:07 | \n",
257 | " 5299909.0 | \n",
258 | " 524289 | \n",
259 | " 45620 | \n",
260 | " 31388981 | \n",
261 | " b | \n",
262 | " Mozilla | \n",
263 | " NaN | \n",
264 | "
\n",
265 | " \n",
266 | "
\n",
267 | "
\n",
8 | "Majorly, following are the steps:\n",
9 | "\n",
10 | "1. Scale train and test data.\n",
11 | "2. Encode the target variables (one-hot)\n",
12 | "3. Setup the model architechture\n",
13 | "4. Train the model\n",
14 | "5. Predict\n"
15 | ]
16 | },
17 | {
18 | "cell_type": "code",
19 | "execution_count": 1,
20 | "metadata": {
21 | "collapsed": true
22 | },
23 | "outputs": [],
24 | "source": [
25 | "import numpy as np\n",
26 | "import pandas as pd\n",
27 | "\n",
28 | "from keras.models import Sequential\n",
29 | "from keras.layers import Dense\n",
30 | "from keras.utils import to_categorical\n",
31 | "from keras.callbacks import EarlyStopping\n",
32 | "\n",
33 | "from sklearn.model_selection import train_test_split\n",
34 | "from sklearn.preprocessing import StandardScaler"
35 | ]
36 | },
37 | {
38 | "cell_type": "code",
39 | "execution_count": 2,
40 | "metadata": {
41 | "collapsed": true
42 | },
43 | "outputs": [],
44 | "source": [
45 | "train = pd.read_csv('train.csv')\n",
46 | "test = pd.read_csv('test.csv')"
47 | ]
48 | },
49 | {
50 | "cell_type": "code",
51 | "execution_count": 3,
52 | "metadata": {
53 | "collapsed": false
54 | },
55 | "outputs": [
56 | {
57 | "name": "stdout",
58 | "output_type": "stream",
59 | "text": [
60 | "The train data has 12137810 rows and 10 columns\n",
61 | "The test data has 3706907 rows and 9 columns\n"
62 | ]
63 | }
64 | ],
65 | "source": [
66 | "print ('The train data has {} rows and {} columns'.format(train.shape[0],train.shape[1]))\n",
67 | "print ('The test data has {} rows and {} columns'.format(test.shape[0],test.shape[1]))"
68 | ]
69 | },
70 | {
71 | "cell_type": "code",
72 | "execution_count": 4,
73 | "metadata": {
74 | "collapsed": false
75 | },
76 | "outputs": [
77 | {
78 | "data": {
79 | "text/html": [
80 | "\n",
81 | "
\n",
82 | " \n",
83 | " \n",
84 | " | \n",
85 | " ID | \n",
86 | " datetime | \n",
87 | " siteid | \n",
88 | " offerid | \n",
89 | " category | \n",
90 | " merchant | \n",
91 | " countrycode | \n",
92 | " browserid | \n",
93 | " devid | \n",
94 | " click | \n",
95 | "
\n",
96 | " \n",
97 | " \n",
98 | " \n",
99 | " | 0 | \n",
100 | " IDsrk7SoW | \n",
101 | " 2017-01-14 09:42:09 | \n",
102 | " 4709696.0 | \n",
103 | " 887235 | \n",
104 | " 17714 | \n",
105 | " 20301556 | \n",
106 | " e | \n",
107 | " Firefox | \n",
108 | " NaN | \n",
109 | " 0 | \n",
110 | "
\n",
111 | " \n",
112 | " | 1 | \n",
113 | " IDmMSxHur | \n",
114 | " 2017-01-18 17:50:53 | \n",
115 | " 5189467.0 | \n",
116 | " 178235 | \n",
117 | " 21407 | \n",
118 | " 9434818 | \n",
119 | " b | \n",
120 | " Mozilla Firefox | \n",
121 | " Desktop | \n",
122 | " 0 | \n",
123 | "
\n",
124 | " \n",
125 | " | 2 | \n",
126 | " IDVLNN0Ut | \n",
127 | " 2017-01-11 12:46:49 | \n",
128 | " 98480.0 | \n",
129 | " 518539 | \n",
130 | " 25085 | \n",
131 | " 2050923 | \n",
132 | " a | \n",
133 | " Edge | \n",
134 | " NaN | \n",
135 | " 0 | \n",
136 | "
\n",
137 | " \n",
138 | " | 3 | \n",
139 | " ID32T6wwQ | \n",
140 | " 2017-01-17 10:18:43 | \n",
141 | " 8896401.0 | \n",
142 | " 390352 | \n",
143 | " 40339 | \n",
144 | " 72089744 | \n",
145 | " c | \n",
146 | " Firefox | \n",
147 | " Mobile | \n",
148 | " 0 | \n",
149 | "
\n",
150 | " \n",
151 | " | 4 | \n",
152 | " IDqUShzMg | \n",
153 | " 2017-01-14 16:02:33 | \n",
154 | " 5635120.0 | \n",
155 | " 472937 | \n",
156 | " 12052 | \n",
157 | " 39507200 | \n",
158 | " d | \n",
159 | " Mozilla Firefox | \n",
160 | " Desktop | \n",
161 | " 0 | \n",
162 | "
\n",
163 | " \n",
164 | "
\n",
165 | "
\n",
186 | "
\n",
187 | " \n",
188 | " \n",
189 | " | \n",
190 | " ID | \n",
191 | " datetime | \n",
192 | " siteid | \n",
193 | " offerid | \n",
194 | " category | \n",
195 | " merchant | \n",
196 | " countrycode | \n",
197 | " browserid | \n",
198 | " devid | \n",
199 | " click | \n",
200 | " tweekday | \n",
201 | " thour | \n",
202 | " tminute | \n",
203 | "
\n",
204 | " \n",
205 | " \n",
206 | " \n",
207 | " | 0 | \n",
208 | " IDsrk7SoW | \n",
209 | " 2017-01-14 09:42:09 | \n",
210 | " 128865 | \n",
211 | " 784773 | \n",
212 | " 48 | \n",
213 | " 127 | \n",
214 | " 4 | \n",
215 | " 2 | \n",
216 | " 2 | \n",
217 | " 0 | \n",
218 | " 5 | \n",
219 | " 9 | \n",
220 | " 42 | \n",
221 | "
\n",
222 | " \n",
223 | " | 1 | \n",
224 | " IDmMSxHur | \n",
225 | " 2017-01-18 17:50:53 | \n",
226 | " 142053 | \n",
227 | " 157563 | \n",
228 | " 59 | \n",
229 | " 65 | \n",
230 | " 1 | \n",
231 | " 8 | \n",
232 | " 0 | \n",
233 | " 0 | \n",
234 | " 2 | \n",
235 | " 17 | \n",
236 | " 50 | \n",
237 | "
\n",
238 | " \n",
239 | " | 2 | \n",
240 | " IDVLNN0Ut | \n",
241 | " 2017-01-11 12:46:49 | \n",
242 | " 2618 | \n",
243 | " 458279 | \n",
244 | " 69 | \n",
245 | " 15 | \n",
246 | " 0 | \n",
247 | " 1 | \n",
248 | " 2 | \n",
249 | " 0 | \n",
250 | " 2 | \n",
251 | " 12 | \n",
252 | " 46 | \n",
253 | "
\n",
254 | " \n",
255 | " | 3 | \n",
256 | " ID32T6wwQ | \n",
257 | " 2017-01-17 10:18:43 | \n",
258 | " 243406 | \n",
259 | " 345067 | \n",
260 | " 117 | \n",
261 | " 507 | \n",
262 | " 2 | \n",
263 | " 2 | \n",
264 | " 1 | \n",
265 | " 0 | \n",
266 | " 1 | \n",
267 | " 10 | \n",
268 | " 18 | \n",
269 | "
\n",
270 | " \n",
271 | " | 4 | \n",
272 | " IDqUShzMg | \n",
273 | " 2017-01-14 16:02:33 | \n",
274 | " 154278 | \n",
275 | " 417948 | \n",
276 | " 36 | \n",
277 | " 276 | \n",
278 | " 3 | \n",
279 | " 8 | \n",
280 | " 0 | \n",
281 | " 0 | \n",
282 | " 5 | \n",
283 | " 16 | \n",
284 | " 2 | \n",
285 | "
\n",
286 | " \n",
287 | "
\n",
288 | "
4 | This repository contains starter scripts for participants to provide a quick sneak-peak into data and model training.
5 | In this competition, the participants have to predict whether an ad will get clicked or not.
6 |
7 | **Competition Duration:** 22nd July 2017 to 14th August 2017
8 | **Evaluation Metric:** AUC-ROC
9 | **Competition Prize:** $700 (1st Rank), $500 (2nd Rank)
10 | **Competition Link:** [Predict Ad Click](https://www.hackerearth.com/challenge/competitive/machine-learning-challenge-3/problems/)
11 |
12 |
13 |
--------------------------------------------------------------------------------
/lgb_starter.R:
--------------------------------------------------------------------------------
1 | path <- #set path
2 | setwd(path)
3 |
4 | # load libraries
5 |
6 | library(data.table)
7 | library(lubridate)
8 | library(lightgbm)
9 |
10 | # load data
11 |
12 | train <- fread("train.csv", na.strings = c(" ","",NA))
13 | test <- fread("test.csv",na.strings = c(" ","",NA))
14 |
15 | # check target distribution
16 | # In train data, only 3.6% of the ads got clicked
17 | train[,.N/nrow(train),click]
18 |
19 | # check time range
20 | # The train data set contains information for 10 days from 10th Jan 2017 to 20th Jan 2017
21 | train[,range(datetime)]
22 |
23 | # check missing values
24 | # siteid has around 9% missing values, browserid has around 5%, dev id has around ~14% missing values
25 | sapply(train, function(x)(sum(is.na(x))/length(x)))
26 |
27 | # check ID variables
28 | # This shows train data has more number of sites, offers, categories and merchants.
29 | for(x in c('siteid','offerid','category','merchant'))
30 | {
31 | print(sprintf("There are %d unique %s in the train set and %d unique %s in the test set",train[,uniqueN(get(x))], x,test[,uniqueN(get(x))], x))
32 | }
33 |
34 | # Since ID variable have no ordered in them, we can't use them as is.
35 | # One possible option is replace them with their counts
36 | # let's check their class first
37 |
38 | sapply(train, class)
39 | sapply(test, class)
40 |
41 | # convert ID variables to their respective count
42 | # using for-set for faster speed
43 | cols1 <- c('siteid','offerid','category','merchant')
44 | for(x in seq(cols))
45 | {
46 | train[,eval(cols[x]) := .N, by = eval(cols[x])]
47 | test[,eval(cols[x]) := .N, by=eval(cols[x])]
48 | }
49 |
50 | # converting other variables to count
51 | # here we need to convert character class values to integer to show count value under the same column
52 | # otherwise we'll have to create new columns for these respective features and remove the old ones.
53 | cols2 <- c('countrycode','browserid','devid')
54 | for(x in seq(cols))
55 | {
56 | train[, eval(cols[x]) := as.integer(as.factor(get(cols[x])))][,eval(cols[x]) := .N, by = eval(cols[x])]
57 | test[, eval(cols[x]) := as.integer(as.factor(get(cols[x])))][,eval(cols[x]) := .N, by = eval(cols[x])]
58 | }
59 |
60 |
61 | # Note: You might see different values such as Mozilla Firefox, Firefox, Mozilla etc.
62 | # You might like to club them into one value ( but look for their respective device)
63 |
64 |
65 | # Extract features from datetime variable
66 | # datetime is the timestamp at which ad got live on web
67 | train[,datetime := as.POSIXct(datetime, format = "%Y-%m-%d %H:%M:%S")]
68 | test[,datetime := as.POSIXct(datetime, format = "%Y-%m-%d %H:%M:%S")]
69 |
70 | train[,tweekday := as.integer(as.factor(weekdays(datetime)))-1]
71 | train[,thour := hour(datetime)]
72 | train[,tminute := minute(datetime)]
73 |
74 | test[,tweekday := as.integer(as.factor(weekdays(datetime)))-1]
75 | test[,thour := hour(datetime)]
76 | test[,tminute := minute(datetime)]
77 |
78 |
79 | # Model Training
80 | # using lightgbm for faster training than xgb
81 | # since data is large, instead of cross validation, we'll do hold out validation
82 | library(lightgbm)
83 | library(caret)
84 |
85 | train <- train[,.(siteid, offerid, category, merchant, countrycode, browserid, devid, tweekday, thour, tminute, click)]
86 | test <- test[,.(siteid, offerid, category, merchant, countrycode, browserid, devid, tweekday, thour, tminute)]
87 |
88 | folds <- createDataPartition(train$click, p = 0.5, list=F)
89 |
90 | trainX <- train[!folds]
91 | valX <- train[folds]
92 |
93 | lgb.trainX <- lgb.Dataset(as.matrix(trainX[,-c('click'),with=F]), label = trainX$click)
94 | lgb.valX <- lgb.Dataset(as.matrix(valX[,-c('click'),with=F]), label = valX$click)
95 |
96 | params <- list(
97 | objective = 'binary',
98 | metric = 'auc',
99 | feature_fraction = 0.7,
100 | bagging_fraction = 0.5,
101 | max_depth = 10
102 | )
103 |
104 | model <- lgb.train(params = params
105 | ,data = lgb.trainX
106 | ,valids = list(valid = lgb.valX)
107 | ,learning_rate = 0.1
108 | ,early_stopping_rounds = 40
109 | ,eval_freq = 20
110 | ,nrounds = 500
111 | )
112 |
113 | # get feature importance
114 | lgb.plot.importance(tree_imp = lgb.importance(model,percentage = TRUE))
115 |
116 | # make predictions
117 | preds <- predict(model, data = as.matrix(test), n = model$best_iter)
118 |
119 | # make submission
120 | sub <- data.table(ID = test$ID, click = preds)
121 | fwrite(sub, "lgb_starter.csv") #~ 0.6298
122 |
123 | # What to do next ?
124 | # 1. Tune the parameters
125 | # 2. Instead of count encoding, try doing label encoding (as.integer(as.factor(x))-1)
126 | # 3. Create more features
127 | # 3. Try different models and ensemble
128 |
129 |
130 |
131 |
--------------------------------------------------------------------------------
/rank1_tinku/lgb_script.R:
--------------------------------------------------------------------------------
1 | path <- "set_path"
2 | setwd(path)
3 |
4 | # load libraries
5 |
6 | library(data.table)
7 | library(lubridate)
8 | library(lightgbm)
9 | library(dummies)
10 | library(Matrix)
11 | library(xgboost)
12 |
13 | # load data
14 |
15 | train <- fread("train.csv", na.strings = c(" ","",NA))
16 | test <- fread("test.csv",na.strings = c(" ","",NA))
17 |
18 | testid = test$ID
19 | nrowtrain = nrow(train)
20 | test[, click := 5]
21 |
22 | combin = rbind(train, test)
23 | rm(train, test)
24 | combin[,ID := NULL]
25 | combin[,countrycode := as.integer(factor(countrycode))-1]
26 |
27 | combin[,browserid := ifelse(is.na(browserid) == TRUE, "Other", combin$browserid)]
28 | combin[,browserid := factor(browserid)]
29 | levels(combin$browserid)
30 | levels(combin$browserid) = c("Chrome","Edge","Firefox","Chrome","IE","IE",
31 | "IE","Firefox","Firefox","Opera","Other","Safari")
32 | combin[,browserid := as.integer(factor(browserid))-1]
33 |
34 | combin[,devid := ifelse(is.na(devid) == TRUE, "Other", combin$devid)]
35 | combin[,devid := as.integer(factor(devid))-1]
36 | str(combin)
37 |
38 | combin[,siteid := ifelse(is.na(siteid) == TRUE, -999, siteid)]
39 |
40 | combin[,datetime := as.POSIXct(datetime, format = "%Y-%m-%d %H:%M:%S")]
41 | combin[,tweekday := as.integer(as.factor(weekdays(datetime)))-1]
42 | combin[,thour := hour(datetime)]
43 |
44 | combin[,datetime := NULL]
45 |
46 | str(combin)
47 | combin[,siteid_count := .N, by = siteid]
48 | combin[,offerid_count := .N, by = offerid]
49 | combin[,category_count := .N, by = category]
50 | combin[,merchant_count := .N, by = merchant]
51 | combin[,siteid_offerid := .N, by = c("siteid", "offerid")]
52 | combin[,siteid_cate_mer := .N , by = c("siteid", "category", "merchant")]
53 |
54 | combin[,siteid_count := siteid_count/max(siteid_count)]
55 | combin[,offerid_count := offerid_count/max(offerid_count)]
56 | combin[,category_count := category_count/max(category_count)]
57 | combin[,merchant_count := merchant_count/max(merchant_count)]
58 | combin[,siteid_offerid := siteid_offerid/max(siteid_offerid)]
59 | combin[,siteid_cate_mer := siteid_cate_mer/max(siteid_cate_mer)]
60 |
61 |
62 | str(combin)
63 | combin[,siteid_countrycode := .N, by = c("siteid", "countrycode")]
64 | combin[,offerid_countrycode := .N, by = c("offerid", "countrycode")]
65 | combin[,category_countrycode := .N, by = c("category", "countrycode")]
66 | combin[,category_browserid := .N, by = c("category", "browserid")]
67 |
68 | combin[,merchant_countrycode := .N, by = c("merchant", "countrycode")]
69 | combin[,merchant_browserid := .N, by = c("merchant", "browserid")]
70 |
71 | combin[,siteid_countrycode := siteid_countrycode/max(siteid_countrycode)]
72 | combin[,offerid_countrycode := offerid_countrycode/max(offerid_countrycode)]
73 | combin[,merchant_countrycode := merchant_countrycode/max(merchant_countrycode)]
74 | combin[,category_countrycode := category_countrycode/max(category_countrycode)]
75 | combin[,merchant_browserid := merchant_browserid/max(merchant_browserid)]
76 | combin[,category_browserid := category_browserid/max(category_browserid)]
77 |
78 | str(combin)
79 | combin[,siteid_off_cat_coun := .N, by = c("siteid","offerid","category","countrycode")]
80 | combin[,siteid_off_cat_bro := .N, by = c("siteid","offerid","category","browserid")]
81 | combin[,siteid_off_cat_dev := .N, by = c("siteid","offerid","category","devid")]
82 |
83 | colnames(combin)
84 | combin[,c(5:25)] # or combin[,-c(1:4)]
85 | str(combin)
86 |
87 | traindata = combin[c(1:nrowtrain),]
88 | testdata = combin[-c(1:nrowtrain),]
89 | testdata[,click := NULL]
90 | rm(combin)
91 |
92 | dtrain = lgb.Dataset(data = as.matrix(traindata[,-c('click')]),label = traindata$click)
93 |
94 | bst11 <- lightgbm(data = dtrain,
95 | max_depth = 8,
96 | learning_rate = 0.2,
97 | nrounds = 70,
98 | objective = "binary",
99 | metric = "auc"
100 | )
101 |
102 |
103 | # imp1 = lgb.importance(bst11, percentage = T)
104 | # imp1
105 | # colnames(testdata)
106 | predd = predict(bst11, data = as.matrix(testdata))
107 | head(predd)
108 | sub3 <- data.table(ID = testid, click = predd)
109 | fwrite(sub3, "tinku_preds_file.csv")
110 |
--------------------------------------------------------------------------------
/rank2_munish/boost_final.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | """
3 | Created on Mon Jul 31 15:36:56 2017
4 |
5 | @author: mbansa001c
6 | """
7 |
8 | print ('loading libraries and data')
9 |
10 | import pandas as pd
11 | from catboost import CatBoostClassifier
12 | from sklearn.model_selection import train_test_split
13 | from sklearn.metrics import accuracy_score
14 | import os
15 |
16 | train = pd.read_csv("train.csv")
17 | test = pd.read_csv("test.csv")
18 |
19 | train['siteid'].fillna(-999, inplace=True)
20 | test['siteid'].fillna(-999, inplace=True)
21 |
22 | train['browserid'].fillna("None", inplace=True)
23 | test['browserid'].fillna("None", inplace=True)
24 |
25 | train['devid'].fillna("None", inplace=True)
26 | test['devid'].fillna("None", inplace=True)
27 |
28 | train['datetime'] = pd.to_datetime(train['datetime'])
29 | test['datetime'] = pd.to_datetime(test['datetime'])
30 |
31 | train['tweekday'] = train['datetime'].dt.weekday
32 | train['thour'] = train['datetime'].dt.hour
33 | train['tminute'] = train['datetime'].dt.minute
34 |
35 | test['tweekday'] = test['datetime'].dt.weekday
36 | test['thour'] = test['datetime'].dt.hour
37 | test['tminute'] = test['datetime'].dt.minute
38 |
39 |
40 |
41 | #### training only for the offers in test data #####
42 | train_new=train[train['offerid'].isin(test['offerid'])]
43 |
44 | train_a_b=train_new[train_new['countrycode'].isin(['a','b'])]
45 | train_c_f=train_new[train_new['countrycode'].isin(['c','d','e','f'])]
46 |
47 | test_a_b=test[test['countrycode'].isin(['a','b'])]
48 | test_c_f=test[test['countrycode'].isin(['c','d','e','f'])]
49 |
50 |
51 |
52 | trainY_df = train_a_b[['ID','click']]
53 |
54 | train_a_b.drop(['click'], axis=1, inplace=True)
55 |
56 |
57 | ########join train and test########
58 | df_all = pd.concat([train_a_b, test_a_b])
59 |
60 |
61 | ############ feature engineering, category count,offer count, offer access to users,week and hour level counts############
62 | catg_cnt=df_all.groupby(['siteid'])['category'].nunique().reset_index().rename(columns = {'category':'catg_cnt'})
63 | offer_cnt=df_all.groupby(['siteid'])['offerid'].nunique().reset_index().rename(columns = {'offerid':'offer_cnt'})
64 | offer_id_cnt=df_all.groupby(['siteid','offerid'])['ID'].nunique().reset_index().rename(columns = {'ID':'offer_id_cnt'})
65 |
66 | df1=pd.merge(df_all, catg_cnt, how='left', on=['siteid'])
67 | df2=pd.merge(df1, offer_cnt, how='left', on=['siteid'])
68 | df3=pd.merge(df2, offer_id_cnt, how='left', on=['siteid','offerid'])
69 |
70 |
71 | df3['site_user_cnt'] = df3.groupby(['siteid'])['ID'].transform('count')
72 | df3['user_week_cnt'] = df3.groupby(['siteid','tweekday'])['ID'].transform('count')
73 | df3['user_hr_cnt'] = df3.groupby(['siteid','tweekday','thour'])['ID'].transform('count')
74 |
75 | df3['offer_user_share']=df3['offer_id_cnt']/df3['site_user_cnt']
76 | df3['week_user_share']=df3['user_week_cnt']/df3['site_user_cnt']
77 | df3['hr_user_share']=df3['user_hr_cnt']/df3['user_week_cnt']
78 | ##################
79 |
80 | cols = ['siteid','offerid','category','merchant']
81 |
82 | for x in cols:
83 | df3[x] = df3[x].astype('object')
84 |
85 |
86 | cols_to_use1=['siteid','user_hr_cnt','week_user_share','hr_user_share','merchant', 'catg_cnt', 'user_week_cnt', 'offer_cnt', 'site_user_cnt', 'browserid', 'tweekday', 'devid', 'offerid', 'thour', 'category','countrycode','offer_user_share','tminute']
87 |
88 |
89 | ###########splitting train test after feature creation###########
90 | num_train = len(train_a_b)
91 |
92 | X_train = df3[:num_train]
93 | X_test = df3[num_train:]
94 |
95 | train_target=pd.merge(X_train,trainY_df,how='inner',on=['ID'])
96 |
97 | sampled=train_target.sample(frac=0.3)
98 |
99 | trainX = sampled[cols_to_use1]
100 | trainX.head()
101 | trainY = sampled['click']
102 |
103 | # catboost accepts categorical variables as indexes
104 |
105 | cat_cols = [0,4,9,10,11,12,13,14,15]
106 |
107 |
108 | print ('train model........')
109 | X_train1, X_test1, y_train1, y_test1 = train_test_split(trainX, trainY, test_size = 0.40)
110 | model = CatBoostClassifier(depth=8, iterations=200, learning_rate=0.1, eval_metric='AUC', random_seed=1,calc_feature_importance=True)
111 |
112 | model.fit(X_train1
113 | ,y_train1
114 | ,cat_features=cat_cols
115 | ,eval_set = (X_test1, y_test1)
116 | ,use_best_model = True
117 | )
118 |
119 | preds_class = model.predict(X_test1)
120 | print("accuracy = {}".format(accuracy_score(y_test1, preds_class)))
121 | print(model.feature_importance_)
122 |
123 | print ('making predictions.................')
124 | #### predictions for test #########
125 | pred_a_b = model.predict_proba(X_test[cols_to_use1])[:,1]
126 | sub_a_b=pd.DataFrame({'ID':test_a_b['ID'],'click':pred_a_b})
127 |
128 |
129 | ################################ Repeat from here for c,d,e,f countries again using train_c_f and test_c_f##############
130 | trainY_df = train_c_f[['ID','click']]
131 |
132 | train_c_f.drop(['click'], axis=1, inplace=True)
133 |
134 |
135 | ########join train and test########
136 | df_all = pd.concat([train_c_f, test_c_f])
137 |
138 | ############ feature engineering, category count,offer count, offer access to users,week and hour level counts############
139 | catg_cnt=df_all.groupby(['siteid'])['category'].nunique().reset_index().rename(columns = {'category':'catg_cnt'})
140 | offer_cnt=df_all.groupby(['siteid'])['offerid'].nunique().reset_index().rename(columns = {'offerid':'offer_cnt'})
141 | offer_id_cnt=df_all.groupby(['siteid','offerid'])['ID'].nunique().reset_index().rename(columns = {'ID':'offer_id_cnt'})
142 |
143 | df1=pd.merge(df_all, catg_cnt, how='left', on=['siteid'])
144 | df2=pd.merge(df1, offer_cnt, how='left', on=['siteid'])
145 | df3=pd.merge(df2, offer_id_cnt, how='left', on=['siteid','offerid'])
146 |
147 |
148 | df3['site_user_cnt'] = df3.groupby(['siteid'])['ID'].transform('count')
149 | df3['user_week_cnt'] = df3.groupby(['siteid','tweekday'])['ID'].transform('count')
150 | df3['user_hr_cnt'] = df3.groupby(['siteid','tweekday','thour'])['ID'].transform('count')
151 |
152 | df3['offer_user_share']=df3['offer_id_cnt']/df3['site_user_cnt']
153 | df3['week_user_share']=df3['user_week_cnt']/df3['site_user_cnt']
154 | df3['hr_user_share']=df3['user_hr_cnt']/df3['user_week_cnt']
155 | ##################
156 |
157 | cols = ['siteid','offerid','category','merchant']
158 |
159 | for x in cols:
160 | df3[x] = df3[x].astype('object')
161 |
162 |
163 | cols_to_use1=['siteid','user_hr_cnt','week_user_share','hr_user_share','merchant', 'catg_cnt', 'user_week_cnt', 'offer_cnt', 'site_user_cnt', 'browserid', 'tweekday', 'devid', 'offerid', 'thour', 'category','countrycode','offer_user_share','tminute']
164 |
165 | ###########splitting train test after feature creation###########
166 | num_train = len(train_c_f)
167 |
168 | X_train = df3[:num_train]
169 | X_test = df3[num_train:]
170 |
171 | train_target=pd.merge(X_train,trainY_df,how='inner',on=['ID'])
172 |
173 | sampled=train_target.sample(frac=0.3)
174 |
175 | trainX = sampled[cols_to_use1]
176 | trainX.head()
177 | trainY = sampled['click']
178 |
179 | # catboost accepts categorical variables as indexes
180 |
181 | cat_cols = [0,4,9,10,11,12,13,14,15]
182 |
183 |
184 | X_train1, X_test1, y_train1, y_test1 = train_test_split(trainX, trainY, test_size = 0.40)
185 | model = CatBoostClassifier(depth=8, iterations=200, learning_rate=0.1, eval_metric='AUC', random_seed=1,calc_feature_importance=True)
186 |
187 | model.fit(X_train1
188 | ,y_train1
189 | ,cat_features=cat_cols
190 | ,eval_set = (X_test1, y_test1)
191 | ,use_best_model = True
192 | )
193 |
194 | preds_class = model.predict(X_test1)
195 | print("accuracy = {}".format(accuracy_score(y_test1, preds_class)))
196 | print(model.feature_importance_)
197 |
198 | #### predictions for test #########
199 |
200 | pred_c_f = model.predict_proba(X_test[cols_to_use1])[:,1]
201 | sub_c_f=pd.DataFrame({'ID':test_c_f['ID'],'click':pred_c_f})
202 |
203 |
204 | ##join predictions after repeating for c to f countries###
205 | final=sub_a_b.append(sub_c_f)
206 |
207 | #write output
208 | final.to_csv("submission.csv",index=False)
209 |
210 |
211 |
212 |
213 |
--------------------------------------------------------------------------------
/stackingClicks.R:
--------------------------------------------------------------------------------
1 |
2 | # This script will help you get started with building stacked models in R.
3 | # Although, there are libraries like caret, H2o where stacking happens automatically and many of us never know what happens at back.
4 | # This script will help you understand how stacking happens and will provide your more control of the process.
5 |
6 | # This script builds 5 XGB Models in L1 and finally a Logistic Regression model on top of them.
7 |
8 | path <- "add path"
9 | setwd(path)
10 |
11 | library(data.table)
12 | library(lubridate)
13 | library(caret)
14 | library(xgboost)
15 |
16 | train <- fread("train.csv",na.strings = c(" ","",NA,"NA"))
17 | test <- fread("test.csv",na.strings = c(" ","",NA,"NA"))
18 |
19 |
20 | # Data Pre-Processing -----------------------------------------------------
21 |
22 | # Impute Missing values
23 | colSums(is.na(train))
24 | train[is.na(siteid), siteid := -999]
25 | test[is.na(siteid), siteid := -999]
26 |
27 | train[is.na(browserid), browserid := "None"]
28 | test[is.na(browserid), browserid := "None"]
29 |
30 | train[is.na(devid), devid := "None"]
31 | test[is.na(devid), devid := "None"]
32 |
33 | # Create Date Features
34 | train[,datetime := as.Date(datetime, format = "%Y-%m-%d %H:%M:%S")]
35 | test[,datetime := as.Date(datetime, format = "%Y-%m-%d %H:%M:%S")]
36 |
37 | train[,tweekday := weekdays(datetime)]
38 | test[,tweekday := weekdays(datetime)]
39 |
40 | train[,thour := hour(datetime)]
41 | test[,thour := hour(datetime)]
42 |
43 | train[,tminute := minute(datetime)]
44 | test[,tminute := minute(datetime)]
45 |
46 |
47 | # Label Encoding
48 |
49 | feats <- setdiff(colnames(train), c("ID"))
50 |
51 | for ( f in feats)
52 | {
53 | if (class(train[[f]]) == 'character')
54 | {
55 | levels <- unique(c(train[[f]], test[[f]]))
56 | train[[f]] <- as.numeric(factor(train[[f]], levels = levels))
57 | test[[f]] <- as.numeric(factor(test[[f]], levels = levels))
58 | }
59 | }
60 |
61 | # Some Aggregate Features
62 | train[,agg1 := .N, .(siteid, offerid)]
63 | test[,agg1 := .N, .(siteid, offerid)]
64 |
65 | train[,agg2 := .N, .(siteid, category)]
66 | test[,agg2 := .N, .(siteid, category)]
67 |
68 | train[,agg3 := .N, .(siteid, merchant)]
69 | test[,agg3 := .N, .(siteid, merchant)]
70 |
71 | # To avoid memory troubles, lets train the model on 10% of the data - you can always increase it
72 | trainX <- train[sample(.N, 1e6)]
73 |
74 |
75 | # Stacking ----------------------------------------------------------------
76 |
77 | #The following script works this way:
78 | # 1. For every seed and a different nround value, it creates a prediction for train and test
79 | # 2. All the train and test predictions are stored in oof_train and of_test respectively.
80 | # 3. Finally, a logistic regression model is trained on L1 predictions.
81 | # 4. For understanding, comments are added to help you understand what's going on.
82 |
83 | folds <- createFolds(trainX$click, k = 10, list = T)
84 |
85 | #set seeds
86 | coseed <- c(1,13,999,10001,9009)
87 |
88 | num_rounds <- c(50,100,150,200,250)
89 |
90 | params <- list(
91 |
92 | objective = 'binary:logistic',
93 | base_score = mean(trainX$click == 0),
94 | max_depth = 10,
95 | eta = 0.03,
96 | subsample = 0.5,
97 | colsample_bytree = 0.5,
98 | min_child_weight = 1,
99 | eval_metric = "auc"
100 |
101 | )
102 |
103 | # create data frame to store predictions
104 | oof_train <- data.frame(ID = numeric(), actual = numeric(), predict = numeric(), seed = numeric())
105 | oof_test <- data.frame(ID = test$ID)
106 |
107 | # Start Stacking
108 | for (i in seq(coseed))
109 | {
110 | cat('Building on seed: ',coseed[i],'\n')
111 |
112 | feature.names <- setdiff(colnames(trainX), c('ID','datetime','click'))
113 |
114 | for(j in seq(folds))
115 | {
116 | cat('\n')
117 | cat(sprintf('Training on Fold: %d ',j),'\n')
118 |
119 | idex <- folds[[j]]
120 |
121 | X_train <- trainX[!idex]
122 | X_valid <- trainX[idex]
123 |
124 | cat(sprintf("Training model on: %d rows ", nrow(X_train)),'\n')
125 | cat(sprintf("Testing model on: %d rows", nrow(X_valid)),'\n \n')
126 |
127 | dtrain <- xgb.DMatrix(data = as.matrix(X_train[,feature.names,with=F]), label = X_train$click)
128 | dvalid <- xgb.DMatrix(data = as.matrix(X_valid[,feature.names,with=F]), label = X_valid$click)
129 |
130 | watchlist <- list(train = dtrain, valid = dvalid)
131 |
132 | bst <- xgb.train(params = params
133 | ,data = dtrain
134 | ,nrounds = as.integer(num_rounds[i])
135 | ,print_every_n = 20
136 | ,watchlist = watchlist
137 | ,early_stopping_rounds = 40
138 | ,maximize = T
139 | ,nthreads = -1)
140 |
141 | preds <- predict(bst, dvalid)
142 | df = data.frame(ID = X_valid$ID, actual = X_valid$click, predict = preds, seed = i)
143 | oof_train <- rbind(oof_train, df)
144 |
145 | #cat(sprintf("Now, oof_train has %d rows and %d columns",nrow(oof_train), ncol(oof_train)),'\n')
146 |
147 | rm(bst, dtrain, dvalid, preds,df)
148 | gc()
149 |
150 | }
151 |
152 | cat('\n')
153 | cat(sprintf("After %s seed, oof_train has %s rows and %s columns",i,nrow(oof_train),ncol(oof_train)),'\n \n')
154 | cat("Now Training on full data........",'\n')
155 |
156 | dtest <- xgb.DMatrix(data = as.matrix(test[,feature.names,with=F]))
157 | dtrain <- xgb.DMatrix(data = as.matrix(trainX[,feature.names,with=F]), label = trainX$click)
158 |
159 | print ('Training model...')
160 | bst <- xgb.train(params = params
161 | ,data = dtrain
162 | ,nrounds = as.integer(num_rounds[i])
163 | ,print_every_n = 20
164 | ,maximize = T
165 | ,nthreads = -1)
166 |
167 | print('predicting....')
168 |
169 | preds <- predict(bst, dtest)
170 | oof_test[[paste0("pred_seed_",i)]] <- preds
171 |
172 | cat(sprintf("oof_test has %s rows and %s columns",nrow(oof_test),ncol(oof_test)),'\n \n')
173 | rm(df, bst, preds)
174 |
175 | }
176 |
177 |
178 |
179 | # Now creating Level 1 Data -----------------------------------------------
180 |
181 | oof_train$seed <- as.factor(oof_train$seed)
182 | mastertrain <- split(oof_train, oof_train$seed)
183 |
184 | for(x in seq(mastertrain))
185 | {
186 | assign(paste0("oof_train_",x), data.table(mastertrain[[x]]))
187 | }
188 |
189 | setnames(oof_train_1,"predict","pred_seed_1")
190 | setnames(oof_train_2,"predict","pred_seed_2")
191 | setnames(oof_train_3,"predict","pred_seed_3")
192 | setnames(oof_train_4,"predict","pred_seed_4")
193 | setnames(oof_train_5,"predict","pred_seed_5")
194 |
195 | mastertrain <- merge(x = oof_train_1, y = oof_train_2[,.(ID, pred_seed_2)], by = 'ID', all.x = T)
196 | mastertrain <- merge(x = mastertrain, y = oof_train_3[,.(ID, pred_seed_3)], by = 'ID', all.x = T)
197 | mastertrain <- merge(x = mastertrain, y = oof_train_4[,.(ID, pred_seed_4)], by = 'ID', all.x = T)
198 | mastertrain <- merge(x = mastertrain, y = oof_train_5[,.(ID, pred_seed_5)], by = 'ID', all.x = T)
199 | head(mastertrain)
200 |
201 | rm(oof_train, oof_train_1, oof_train_2, oof_train_3, oof_train_4, oof_train_5,trainX,X_train,X_valid)
202 | gc()
203 |
204 | setnames(mastertrain,"actual","click")
205 |
206 |
207 | # Training Model ----------------------------------------------------------
208 |
209 | # Logistic Regression
210 | mastertrain[,click := as.factor(click)]
211 |
212 | lr_model <- glm(click ~ ., data = mastertrain[,-c('ID','seed'),with=F], family = binomial(link = 'logit'),maxit = 500)
213 | summary(lr_model)
214 |
215 | preds <- predict(lr_model, oof_test, type = 'response')
216 |
217 | submit <- data.table(ID = oof_test$ID, click = preds)
218 | fwrite(submit, "stacked_lgr.csv")
219 |
220 |
221 | # What can you do next ?
222 |
223 | # 1. Train on more data
224 | # 2. Try to create more features
225 | # 3. Make L1 predictions more diverse.
226 | # 3. Try using boosting L1 predictions.
227 |
228 |
229 |
230 |
231 |
232 |
233 |
--------------------------------------------------------------------------------