├── .github └── FUNDING.yml ├── README.md ├── a00_common_functions.py ├── a00_common_functions_scores.py ├── a00_custom_layers.py ├── a01_densenet_121.py ├── a01_densenet_161.py ├── a01_densenet_169.py ├── a01_inception_v4.py ├── a01_resnet_101.py ├── a01_resnet_152.py ├── a01_squeezenet.py ├── a02_zoo.py ├── a11_find_neighbours.py ├── a30_create_keras_models.py ├── a30_create_keras_models_land.py ├── a30_create_keras_models_single_class.py ├── a30_create_keras_models_weather.py ├── a31_create_cnn_features_basic.py ├── a31_create_cnn_features_land.py ├── a31_create_cnn_features_weather.py ├── a32_create_cnn_features_single_class.py ├── a32_find_neighbours_features.py ├── a42_gbm_blender.py ├── a42_keras_blender.py ├── a50_ensemble_from_cache_v1.py └── images └── Dataflow.png /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | custom: https://paypal.me/zfturbo 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Repository contains code to solve Kaggle problem [Planet: Understanding the Amazon from Space](https://www.kaggle.com/c/planet-understanding-the-amazon-from-space). This solution won 3rd place in competition. 2 | 3 | # Requirements: 4 | 5 | Python >= 3.4, Keras 1.2.1, Theano 0.9.2, Tensorflow, XGBoost 0.6 6 | 7 | # How to run: 8 | 9 | You need to execute set of scripts one by one: 10 | * python a11_find_neighbours.py 11 | * python a30_create_keras_models.py 12 | * python a30_create_keras_models_land.py 13 | * python a30_create_keras_models_weather.py 14 | * python a30_create_keras_models_single_class.py 15 | * python a31_create_cnn_features_basic.py 16 | * python a31_create_cnn_features_land.py 17 | * python a31_create_cnn_features_weather.py 18 | * python a32_create_cnn_features_single_class.py 19 | * python a32_find_neighbours_features.py 20 | * python a42_gbm_blender.py 21 | * python a42_keras_blender.py 22 | * python a50_ensemble_from_cache_v1.py 23 | 24 | # Notes: 25 | 1. Recreating all CNN models from scratch on single GPU will require a lot of time (around a month). It can be parallelized using separate GPU on different CNN models. Final models weights size ~50 GB. Msg me if you need these weights. 26 | 2. Creating neighbours features requires around a day to complete. 27 | 3. Due to high parallelization, CNN models trained on GPU can slightly differ even in case it was trained on the same code. 28 | 4. A little bit details about solution available on [Kaggle forum](https://www.kaggle.com/c/planet-understanding-the-amazon-from-space/discussion/38831) 29 | 30 | # Directory structure: 31 | * -- input - input data as it was given on Kaggle 32 | * -- Kaggle-Planet-Understanding-the-Amazon-from-Space - all the Python code (this repo) 33 | * -- models - all generated models from neural nets will be in this folder. 34 | * -- weights - files with weights for pretrained models. Link: [Download](https://mega.nz/#!zMZmCQ7T!ENqFmKZGXKoX6nXRWN7xGRpkUKZpWTwD8rJ_dXA0Sro) 35 | * -- modified_data - some intermediate files for neighbour analysis 36 | * -- features - all raw features generated by neural nets will be stored in this folder. We already have them calculated. Link: [Download]( 37 | https://mega.nz/#!nYxwRYjY!wyEEKB0jr3D9bzO_sB8MWOASvy7GDtqDZSiudrTUp98 38 | ) 39 | * -- cache - this folder will contain arrays with predictions from XGBoost and Keras blenders 40 | * -- subm - final predictions (in format of submit file for Kaggle) 41 | 42 | # Dataflow 43 | 44 | ![Dataflow](https://raw.githubusercontent.com/ZFTurbo/Kaggle-Planet-Understanding-the-Amazon-from-Space/master/images/Dataflow.png) 45 | 46 | -------------------------------------------------------------------------------- /a00_common_functions_scores.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | __author__ = 'ZFTurbo: https://kaggle.com/zfturbo' 3 | 4 | from a00_common_functions import * 5 | 6 | random.seed(2017) 7 | np.random.seed(2017) 8 | 9 | 10 | def get_f2_score(thr_arr, p, lbl, indexes): 11 | preds = p.copy() 12 | for i in range(len(indexes)): 13 | preds[:, i][preds[:, i] > thr_arr[i]] = 1 14 | preds[:, i][preds[:, i] <= thr_arr[i]] = 0 15 | score = f2_score(lbl, preds) 16 | # print(score) 17 | # print(thr_arr) 18 | return -score 19 | 20 | 21 | def test_1(): 22 | p = np.arange(150.).reshape(5, 10, 3) 23 | print(p[:, :, 0]) 24 | cond = p[:, :, 0] > 80 25 | print(cond) 26 | cond = np.sum(cond, axis=0) 27 | set_1 = cond*2 > p.shape[0] 28 | set_0 = cond*2 < p.shape[0] 29 | print(set_1) 30 | print(set_0) 31 | exit() 32 | 33 | 34 | def get_f2_score_full_arr(thr_arr, p, lbl, indexes): 35 | preds = np.zeros(p.shape[1:]) 36 | for i in range(len(indexes)): 37 | cond = p[:, :, i] > thr_arr[i] 38 | cond = 2 * np.sum(cond, axis=0) 39 | preds[:, i][cond >= p.shape[0]] = 1 40 | preds[:, i][cond < p.shape[0]] = 0 41 | score = f2_score(lbl, preds) 42 | # print(score) 43 | # print(thr_arr) 44 | return -score 45 | 46 | 47 | def f2beta_loss(Y_true, Y_pred): 48 | from keras import backend as K 49 | eps = 0.000001 50 | false_positive = K.sum(Y_pred * (1 - Y_true), axis=-1) 51 | false_negative = K.sum((1 - Y_pred) * Y_true, axis=-1) 52 | true_positive = K.sum(Y_true * Y_pred, axis=-1) 53 | p = (true_positive + eps) / (true_positive + false_positive + eps) 54 | r = (true_positive + eps) / (true_positive + false_negative + eps) 55 | out = (5*p*r + eps) / (4*p + r + eps) 56 | return -K.mean(out) 57 | 58 | 59 | def get_optimal_score_normal(indexes, lbl, validation_arr, koeff=100): 60 | start_time = time.time() 61 | # Step 1 (Low precision) 62 | best_score1 = -1 63 | best_index = -1 64 | no_improve = 0 65 | for i in range(koeff): 66 | thr = i / koeff 67 | score = abs(get_f2_score(np.array([thr] * len(indexes)), validation_arr, lbl, indexes)) 68 | # print(thr, score) 69 | if score > best_score1: 70 | best_score1 = score 71 | best_index = i 72 | no_improve = 0 73 | else: 74 | no_improve += 1 75 | if no_improve >= koeff // 5: 76 | break 77 | 78 | # Step 2 (High precision) 79 | best_score = -1 80 | best_thr = -1 81 | no_improve = 0 82 | for i in range(max(koeff * (best_index - 10), 0), koeff*(best_index + 10)): 83 | thr = i / (koeff*koeff) 84 | score = abs(get_f2_score(np.array([thr] * len(indexes)), validation_arr, lbl, indexes)) 85 | # print(thr, score) 86 | if score > best_score: 87 | best_score = score 88 | best_thr = thr 89 | no_improve = 0 90 | else: 91 | no_improve += 1 92 | if no_improve >= koeff*2: 93 | break 94 | 95 | # Try to find optimum independently for each class 96 | searcher = np.array([best_thr] * len(indexes)) 97 | best_score = -1 98 | for j in range(len(indexes)): 99 | best_thr1 = searcher[j] 100 | 101 | # Step 1 (Low precision) 102 | best_score1 = -1 103 | best_index = -1 104 | no_improve = 0 105 | for i in range(koeff): 106 | thr = i / koeff 107 | searcher[j] = thr 108 | score = abs(get_f2_score(searcher, validation_arr, lbl, indexes)) 109 | # print(thr, score) 110 | if score > best_score1: 111 | best_score1 = score 112 | best_index = i 113 | no_improve = 0 114 | else: 115 | no_improve += 1 116 | if no_improve >= koeff // 5: 117 | break 118 | 119 | # Step 2 (High precision) 120 | best_score1 = -1 121 | no_improve = 0 122 | for i in range(max(koeff * (best_index - 10), 0), koeff * (best_index + 10)): 123 | thr = i / (koeff*koeff) 124 | searcher[j] = thr 125 | score = abs(get_f2_score(searcher, validation_arr, lbl, indexes)) 126 | # print(thr, score) 127 | if score > best_score: 128 | best_score = score 129 | best_thr1 = thr 130 | if score > best_score1: 131 | best_score1 = score 132 | no_improve = 0 133 | else: 134 | no_improve += 1 135 | if no_improve >= koeff: 136 | break 137 | 138 | searcher[j] = best_thr1 139 | print('Class: {} Best score {} for THR: {}'.format(j, best_score, best_thr1)) 140 | print("Time search thr: %s sec" % (round(time.time() - start_time, 0))) 141 | return best_score, searcher 142 | 143 | 144 | def get_optimal_score_fast(indexes, lbl, validation_arr, koeff=100): 145 | start_time = time.time() 146 | # Step 1 (Low precision) 147 | best_score1 = -1 148 | best_thr = -1 149 | no_improve = 0 150 | for i in range(koeff): 151 | thr = i / koeff 152 | score = abs(get_f2_score(np.array([thr] * len(indexes)), validation_arr, lbl, indexes)) 153 | if score > best_score1: 154 | best_score1 = score 155 | best_thr = thr 156 | no_improve = 0 157 | else: 158 | no_improve += 1 159 | if no_improve >= koeff // 10: 160 | break 161 | 162 | # Try to find optimum independently for each class 163 | searcher = np.array([best_thr] * len(indexes)) 164 | best_score = -1 165 | for j in range(len(indexes)): 166 | best_thr1 = searcher[j] 167 | best_score1 = -1 168 | no_improve = 0 169 | for i in range(koeff): 170 | thr = i / koeff 171 | searcher[j] = thr 172 | score = abs(get_f2_score(searcher, validation_arr, lbl, indexes)) 173 | if score > best_score: 174 | best_score = score 175 | best_thr1 = thr 176 | 177 | if score > best_score1: 178 | best_score1 = score 179 | no_improve = 0 180 | else: 181 | no_improve += 1 182 | if no_improve >= koeff // 10: 183 | break 184 | searcher[j] = best_thr1 185 | print('Class: {} Best score {} for THR: {}'.format(j, best_score, best_thr1)) 186 | print("Time search thr: %s seconds" % (round(time.time() - start_time, 0))) 187 | return best_score, searcher 188 | 189 | 190 | def get_optimal_score_slow(indexes, lbl, validation_arr, koeff=100): 191 | start_time = time.time() 192 | # Step 1 (Low precision) 193 | best_score = -1 194 | best_thr = -1 195 | no_improve = 0 196 | for i in range(koeff): 197 | thr = i / koeff 198 | score = abs(get_f2_score(np.array([thr] * len(indexes)), validation_arr, lbl, indexes)) 199 | if score > best_score: 200 | best_score = score 201 | best_thr = thr 202 | no_improve = 0 203 | else: 204 | no_improve += 1 205 | if no_improve >= koeff: 206 | break 207 | 208 | # Try to find optimum independently for each class 209 | searcher = np.array([best_thr] * len(indexes)) 210 | for j in range(len(indexes)): 211 | best_thr1 = searcher[j] 212 | best_score1 = -1 213 | no_improve = 0 214 | for i in range(koeff): 215 | thr = i / koeff 216 | searcher[j] = thr 217 | score = abs(get_f2_score(searcher, validation_arr, lbl, indexes)) 218 | if score > best_score: 219 | best_score = score 220 | best_thr1 = thr 221 | 222 | if score > best_score1: 223 | best_score1 = score 224 | no_improve = 0 225 | else: 226 | no_improve += 1 227 | if no_improve >= koeff: 228 | break 229 | searcher[j] = best_thr1 230 | print('Class: {} Best score {} for THR: {}'.format(j, best_score, best_thr1)) 231 | 232 | # Try to find optimum independently for each class 233 | koeff *= 100 234 | for j in range(len(indexes)): 235 | best_thr1 = searcher[j] 236 | best_score1 = -1 237 | no_improve = 0 238 | for i in range(koeff): 239 | thr = i / koeff 240 | searcher[j] = thr 241 | score = abs(get_f2_score(searcher, validation_arr, lbl, indexes)) 242 | if score > best_score: 243 | best_score = score 244 | best_thr1 = thr 245 | 246 | if score > best_score1: 247 | best_score1 = score 248 | no_improve = 0 249 | else: 250 | no_improve += 1 251 | if no_improve >= koeff: 252 | break 253 | searcher[j] = best_thr1 254 | print('Class: {} Best score {} for THR: {}'.format(j, best_score, best_thr1)) 255 | print("Time search thr: %s seconds" % (round(time.time() - start_time, 0))) 256 | return best_score, searcher 257 | 258 | 259 | def get_optimal_score_very_fast(indexes, lbl, validation_arr, splits=7, eps=0.01, iterations=2, log=True): 260 | start_time = time.time() 261 | 262 | best_score = -1 263 | best_thr = -1 264 | start = 0.0 265 | stop = 1.0 266 | while 1: 267 | scores = np.zeros(splits) 268 | thr_list = np.zeros(splits) 269 | for i in range(splits): 270 | thr = start + i*(stop - start)/splits 271 | thr_list[i] = thr 272 | scores[i] = abs(get_f2_score(np.array([thr] * len(indexes)), validation_arr, lbl, indexes)) 273 | index = int(np.argmax(scores)) 274 | score = scores[index] 275 | if score > best_score: 276 | best_score = score 277 | best_thr = thr_list[index] 278 | if index > 0: 279 | start = thr_list[index - 1] 280 | if index < splits-1: 281 | stop = thr_list[index + 1] 282 | if stop - start < eps: 283 | break 284 | if log: 285 | print('Best score {} for single THR: {}'.format(best_score, best_thr)) 286 | 287 | # Try to find optimum independently for each class 288 | searcher = np.array([best_thr] * len(indexes)) 289 | orig_split = splits 290 | for z in range(iterations): 291 | if z > 0: 292 | splits = random.randint(orig_split-3, orig_split+3) 293 | if splits == orig_split: 294 | splits = random.randint(orig_split - 3, orig_split + 3) 295 | if log: 296 | print('Iteration: {} New split: {}'.format(z, splits)) 297 | for j in range(len(indexes)): 298 | best_thr1 = searcher[j] 299 | start = 0.0 300 | stop = 1.0 301 | while 1: 302 | scores = np.zeros(splits) 303 | thr_list = np.zeros(splits) 304 | for i in range(splits): 305 | thr = start + i * (stop - start) / splits 306 | thr_list[i] = thr 307 | searcher[j] = thr 308 | scores[i] = abs(get_f2_score(searcher, validation_arr, lbl, indexes)) 309 | index = int(np.argmax(scores)) 310 | score = scores[index] 311 | if score > best_score: 312 | best_score = score 313 | best_thr1 = thr_list[index] 314 | if index > 0: 315 | start = thr_list[index - 1] 316 | if index < splits - 1: 317 | stop = thr_list[index + 1] 318 | if stop - start < eps: 319 | break 320 | searcher[j] = best_thr1 321 | if log: 322 | print('Class: {} Best score {} for THR: {}'.format(j, best_score, best_thr1)) 323 | if log: 324 | print("Time search thr: %s seconds" % (round(time.time() - start_time, 0))) 325 | return best_score, searcher 326 | 327 | 328 | def get_optimal_score_very_fast_for_full_array(indexes, lbl, validation_arr, splits=7, eps=0.01, iterations=2): 329 | start_time = time.time() 330 | 331 | best_score = -1 332 | best_thr = -1 333 | start = 0.0 334 | stop = 1.0 335 | while 1: 336 | scores = np.zeros(splits) 337 | thr_list = np.zeros(splits) 338 | for i in range(splits): 339 | thr = start + i*(stop - start)/splits 340 | thr_list[i] = thr 341 | scores[i] = abs(get_f2_score_full_arr(np.array([thr] * len(indexes)), validation_arr, lbl, indexes)) 342 | index = int(np.argmax(scores)) 343 | score = scores[index] 344 | if score > best_score: 345 | best_score = score 346 | best_thr = thr_list[index] 347 | if index > 0: 348 | start = thr_list[index - 1] 349 | if index < splits-1: 350 | stop = thr_list[index + 1] 351 | if stop - start < eps: 352 | break 353 | print('Best score {} for single THR: {}'.format(best_score, best_thr)) 354 | 355 | # Try to find optimum independently for each class 356 | searcher = np.array([best_thr] * len(indexes)) 357 | orig_split = splits 358 | for z in range(iterations): 359 | if z > 0: 360 | splits = random.randint(orig_split-3, orig_split+3) 361 | if splits == orig_split: 362 | splits = random.randint(orig_split - 3, orig_split + 3) 363 | print('Iteration: {} New split: {}'.format(z, splits)) 364 | for j in range(len(indexes)): 365 | best_thr1 = searcher[j] 366 | start = 0.0 367 | stop = 1.0 368 | while 1: 369 | scores = np.zeros(splits) 370 | thr_list = np.zeros(splits) 371 | for i in range(splits): 372 | thr = start + i * (stop - start) / splits 373 | thr_list[i] = thr 374 | searcher[j] = thr 375 | scores[i] = abs(get_f2_score_full_arr(searcher, validation_arr, lbl, indexes)) 376 | index = int(np.argmax(scores)) 377 | score = scores[index] 378 | if score > best_score: 379 | best_score = score 380 | best_thr1 = thr_list[index] 381 | if index > 0: 382 | start = thr_list[index - 1] 383 | if index < splits - 1: 384 | stop = thr_list[index + 1] 385 | if stop - start < eps: 386 | break 387 | searcher[j] = best_thr1 388 | print('Class: {} Best score {} for THR: {}'.format(j, best_score, best_thr1)) 389 | print("Time search thr: %s seconds" % (round(time.time() - start_time, 0))) 390 | return best_score, searcher 391 | -------------------------------------------------------------------------------- /a00_custom_layers.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | # Based on: https://github.com/flyyufelix/DenseNet-Keras 3 | __author__ = 'ZFTurbo: https://kaggle.com/zfturbo' 4 | 5 | try: 6 | from keras import initializations 7 | except ImportError: 8 | from keras import initializers as initializations 9 | 10 | from keras.engine import Layer, InputSpec 11 | import keras.backend as K 12 | 13 | 14 | class Scale(Layer): 15 | '''Learns a set of weights and biases used for scaling the input data. 16 | the output consists simply in an element-wise multiplication of the input 17 | and a sum of a set of constants: 18 | 19 | out = in * gamma + beta, 20 | 21 | where 'gamma' and 'beta' are the weights and biases larned. 22 | 23 | # Arguments 24 | axis: integer, axis along which to normalize in mode 0. For instance, 25 | if your input tensor has shape (samples, channels, rows, cols), 26 | set axis to 1 to normalize per feature map (channels axis). 27 | momentum: momentum in the computation of the 28 | exponential average of the mean and standard deviation 29 | of the data, for feature-wise normalization. 30 | weights: Initialization weights. 31 | List of 2 Numpy arrays, with shapes: 32 | `[(input_shape,), (input_shape,)]` 33 | beta_init: name of initialization function for shift parameter 34 | (see [initializations](../initializations.md)), or alternatively, 35 | Theano/TensorFlow function to use for weights initialization. 36 | This parameter is only relevant if you don't pass a `weights` argument. 37 | gamma_init: name of initialization function for scale parameter (see 38 | [initializations](../initializations.md)), or alternatively, 39 | Theano/TensorFlow function to use for weights initialization. 40 | This parameter is only relevant if you don't pass a `weights` argument. 41 | gamma_init: name of initialization function for scale parameter (see 42 | [initializations](../initializations.md)), or alternatively, 43 | Theano/TensorFlow function to use for weights initialization. 44 | This parameter is only relevant if you don't pass a `weights` argument. 45 | ''' 46 | def __init__(self, weights=None, axis=-1, momentum = 0.9, beta_init='zero', gamma_init='one', **kwargs): 47 | self.momentum = momentum 48 | self.axis = axis 49 | self.beta_init = initializations.get(beta_init) 50 | self.gamma_init = initializations.get(gamma_init) 51 | self.initial_weights = weights 52 | super(Scale, self).__init__(**kwargs) 53 | 54 | 55 | def build(self, input_shape): 56 | self.input_spec = [InputSpec(shape=input_shape)] 57 | shape = (int(input_shape[self.axis]),) 58 | 59 | try: 60 | self.gamma = self.gamma_init(shape, name='{}_gamma'.format(self.name)) 61 | self.beta = self.beta_init(shape, name='{}_beta'.format(self.name)) 62 | except: 63 | # Keras 2.0 64 | self.gamma = K.variable(self.gamma_init(shape), name='{}_gamma'.format(self.name)) 65 | self.beta = K.variable(self.beta_init(shape), name='{}_beta'.format(self.name)) 66 | 67 | self.trainable_weights = [self.gamma, self.beta] 68 | if self.initial_weights is not None: 69 | self.set_weights(self.initial_weights) 70 | del self.initial_weights 71 | 72 | 73 | def call(self, x, mask=None): 74 | input_shape = self.input_spec[0].shape 75 | broadcast_shape = [1] * len(input_shape) 76 | broadcast_shape[self.axis] = input_shape[self.axis] 77 | 78 | out = K.reshape(self.gamma, broadcast_shape) * x + K.reshape(self.beta, broadcast_shape) 79 | return out 80 | 81 | 82 | def get_config(self): 83 | config = {"momentum": self.momentum, "axis": self.axis} 84 | base_config = super(Scale, self).get_config() 85 | return dict(list(base_config.items()) + list(config.items())) -------------------------------------------------------------------------------- /a01_densenet_121.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | # Based on: https://github.com/flyyufelix/DenseNet-Keras 3 | __author__ = 'ZFTurbo: https://kaggle.com/zfturbo' 4 | 5 | 6 | from keras.models import Model 7 | from keras.layers import Input, merge, ZeroPadding2D 8 | from keras.layers.core import Dense, Dropout, Activation 9 | from keras.layers.convolutional import Convolution2D 10 | from keras.layers.pooling import AveragePooling2D, GlobalAveragePooling2D, MaxPooling2D 11 | from keras.layers.normalization import BatchNormalization 12 | import keras.backend as K 13 | 14 | from a00_custom_layers import Scale 15 | 16 | 17 | def DenseNet_121(nb_dense_block=4, growth_rate=32, nb_filter=64, reduction=0.0, dropout_rate=0.0, weight_decay=1e-4, classes=1000, weights_path=None): 18 | '''Instantiate the DenseNet 121 architecture, 19 | # Arguments 20 | nb_dense_block: number of dense blocks to add to end 21 | growth_rate: number of filters to add per dense block 22 | nb_filter: initial number of filters 23 | reduction: reduction factor of transition blocks. 24 | dropout_rate: dropout rate 25 | weight_decay: weight decay factor 26 | classes: optional number of classes to classify images 27 | weights_path: path to pre-trained weights 28 | # Returns 29 | A Keras model instance. 30 | ''' 31 | eps = 1.1e-5 32 | 33 | # compute compression factor 34 | compression = 1.0 - reduction 35 | 36 | # Handle Dimension Ordering for different backends 37 | global concat_axis 38 | if K.image_dim_ordering() == 'tf': 39 | concat_axis = 3 40 | img_input = Input(shape=(224, 224, 3), name='data') 41 | else: 42 | concat_axis = 1 43 | img_input = Input(shape=(3, 224, 224), name='data') 44 | 45 | # From architecture for ImageNet (Table 1 in the paper) 46 | nb_filter = 64 47 | nb_layers = [6, 12, 24, 16] # For DenseNet-121 48 | 49 | # Initial convolution 50 | x = ZeroPadding2D((3, 3), name='conv1_zeropadding')(img_input) 51 | x = Convolution2D(nb_filter, 7, 7, subsample=(2, 2), name='conv1', bias=False)(x) 52 | x = BatchNormalization(epsilon=eps, axis=concat_axis, name='conv1_bn')(x) 53 | x = Scale(axis=concat_axis, name='conv1_scale')(x) 54 | x = Activation('relu', name='relu1')(x) 55 | x = ZeroPadding2D((1, 1), name='pool1_zeropadding')(x) 56 | x = MaxPooling2D((3, 3), strides=(2, 2), name='pool1')(x) 57 | 58 | # Add dense blocks 59 | for block_idx in range(nb_dense_block - 1): 60 | stage = block_idx+2 61 | x, nb_filter = dense_block(x, stage, nb_layers[block_idx], nb_filter, growth_rate, dropout_rate=dropout_rate, weight_decay=weight_decay) 62 | 63 | # Add transition_block 64 | x = transition_block(x, stage, nb_filter, compression=compression, dropout_rate=dropout_rate, weight_decay=weight_decay) 65 | nb_filter = int(nb_filter * compression) 66 | 67 | final_stage = stage + 1 68 | x, nb_filter = dense_block(x, final_stage, nb_layers[-1], nb_filter, growth_rate, dropout_rate=dropout_rate, weight_decay=weight_decay) 69 | 70 | x = BatchNormalization(epsilon=eps, axis=concat_axis, name='conv'+str(final_stage)+'_blk_bn')(x) 71 | x = Scale(axis=concat_axis, name='conv'+str(final_stage)+'_blk_scale')(x) 72 | x = Activation('relu', name='relu'+str(final_stage)+'_blk')(x) 73 | x = GlobalAveragePooling2D(name='pool'+str(final_stage))(x) 74 | 75 | x = Dense(classes, name='fc6')(x) 76 | x = Activation('softmax', name='prob')(x) 77 | 78 | model = Model(img_input, x, name='densenet') 79 | 80 | if weights_path is not None: 81 | model.load_weights(weights_path) 82 | 83 | return model 84 | 85 | 86 | def conv_block(x, stage, branch, nb_filter, dropout_rate=None, weight_decay=1e-4): 87 | '''Apply BatchNorm, Relu, bottleneck 1x1 Conv2D, 3x3 Conv2D, and option dropout 88 | # Arguments 89 | x: input tensor 90 | stage: index for dense block 91 | branch: layer index within each dense block 92 | nb_filter: number of filters 93 | dropout_rate: dropout rate 94 | weight_decay: weight decay factor 95 | ''' 96 | eps = 1.1e-5 97 | conv_name_base = 'conv' + str(stage) + '_' + str(branch) 98 | relu_name_base = 'relu' + str(stage) + '_' + str(branch) 99 | 100 | # 1x1 Convolution (Bottleneck layer) 101 | inter_channel = nb_filter * 4 102 | x = BatchNormalization(epsilon=eps, axis=concat_axis, name=conv_name_base+'_x1_bn')(x) 103 | x = Scale(axis=concat_axis, name=conv_name_base+'_x1_scale')(x) 104 | x = Activation('relu', name=relu_name_base+'_x1')(x) 105 | x = Convolution2D(inter_channel, 1, 1, name=conv_name_base+'_x1', bias=False)(x) 106 | 107 | if dropout_rate: 108 | x = Dropout(dropout_rate)(x) 109 | 110 | # 3x3 Convolution 111 | x = BatchNormalization(epsilon=eps, axis=concat_axis, name=conv_name_base+'_x2_bn')(x) 112 | x = Scale(axis=concat_axis, name=conv_name_base+'_x2_scale')(x) 113 | x = Activation('relu', name=relu_name_base+'_x2')(x) 114 | x = ZeroPadding2D((1, 1), name=conv_name_base+'_x2_zeropadding')(x) 115 | x = Convolution2D(nb_filter, 3, 3, name=conv_name_base+'_x2', bias=False)(x) 116 | 117 | if dropout_rate: 118 | x = Dropout(dropout_rate)(x) 119 | 120 | return x 121 | 122 | 123 | def transition_block(x, stage, nb_filter, compression=1.0, dropout_rate=None, weight_decay=1E-4): 124 | ''' Apply BatchNorm, 1x1 Convolution, averagePooling, optional compression, dropout 125 | # Arguments 126 | x: input tensor 127 | stage: index for dense block 128 | nb_filter: number of filters 129 | compression: calculated as 1 - reduction. Reduces the number of feature maps in the transition block. 130 | dropout_rate: dropout rate 131 | weight_decay: weight decay factor 132 | ''' 133 | 134 | eps = 1.1e-5 135 | conv_name_base = 'conv' + str(stage) + '_blk' 136 | relu_name_base = 'relu' + str(stage) + '_blk' 137 | pool_name_base = 'pool' + str(stage) 138 | 139 | x = BatchNormalization(epsilon=eps, axis=concat_axis, name=conv_name_base+'_bn')(x) 140 | x = Scale(axis=concat_axis, name=conv_name_base+'_scale')(x) 141 | x = Activation('relu', name=relu_name_base)(x) 142 | x = Convolution2D(int(nb_filter * compression), 1, 1, name=conv_name_base, bias=False)(x) 143 | 144 | if dropout_rate: 145 | x = Dropout(dropout_rate)(x) 146 | 147 | x = AveragePooling2D((2, 2), strides=(2, 2), name=pool_name_base)(x) 148 | 149 | return x 150 | 151 | 152 | def dense_block(x, stage, nb_layers, nb_filter, growth_rate, dropout_rate=None, weight_decay=1e-4, grow_nb_filters=True): 153 | ''' Build a dense_block where the output of each conv_block is fed to subsequent ones 154 | # Arguments 155 | x: input tensor 156 | stage: index for dense block 157 | nb_layers: the number of layers of conv_block to append to the model. 158 | nb_filter: number of filters 159 | growth_rate: growth rate 160 | dropout_rate: dropout rate 161 | weight_decay: weight decay factor 162 | grow_nb_filters: flag to decide to allow number of filters to grow 163 | ''' 164 | 165 | eps = 1.1e-5 166 | concat_feat = x 167 | 168 | for i in range(nb_layers): 169 | branch = i+1 170 | x = conv_block(concat_feat, stage, branch, growth_rate, dropout_rate, weight_decay) 171 | concat_feat = merge([concat_feat, x], mode='concat', concat_axis=concat_axis, name='concat_'+str(stage)+'_'+str(branch)) 172 | 173 | if grow_nb_filters: 174 | nb_filter += growth_rate 175 | 176 | return concat_feat, nb_filter 177 | -------------------------------------------------------------------------------- /a01_densenet_161.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | # Based on: https://github.com/flyyufelix/DenseNet-Keras 3 | __author__ = 'ZFTurbo: https://kaggle.com/zfturbo' 4 | 5 | from keras.models import Model 6 | from keras.layers import Input, merge, ZeroPadding2D 7 | from keras.layers.core import Dense, Dropout, Activation 8 | from keras.layers.convolutional import Convolution2D 9 | from keras.layers.pooling import AveragePooling2D, GlobalAveragePooling2D, MaxPooling2D 10 | from keras.layers.normalization import BatchNormalization 11 | import keras.backend as K 12 | 13 | from a00_custom_layers import Scale 14 | 15 | def DenseNet_161(nb_dense_block=4, growth_rate=48, nb_filter=96, reduction=0.0, dropout_rate=0.0, weight_decay=1e-4, classes=1000, weights_path=None): 16 | '''Instantiate the DenseNet 161 architecture, 17 | # Arguments 18 | nb_dense_block: number of dense blocks to add to end 19 | growth_rate: number of filters to add per dense block 20 | nb_filter: initial number of filters 21 | reduction: reduction factor of transition blocks. 22 | dropout_rate: dropout rate 23 | weight_decay: weight decay factor 24 | classes: optional number of classes to classify images 25 | weights_path: path to pre-trained weights 26 | # Returns 27 | A Keras model instance. 28 | ''' 29 | eps = 1.1e-5 30 | 31 | # compute compression factor 32 | compression = 1.0 - reduction 33 | 34 | # Handle Dimension Ordering for different backends 35 | global concat_axis 36 | if K.image_dim_ordering() == 'tf': 37 | concat_axis = 3 38 | img_input = Input(shape=(224, 224, 3), name='data') 39 | else: 40 | concat_axis = 1 41 | img_input = Input(shape=(3, 224, 224), name='data') 42 | 43 | # From architecture for ImageNet (Table 1 in the paper) 44 | nb_filter = 96 45 | nb_layers = [6,12,36,24] # For DenseNet-161 46 | 47 | # Initial convolution 48 | x = ZeroPadding2D((3, 3), name='conv1_zeropadding')(img_input) 49 | x = Convolution2D(nb_filter, 7, 7, subsample=(2, 2), name='conv1', bias=False)(x) 50 | x = BatchNormalization(epsilon=eps, axis=concat_axis, name='conv1_bn')(x) 51 | x = Scale(axis=concat_axis, name='conv1_scale')(x) 52 | x = Activation('relu', name='relu1')(x) 53 | x = ZeroPadding2D((1, 1), name='pool1_zeropadding')(x) 54 | x = MaxPooling2D((3, 3), strides=(2, 2), name='pool1')(x) 55 | 56 | # Add dense blocks 57 | for block_idx in range(nb_dense_block - 1): 58 | stage = block_idx+2 59 | x, nb_filter = dense_block(x, stage, nb_layers[block_idx], nb_filter, growth_rate, dropout_rate=dropout_rate, weight_decay=weight_decay) 60 | 61 | # Add transition_block 62 | x = transition_block(x, stage, nb_filter, compression=compression, dropout_rate=dropout_rate, weight_decay=weight_decay) 63 | nb_filter = int(nb_filter * compression) 64 | 65 | final_stage = stage + 1 66 | x, nb_filter = dense_block(x, final_stage, nb_layers[-1], nb_filter, growth_rate, dropout_rate=dropout_rate, weight_decay=weight_decay) 67 | 68 | x = BatchNormalization(epsilon=eps, axis=concat_axis, name='conv'+str(final_stage)+'_blk_bn')(x) 69 | x = Scale(axis=concat_axis, name='conv'+str(final_stage)+'_blk_scale')(x) 70 | x = Activation('relu', name='relu'+str(final_stage)+'_blk')(x) 71 | x = GlobalAveragePooling2D(name='pool'+str(final_stage))(x) 72 | 73 | x = Dense(classes, name='fc6')(x) 74 | x = Activation('softmax', name='prob')(x) 75 | 76 | model = Model(img_input, x, name='densenet') 77 | 78 | if weights_path is not None: 79 | model.load_weights(weights_path) 80 | 81 | return model 82 | 83 | 84 | def conv_block(x, stage, branch, nb_filter, dropout_rate=None, weight_decay=1e-4): 85 | '''Apply BatchNorm, Relu, bottleneck 1x1 Conv2D, 3x3 Conv2D, and option dropout 86 | # Arguments 87 | x: input tensor 88 | stage: index for dense block 89 | branch: layer index within each dense block 90 | nb_filter: number of filters 91 | dropout_rate: dropout rate 92 | weight_decay: weight decay factor 93 | ''' 94 | eps = 1.1e-5 95 | conv_name_base = 'conv' + str(stage) + '_' + str(branch) 96 | relu_name_base = 'relu' + str(stage) + '_' + str(branch) 97 | 98 | # 1x1 Convolution (Bottleneck layer) 99 | inter_channel = nb_filter * 4 100 | x = BatchNormalization(epsilon=eps, axis=concat_axis, name=conv_name_base+'_x1_bn')(x) 101 | x = Scale(axis=concat_axis, name=conv_name_base+'_x1_scale')(x) 102 | x = Activation('relu', name=relu_name_base+'_x1')(x) 103 | x = Convolution2D(inter_channel, 1, 1, name=conv_name_base+'_x1', bias=False)(x) 104 | 105 | if dropout_rate: 106 | x = Dropout(dropout_rate)(x) 107 | 108 | # 3x3 Convolution 109 | x = BatchNormalization(epsilon=eps, axis=concat_axis, name=conv_name_base+'_x2_bn')(x) 110 | x = Scale(axis=concat_axis, name=conv_name_base+'_x2_scale')(x) 111 | x = Activation('relu', name=relu_name_base+'_x2')(x) 112 | x = ZeroPadding2D((1, 1), name=conv_name_base+'_x2_zeropadding')(x) 113 | x = Convolution2D(nb_filter, 3, 3, name=conv_name_base+'_x2', bias=False)(x) 114 | 115 | if dropout_rate: 116 | x = Dropout(dropout_rate)(x) 117 | 118 | return x 119 | 120 | 121 | def transition_block(x, stage, nb_filter, compression=1.0, dropout_rate=None, weight_decay=1E-4): 122 | ''' Apply BatchNorm, 1x1 Convolution, averagePooling, optional compression, dropout 123 | # Arguments 124 | x: input tensor 125 | stage: index for dense block 126 | nb_filter: number of filters 127 | compression: calculated as 1 - reduction. Reduces the number of feature maps in the transition block. 128 | dropout_rate: dropout rate 129 | weight_decay: weight decay factor 130 | ''' 131 | 132 | eps = 1.1e-5 133 | conv_name_base = 'conv' + str(stage) + '_blk' 134 | relu_name_base = 'relu' + str(stage) + '_blk' 135 | pool_name_base = 'pool' + str(stage) 136 | 137 | x = BatchNormalization(epsilon=eps, axis=concat_axis, name=conv_name_base+'_bn')(x) 138 | x = Scale(axis=concat_axis, name=conv_name_base+'_scale')(x) 139 | x = Activation('relu', name=relu_name_base)(x) 140 | x = Convolution2D(int(nb_filter * compression), 1, 1, name=conv_name_base, bias=False)(x) 141 | 142 | if dropout_rate: 143 | x = Dropout(dropout_rate)(x) 144 | 145 | x = AveragePooling2D((2, 2), strides=(2, 2), name=pool_name_base)(x) 146 | 147 | return x 148 | 149 | 150 | def dense_block(x, stage, nb_layers, nb_filter, growth_rate, dropout_rate=None, weight_decay=1e-4, grow_nb_filters=True): 151 | ''' Build a dense_block where the output of each conv_block is fed to subsequent ones 152 | # Arguments 153 | x: input tensor 154 | stage: index for dense block 155 | nb_layers: the number of layers of conv_block to append to the model. 156 | nb_filter: number of filters 157 | growth_rate: growth rate 158 | dropout_rate: dropout rate 159 | weight_decay: weight decay factor 160 | grow_nb_filters: flag to decide to allow number of filters to grow 161 | ''' 162 | 163 | eps = 1.1e-5 164 | concat_feat = x 165 | 166 | for i in range(nb_layers): 167 | branch = i+1 168 | x = conv_block(concat_feat, stage, branch, growth_rate, dropout_rate, weight_decay) 169 | concat_feat = merge([concat_feat, x], mode='concat', concat_axis=concat_axis, name='concat_'+str(stage)+'_'+str(branch)) 170 | 171 | if grow_nb_filters: 172 | nb_filter += growth_rate 173 | 174 | return concat_feat, nb_filter -------------------------------------------------------------------------------- /a01_densenet_169.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | # Based on: https://github.com/flyyufelix/DenseNet-Keras 3 | __author__ = 'ZFTurbo: https://kaggle.com/zfturbo' 4 | 5 | from keras.models import Model 6 | from keras.layers import Input, merge, ZeroPadding2D 7 | from keras.layers.core import Dense, Dropout, Activation 8 | from keras.layers.convolutional import Convolution2D 9 | from keras.layers.pooling import AveragePooling2D, GlobalAveragePooling2D, MaxPooling2D 10 | from keras.layers.normalization import BatchNormalization 11 | import keras.backend as K 12 | 13 | from a00_custom_layers import Scale 14 | 15 | def DenseNet_169(nb_dense_block=4, growth_rate=32, nb_filter=64, reduction=0.0, dropout_rate=0.0, weight_decay=1e-4, classes=1000, weights_path=None): 16 | '''Instantiate the DenseNet architecture, 17 | # Arguments 18 | nb_dense_block: number of dense blocks to add to end 19 | growth_rate: number of filters to add per dense block 20 | nb_filter: initial number of filters 21 | reduction: reduction factor of transition blocks. 22 | dropout_rate: dropout rate 23 | weight_decay: weight decay factor 24 | classes: optional number of classes to classify images 25 | weights_path: path to pre-trained weights 26 | # Returns 27 | A Keras model instance. 28 | ''' 29 | eps = 1.1e-5 30 | 31 | # compute compression factor 32 | compression = 1.0 - reduction 33 | 34 | # Handle Dimension Ordering for different backends 35 | global concat_axis 36 | if K.image_dim_ordering() == 'tf': 37 | concat_axis = 3 38 | img_input = Input(shape=(224, 224, 3), name='data') 39 | else: 40 | concat_axis = 1 41 | img_input = Input(shape=(3, 224, 224), name='data') 42 | 43 | # From architecture for ImageNet (Table 1 in the paper) 44 | nb_filter = 64 45 | nb_layers = [6,12,32,32] # For DenseNet-169 46 | 47 | # Initial convolution 48 | x = ZeroPadding2D((3, 3), name='conv1_zeropadding')(img_input) 49 | x = Convolution2D(nb_filter, 7, 7, subsample=(2, 2), name='conv1', bias=False)(x) 50 | x = BatchNormalization(epsilon=eps, axis=concat_axis, name='conv1_bn')(x) 51 | x = Scale(axis=concat_axis, name='conv1_scale')(x) 52 | x = Activation('relu', name='relu1')(x) 53 | x = ZeroPadding2D((1, 1), name='pool1_zeropadding')(x) 54 | x = MaxPooling2D((3, 3), strides=(2, 2), name='pool1')(x) 55 | 56 | # Add dense blocks 57 | for block_idx in range(nb_dense_block - 1): 58 | stage = block_idx+2 59 | x, nb_filter = dense_block(x, stage, nb_layers[block_idx], nb_filter, growth_rate, dropout_rate=dropout_rate, weight_decay=weight_decay) 60 | 61 | # Add transition_block 62 | x = transition_block(x, stage, nb_filter, compression=compression, dropout_rate=dropout_rate, weight_decay=weight_decay) 63 | nb_filter = int(nb_filter * compression) 64 | 65 | final_stage = stage + 1 66 | x, nb_filter = dense_block(x, final_stage, nb_layers[-1], nb_filter, growth_rate, dropout_rate=dropout_rate, weight_decay=weight_decay) 67 | 68 | x = BatchNormalization(epsilon=eps, axis=concat_axis, name='conv'+str(final_stage)+'_blk_bn')(x) 69 | x = Scale(axis=concat_axis, name='conv'+str(final_stage)+'_blk_scale')(x) 70 | x = Activation('relu', name='relu'+str(final_stage)+'_blk')(x) 71 | x = GlobalAveragePooling2D(name='pool'+str(final_stage))(x) 72 | 73 | x = Dense(classes, name='fc6')(x) 74 | x = Activation('softmax', name='prob')(x) 75 | 76 | model = Model(img_input, x, name='densenet') 77 | 78 | if weights_path is not None: 79 | model.load_weights(weights_path) 80 | 81 | return model 82 | 83 | 84 | def conv_block(x, stage, branch, nb_filter, dropout_rate=None, weight_decay=1e-4): 85 | '''Apply BatchNorm, Relu, bottleneck 1x1 Conv2D, 3x3 Conv2D, and option dropout 86 | # Arguments 87 | x: input tensor 88 | stage: index for dense block 89 | branch: layer index within each dense block 90 | nb_filter: number of filters 91 | dropout_rate: dropout rate 92 | weight_decay: weight decay factor 93 | ''' 94 | eps = 1.1e-5 95 | conv_name_base = 'conv' + str(stage) + '_' + str(branch) 96 | relu_name_base = 'relu' + str(stage) + '_' + str(branch) 97 | 98 | # 1x1 Convolution (Bottleneck layer) 99 | inter_channel = nb_filter * 4 100 | x = BatchNormalization(epsilon=eps, axis=concat_axis, name=conv_name_base+'_x1_bn')(x) 101 | x = Scale(axis=concat_axis, name=conv_name_base+'_x1_scale')(x) 102 | x = Activation('relu', name=relu_name_base+'_x1')(x) 103 | x = Convolution2D(inter_channel, 1, 1, name=conv_name_base+'_x1', bias=False)(x) 104 | 105 | if dropout_rate: 106 | x = Dropout(dropout_rate)(x) 107 | 108 | # 3x3 Convolution 109 | x = BatchNormalization(epsilon=eps, axis=concat_axis, name=conv_name_base+'_x2_bn')(x) 110 | x = Scale(axis=concat_axis, name=conv_name_base+'_x2_scale')(x) 111 | x = Activation('relu', name=relu_name_base+'_x2')(x) 112 | x = ZeroPadding2D((1, 1), name=conv_name_base+'_x2_zeropadding')(x) 113 | x = Convolution2D(nb_filter, 3, 3, name=conv_name_base+'_x2', bias=False)(x) 114 | 115 | if dropout_rate: 116 | x = Dropout(dropout_rate)(x) 117 | 118 | return x 119 | 120 | 121 | def transition_block(x, stage, nb_filter, compression=1.0, dropout_rate=None, weight_decay=1E-4): 122 | ''' Apply BatchNorm, 1x1 Convolution, averagePooling, optional compression, dropout 123 | # Arguments 124 | x: input tensor 125 | stage: index for dense block 126 | nb_filter: number of filters 127 | compression: calculated as 1 - reduction. Reduces the number of feature maps in the transition block. 128 | dropout_rate: dropout rate 129 | weight_decay: weight decay factor 130 | ''' 131 | 132 | eps = 1.1e-5 133 | conv_name_base = 'conv' + str(stage) + '_blk' 134 | relu_name_base = 'relu' + str(stage) + '_blk' 135 | pool_name_base = 'pool' + str(stage) 136 | 137 | x = BatchNormalization(epsilon=eps, axis=concat_axis, name=conv_name_base+'_bn')(x) 138 | x = Scale(axis=concat_axis, name=conv_name_base+'_scale')(x) 139 | x = Activation('relu', name=relu_name_base)(x) 140 | x = Convolution2D(int(nb_filter * compression), 1, 1, name=conv_name_base, bias=False)(x) 141 | 142 | if dropout_rate: 143 | x = Dropout(dropout_rate)(x) 144 | 145 | x = AveragePooling2D((2, 2), strides=(2, 2), name=pool_name_base)(x) 146 | 147 | return x 148 | 149 | 150 | def dense_block(x, stage, nb_layers, nb_filter, growth_rate, dropout_rate=None, weight_decay=1e-4, grow_nb_filters=True): 151 | ''' Build a dense_block where the output of each conv_block is fed to subsequent ones 152 | # Arguments 153 | x: input tensor 154 | stage: index for dense block 155 | nb_layers: the number of layers of conv_block to append to the model. 156 | nb_filter: number of filters 157 | growth_rate: growth rate 158 | dropout_rate: dropout rate 159 | weight_decay: weight decay factor 160 | grow_nb_filters: flag to decide to allow number of filters to grow 161 | ''' 162 | 163 | eps = 1.1e-5 164 | concat_feat = x 165 | 166 | for i in range(nb_layers): 167 | branch = i+1 168 | x = conv_block(concat_feat, stage, branch, growth_rate, dropout_rate, weight_decay) 169 | concat_feat = merge([concat_feat, x], mode='concat', concat_axis=concat_axis, name='concat_'+str(stage)+'_'+str(branch)) 170 | 171 | if grow_nb_filters: 172 | nb_filter += growth_rate 173 | 174 | return concat_feat, nb_filter -------------------------------------------------------------------------------- /a01_inception_v4.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | # Based on: https://github.com/titu1994/Inception-v4/releases 3 | __author__ = 'ZFTurbo: https://kaggle.com/zfturbo' 4 | 5 | # Sys 6 | import warnings 7 | # Keras Core 8 | from keras.layers.convolutional import MaxPooling2D, Convolution2D, AveragePooling2D 9 | from keras.layers import Input, merge, Dropout, Dense, Flatten, Activation 10 | from keras.layers.normalization import BatchNormalization 11 | from keras.models import Model 12 | # Backend 13 | from keras import backend as K 14 | # Utils 15 | from keras.utils.layer_utils import convert_all_kernels_in_model 16 | from keras.utils.data_utils import get_file 17 | 18 | ######################################################################################### 19 | # Implements the Inception Network v4 (http://arxiv.org/pdf/1602.07261v1.pdf) in Keras. # 20 | ######################################################################################### 21 | 22 | TH_WEIGHTS_PATH = '../weights/inception-v4_weights_th_dim_ordering_th_kernels.h5' 23 | TF_WEIGHTS_PATH = '../weights/inception-v4_weights_tf_dim_ordering_tf_kernels.h5' 24 | 25 | 26 | def conv2d_bn(x, nb_filter, nb_row, nb_col, 27 | border_mode='same', subsample=(1, 1), bias=False): 28 | """ 29 | Utility function to apply conv + BN. 30 | (Slightly modified from https://github.com/fchollet/keras/blob/master/keras/applications/inception_v3.py) 31 | """ 32 | if K.image_dim_ordering() == "th": 33 | channel_axis = 1 34 | else: 35 | channel_axis = -1 36 | x = Convolution2D(nb_filter, nb_row, nb_col, 37 | subsample=subsample, 38 | border_mode=border_mode, 39 | bias=bias)(x) 40 | x = BatchNormalization(axis=channel_axis)(x) 41 | x = Activation('relu')(x) 42 | return x 43 | 44 | 45 | def block_inception_a(input): 46 | if K.image_dim_ordering() == "th": 47 | channel_axis = 1 48 | else: 49 | channel_axis = -1 50 | 51 | branch_0 = conv2d_bn(input, 96, 1, 1) 52 | 53 | branch_1 = conv2d_bn(input, 64, 1, 1) 54 | branch_1 = conv2d_bn(branch_1, 96, 3, 3) 55 | 56 | branch_2 = conv2d_bn(input, 64, 1, 1) 57 | branch_2 = conv2d_bn(branch_2, 96, 3, 3) 58 | branch_2 = conv2d_bn(branch_2, 96, 3, 3) 59 | 60 | branch_3 = AveragePooling2D((3, 3), strides=(1, 1), border_mode='same')(input) 61 | branch_3 = conv2d_bn(branch_3, 96, 1, 1) 62 | 63 | x = merge([branch_0, branch_1, branch_2, branch_3], mode='concat', concat_axis=channel_axis) 64 | return x 65 | 66 | 67 | def block_reduction_a(input): 68 | if K.image_dim_ordering() == "th": 69 | channel_axis = 1 70 | else: 71 | channel_axis = -1 72 | 73 | branch_0 = conv2d_bn(input, 384, 3, 3, subsample=(2, 2), border_mode='valid') 74 | 75 | branch_1 = conv2d_bn(input, 192, 1, 1) 76 | branch_1 = conv2d_bn(branch_1, 224, 3, 3) 77 | branch_1 = conv2d_bn(branch_1, 256, 3, 3, subsample=(2, 2), border_mode='valid') 78 | 79 | branch_2 = MaxPooling2D((3, 3), strides=(2, 2), border_mode='valid')(input) 80 | 81 | x = merge([branch_0, branch_1, branch_2], mode='concat', concat_axis=channel_axis) 82 | return x 83 | 84 | 85 | def block_inception_b(input): 86 | if K.image_dim_ordering() == "th": 87 | channel_axis = 1 88 | else: 89 | channel_axis = -1 90 | 91 | branch_0 = conv2d_bn(input, 384, 1, 1) 92 | 93 | branch_1 = conv2d_bn(input, 192, 1, 1) 94 | branch_1 = conv2d_bn(branch_1, 224, 1, 7) 95 | branch_1 = conv2d_bn(branch_1, 256, 7, 1) 96 | 97 | branch_2 = conv2d_bn(input, 192, 1, 1) 98 | branch_2 = conv2d_bn(branch_2, 192, 7, 1) 99 | branch_2 = conv2d_bn(branch_2, 224, 1, 7) 100 | branch_2 = conv2d_bn(branch_2, 224, 7, 1) 101 | branch_2 = conv2d_bn(branch_2, 256, 1, 7) 102 | 103 | branch_3 = AveragePooling2D((3, 3), strides=(1, 1), border_mode='same')(input) 104 | branch_3 = conv2d_bn(branch_3, 128, 1, 1) 105 | 106 | x = merge([branch_0, branch_1, branch_2, branch_3], mode='concat', concat_axis=channel_axis) 107 | return x 108 | 109 | 110 | def block_reduction_b(input): 111 | if K.image_dim_ordering() == "th": 112 | channel_axis = 1 113 | else: 114 | channel_axis = -1 115 | 116 | branch_0 = conv2d_bn(input, 192, 1, 1) 117 | branch_0 = conv2d_bn(branch_0, 192, 3, 3, subsample=(2, 2), border_mode='valid') 118 | 119 | branch_1 = conv2d_bn(input, 256, 1, 1) 120 | branch_1 = conv2d_bn(branch_1, 256, 1, 7) 121 | branch_1 = conv2d_bn(branch_1, 320, 7, 1) 122 | branch_1 = conv2d_bn(branch_1, 320, 3, 3, subsample=(2, 2), border_mode='valid') 123 | 124 | branch_2 = MaxPooling2D((3, 3), strides=(2, 2), border_mode='valid')(input) 125 | 126 | x = merge([branch_0, branch_1, branch_2], mode='concat', concat_axis=channel_axis) 127 | return x 128 | 129 | 130 | def block_inception_c(input): 131 | if K.image_dim_ordering() == "th": 132 | channel_axis = 1 133 | else: 134 | channel_axis = -1 135 | 136 | branch_0 = conv2d_bn(input, 256, 1, 1) 137 | 138 | branch_1 = conv2d_bn(input, 384, 1, 1) 139 | branch_10 = conv2d_bn(branch_1, 256, 1, 3) 140 | branch_11 = conv2d_bn(branch_1, 256, 3, 1) 141 | branch_1 = merge([branch_10, branch_11], mode='concat', concat_axis=channel_axis) 142 | 143 | branch_2 = conv2d_bn(input, 384, 1, 1) 144 | branch_2 = conv2d_bn(branch_2, 448, 3, 1) 145 | branch_2 = conv2d_bn(branch_2, 512, 1, 3) 146 | branch_20 = conv2d_bn(branch_2, 256, 1, 3) 147 | branch_21 = conv2d_bn(branch_2, 256, 3, 1) 148 | branch_2 = merge([branch_20, branch_21], mode='concat', concat_axis=channel_axis) 149 | 150 | branch_3 = AveragePooling2D((3, 3), strides=(1, 1), border_mode='same')(input) 151 | branch_3 = conv2d_bn(branch_3, 256, 1, 1) 152 | 153 | x = merge([branch_0, branch_1, branch_2, branch_3], mode='concat', concat_axis=channel_axis) 154 | return x 155 | 156 | 157 | def inception_v4_base(input): 158 | if K.image_dim_ordering() == "th": 159 | channel_axis = 1 160 | else: 161 | channel_axis = -1 162 | 163 | # Input Shape is 299 x 299 x 3 (th) or 3 x 299 x 299 (th) 164 | net = conv2d_bn(input, 32, 3, 3, subsample=(2, 2), border_mode='valid') 165 | net = conv2d_bn(net, 32, 3, 3, border_mode='valid') 166 | net = conv2d_bn(net, 64, 3, 3) 167 | 168 | branch_0 = MaxPooling2D((3, 3), strides=(2, 2), border_mode='valid')(net) 169 | 170 | branch_1 = conv2d_bn(net, 96, 3, 3, subsample=(2, 2), border_mode='valid') 171 | 172 | net = merge([branch_0, branch_1], mode='concat', concat_axis=channel_axis) 173 | 174 | branch_0 = conv2d_bn(net, 64, 1, 1) 175 | branch_0 = conv2d_bn(branch_0, 96, 3, 3, border_mode='valid') 176 | 177 | branch_1 = conv2d_bn(net, 64, 1, 1) 178 | branch_1 = conv2d_bn(branch_1, 64, 1, 7) 179 | branch_1 = conv2d_bn(branch_1, 64, 7, 1) 180 | branch_1 = conv2d_bn(branch_1, 96, 3, 3, border_mode='valid') 181 | 182 | net = merge([branch_0, branch_1], mode='concat', concat_axis=channel_axis) 183 | 184 | branch_0 = conv2d_bn(net, 192, 3, 3, subsample=(2, 2), border_mode='valid') 185 | branch_1 = MaxPooling2D((3, 3), strides=(2, 2), border_mode='valid')(net) 186 | 187 | net = merge([branch_0, branch_1], mode='concat', concat_axis=channel_axis) 188 | 189 | # 35 x 35 x 384 190 | # 4 x Inception-A blocks 191 | for idx in range(4): 192 | net = block_inception_a(net) 193 | 194 | # 35 x 35 x 384 195 | # Reduction-A block 196 | net = block_reduction_a(net) 197 | 198 | # 17 x 17 x 1024 199 | # 7 x Inception-B blocks 200 | for idx in range(7): 201 | net = block_inception_b(net) 202 | 203 | # 17 x 17 x 1024 204 | # Reduction-B block 205 | net = block_reduction_b(net) 206 | 207 | # 8 x 8 x 1536 208 | # 3 x Inception-C blocks 209 | for idx in range(3): 210 | net = block_inception_c(net) 211 | 212 | return net 213 | 214 | 215 | def inception_v4(num_classes, dropout_keep_prob, weights): 216 | ''' 217 | Creates the inception v4 network 218 | 219 | Args: 220 | num_classes: number of classes 221 | dropout_keep_prob: float, the fraction to keep before final layer. 222 | 223 | Returns: 224 | logits: the logits outputs of the model. 225 | ''' 226 | 227 | # Input Shape is 299 x 299 x 3 (tf) or 3 x 299 x 299 (th) 228 | if K.image_dim_ordering() == 'th': 229 | inputs = Input((3, 299, 299)) 230 | else: 231 | inputs = Input((299, 299, 3)) 232 | 233 | # Make inception base 234 | net = inception_v4_base(inputs) 235 | 236 | # Final pooling and prediction 237 | 238 | # 8 x 8 x 1536 239 | net = AveragePooling2D((8, 8), border_mode='valid')(net) 240 | 241 | # 1 x 1 x 1536 242 | net = Dropout(dropout_keep_prob)(net) 243 | net = Flatten()(net) 244 | 245 | # 1536 246 | predictions = Dense(output_dim=num_classes, activation='softmax')(net) 247 | 248 | model = Model(inputs, predictions, name='inception_v4') 249 | 250 | # load weights 251 | if weights == 'imagenet': 252 | if K.image_dim_ordering() == 'th': 253 | model.load_weights(TH_WEIGHTS_PATH, by_name=True) 254 | if K.backend() == 'tensorflow': 255 | warnings.warn('You are using the TensorFlow backend, yet you ' 256 | 'are using the Theano ' 257 | 'image dimension ordering convention ' 258 | '(`image_dim_ordering="th"`). ' 259 | 'For best performance, set ' 260 | '`image_dim_ordering="tf"` in ' 261 | 'your Keras config ' 262 | 'at ~/.keras/keras.json.') 263 | convert_all_kernels_in_model(model) 264 | else: 265 | model.load_weights(TF_WEIGHTS_PATH, by_name=True) 266 | if K.backend() == 'theano': 267 | convert_all_kernels_in_model(model) 268 | print("Loaded Model Weights!") 269 | 270 | return model 271 | 272 | -------------------------------------------------------------------------------- /a01_resnet_101.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | # Based on: https://gist.github.com/flyyufelix/65018873f8cb2bbe95f429c474aa1294 3 | __author__ = 'ZFTurbo: https://kaggle.com/zfturbo' 4 | 5 | from keras.layers import Input, Dense, Convolution2D, MaxPooling2D, AveragePooling2D, ZeroPadding2D, Dropout, Flatten, merge, Reshape, Activation, Lambda, GlobalAveragePooling2D, Merge 6 | from keras.optimizers import SGD 7 | from keras.layers.normalization import BatchNormalization 8 | from keras.models import Model 9 | from keras import backend as K 10 | from a00_custom_layers import Scale 11 | 12 | import sys 13 | sys.setrecursionlimit(3000) 14 | 15 | 16 | WEIGHTS_PATH = '../weights/' 17 | 18 | def identity_block(input_tensor, kernel_size, filters, stage, block): 19 | 20 | '''The identity_block is the block that has no conv layer at shortcut 21 | # Arguments 22 | input_tensor: input tensor 23 | kernel_size: defualt 3, the kernel size of middle conv layer at main path 24 | filters: list of integers, the nb_filters of 3 conv layer at main path 25 | stage: integer, current stage label, used for generating layer names 26 | block: 'a','b'..., current block label, used for generating layer names 27 | ''' 28 | eps = 1.1e-5 29 | 30 | nb_filter1, nb_filter2, nb_filter3 = filters 31 | conv_name_base = 'res' + str(stage) + block + '_branch' 32 | bn_name_base = 'bn' + str(stage) + block + '_branch' 33 | scale_name_base = 'scale' + str(stage) + block + '_branch' 34 | 35 | x = Convolution2D(nb_filter1, 1, 1, name=conv_name_base + '2a', bias=False)(input_tensor) 36 | x = BatchNormalization(epsilon=eps, axis=bn_axis, name=bn_name_base + '2a')(x) 37 | x = Scale(axis=bn_axis, name=scale_name_base + '2a')(x) 38 | x = Activation('relu', name=conv_name_base + '2a_relu')(x) 39 | 40 | x = ZeroPadding2D((1, 1), name=conv_name_base + '2b_zeropadding')(x) 41 | x = Convolution2D(nb_filter2, kernel_size, kernel_size, 42 | name=conv_name_base + '2b', bias=False)(x) 43 | x = BatchNormalization(epsilon=eps, axis=bn_axis, name=bn_name_base + '2b')(x) 44 | x = Scale(axis=bn_axis, name=scale_name_base + '2b')(x) 45 | x = Activation('relu', name=conv_name_base + '2b_relu')(x) 46 | 47 | x = Convolution2D(nb_filter3, 1, 1, name=conv_name_base + '2c', bias=False)(x) 48 | x = BatchNormalization(epsilon=eps, axis=bn_axis, name=bn_name_base + '2c')(x) 49 | x = Scale(axis=bn_axis, name=scale_name_base + '2c')(x) 50 | 51 | x = merge([x, input_tensor], mode='sum', name='res' + str(stage) + block) 52 | x = Activation('relu', name='res' + str(stage) + block + '_relu')(x) 53 | return x 54 | 55 | 56 | def conv_block(input_tensor, kernel_size, filters, stage, block, strides=(2, 2)): 57 | '''conv_block is the block that has a conv layer at shortcut 58 | # Arguments 59 | input_tensor: input tensor 60 | kernel_size: defualt 3, the kernel size of middle conv layer at main path 61 | filters: list of integers, the nb_filters of 3 conv layer at main path 62 | stage: integer, current stage label, used for generating layer names 63 | block: 'a','b'..., current block label, used for generating layer names 64 | Note that from stage 3, the first conv layer at main path is with subsample=(2,2) 65 | And the shortcut should have subsample=(2,2) as well 66 | ''' 67 | eps = 1.1e-5 68 | 69 | 70 | nb_filter1, nb_filter2, nb_filter3 = filters 71 | conv_name_base = 'res' + str(stage) + block + '_branch' 72 | bn_name_base = 'bn' + str(stage) + block + '_branch' 73 | scale_name_base = 'scale' + str(stage) + block + '_branch' 74 | 75 | x = Convolution2D(nb_filter1, 1, 1, subsample=strides, 76 | name=conv_name_base + '2a', bias=False)(input_tensor) 77 | x = BatchNormalization(epsilon=eps, axis=bn_axis, name=bn_name_base + '2a')(x) 78 | x = Scale(axis=bn_axis, name=scale_name_base + '2a')(x) 79 | x = Activation('relu', name=conv_name_base + '2a_relu')(x) 80 | 81 | x = ZeroPadding2D((1, 1), name=conv_name_base + '2b_zeropadding')(x) 82 | x = Convolution2D(nb_filter2, kernel_size, kernel_size, 83 | name=conv_name_base + '2b', bias=False)(x) 84 | x = BatchNormalization(epsilon=eps, axis=bn_axis, name=bn_name_base + '2b')(x) 85 | x = Scale(axis=bn_axis, name=scale_name_base + '2b')(x) 86 | x = Activation('relu', name=conv_name_base + '2b_relu')(x) 87 | 88 | x = Convolution2D(nb_filter3, 1, 1, name=conv_name_base + '2c', bias=False)(x) 89 | x = BatchNormalization(epsilon=eps, axis=bn_axis, name=bn_name_base + '2c')(x) 90 | x = Scale(axis=bn_axis, name=scale_name_base + '2c')(x) 91 | 92 | shortcut = Convolution2D(nb_filter3, 1, 1, subsample=strides, 93 | name=conv_name_base + '1', bias=False)(input_tensor) 94 | shortcut = BatchNormalization(epsilon=eps, axis=bn_axis, name=bn_name_base + '1')(shortcut) 95 | shortcut = Scale(axis=bn_axis, name=scale_name_base + '1')(shortcut) 96 | 97 | x = merge([x, shortcut], mode='sum', name='res' + str(stage) + block) 98 | x = Activation('relu', name='res' + str(stage) + block + '_relu')(x) 99 | return x 100 | 101 | 102 | def resnet101_model(weights_path=None): 103 | '''Instantiate the ResNet101 architecture, 104 | # Arguments 105 | weights_path: path to pretrained weight file 106 | # Returns 107 | A Keras model instance. 108 | ''' 109 | eps = 1.1e-5 110 | 111 | # Handle Dimension Ordering for different backends 112 | global bn_axis 113 | if K.image_dim_ordering() == 'tf': 114 | bn_axis = 3 115 | img_input = Input(shape=(224, 224, 3), name='data') 116 | else: 117 | bn_axis = 1 118 | img_input = Input(shape=(3, 224, 224), name='data') 119 | 120 | x = ZeroPadding2D((3, 3), name='conv1_zeropadding')(img_input) 121 | x = Convolution2D(64, 7, 7, subsample=(2, 2), name='conv1', bias=False)(x) 122 | x = BatchNormalization(epsilon=eps, axis=bn_axis, name='bn_conv1')(x) 123 | x = Scale(axis=bn_axis, name='scale_conv1')(x) 124 | x = Activation('relu', name='conv1_relu')(x) 125 | x = MaxPooling2D((3, 3), strides=(2, 2), name='pool1')(x) 126 | 127 | x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1)) 128 | x = identity_block(x, 3, [64, 64, 256], stage=2, block='b') 129 | x = identity_block(x, 3, [64, 64, 256], stage=2, block='c') 130 | 131 | x = conv_block(x, 3, [128, 128, 512], stage=3, block='a') 132 | for i in range(1, 3): 133 | x = identity_block(x, 3, [128, 128, 512], stage=3, block='b' + str(i)) 134 | 135 | x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a') 136 | for i in range(1, 23): 137 | x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b' + str(i)) 138 | 139 | x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a') 140 | x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b') 141 | x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c') 142 | 143 | x_fc = AveragePooling2D((7, 7), name='avg_pool')(x) 144 | x_fc = Flatten()(x_fc) 145 | x_fc = Dense(1000, activation='softmax', name='fc1000')(x_fc) 146 | 147 | model = Model(img_input, x_fc) 148 | 149 | # load weights 150 | if weights_path: 151 | model.load_weights(weights_path, by_name=True) 152 | 153 | return model 154 | 155 | 156 | def get_resnet101(classes_number, dim_ordering='th'): 157 | base_model = resnet101_model(WEIGHTS_PATH + 'resnet101_weights_th.h5') 158 | x = base_model.layers[-2].output 159 | del base_model.layers[-1:] 160 | x = Dense(classes_number, activation='sigmoid', name='predictions')(x) 161 | model = Model(input=base_model.input, output=x) 162 | return model 163 | -------------------------------------------------------------------------------- /a01_resnet_152.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from keras.layers import Input, Dense, Convolution2D, MaxPooling2D, AveragePooling2D, ZeroPadding2D, Dropout, Flatten, \ 4 | merge, Reshape, Activation, Lambda, GlobalAveragePooling2D, Merge 5 | from keras.optimizers import SGD 6 | from keras.layers.normalization import BatchNormalization 7 | from keras.models import Model 8 | from keras.engine import Layer, InputSpec 9 | from keras import backend as K 10 | from a00_custom_layers import Scale 11 | import sys 12 | 13 | 14 | sys.setrecursionlimit(5000) 15 | 16 | WEIGHTS_PATH = '../weights/' 17 | 18 | 19 | def identity_block(input_tensor, kernel_size, filters, stage, block): 20 | '''The identity_block is the block that has no conv layer at shortcut 21 | # Arguments 22 | input_tensor: input tensor 23 | kernel_size: default 3, the kernel size of middle conv layer at main path 24 | filters: list of integers, the nb_filters of 3 conv layer at main path 25 | stage: integer, current stage label, used for generating layer names 26 | block: 'a','b'..., current block label, used for generating layer names 27 | ''' 28 | eps = 1.1e-5 29 | nb_filter1, nb_filter2, nb_filter3 = filters 30 | conv_name_base = 'res' + str(stage) + block + '_branch' 31 | bn_name_base = 'bn' + str(stage) + block + '_branch' 32 | scale_name_base = 'scale' + str(stage) + block + '_branch' 33 | 34 | x = Convolution2D(nb_filter1, 1, 1, name=conv_name_base + '2a', bias=False)(input_tensor) 35 | x = BatchNormalization(epsilon=eps, axis=bn_axis, name=bn_name_base + '2a')(x) 36 | x = Scale(axis=bn_axis, name=scale_name_base + '2a')(x) 37 | x = Activation('relu', name=conv_name_base + '2a_relu')(x) 38 | 39 | x = ZeroPadding2D((1, 1), name=conv_name_base + '2b_zeropadding')(x) 40 | x = Convolution2D(nb_filter2, kernel_size, kernel_size, 41 | name=conv_name_base + '2b', bias=False)(x) 42 | x = BatchNormalization(epsilon=eps, axis=bn_axis, name=bn_name_base + '2b')(x) 43 | x = Scale(axis=bn_axis, name=scale_name_base + '2b')(x) 44 | x = Activation('relu', name=conv_name_base + '2b_relu')(x) 45 | 46 | x = Convolution2D(nb_filter3, 1, 1, name=conv_name_base + '2c', bias=False)(x) 47 | x = BatchNormalization(epsilon=eps, axis=bn_axis, name=bn_name_base + '2c')(x) 48 | x = Scale(axis=bn_axis, name=scale_name_base + '2c')(x) 49 | 50 | x = merge([x, input_tensor], mode='sum', name='res' + str(stage) + block) 51 | x = Activation('relu', name='res' + str(stage) + block + '_relu')(x) 52 | return x 53 | 54 | 55 | def conv_block(input_tensor, kernel_size, filters, stage, block, strides=(2, 2)): 56 | '''conv_block is the block that has a conv layer at shortcut 57 | # Arguments 58 | input_tensor: input tensor 59 | kernel_size: defualt 3, the kernel size of middle conv layer at main path 60 | filters: list of integers, the nb_filters of 3 conv layer at main path 61 | stage: integer, current stage label, used for generating layer names 62 | block: 'a','b'..., current block label, used for generating layer names 63 | Note that from stage 3, the first conv layer at main path is with subsample=(2,2) 64 | And the shortcut should have subsample=(2,2) as well 65 | ''' 66 | eps = 1.1e-5 67 | nb_filter1, nb_filter2, nb_filter3 = filters 68 | conv_name_base = 'res' + str(stage) + block + '_branch' 69 | bn_name_base = 'bn' + str(stage) + block + '_branch' 70 | scale_name_base = 'scale' + str(stage) + block + '_branch' 71 | 72 | x = Convolution2D(nb_filter1, 1, 1, subsample=strides, 73 | name=conv_name_base + '2a', bias=False)(input_tensor) 74 | x = BatchNormalization(epsilon=eps, axis=bn_axis, name=bn_name_base + '2a')(x) 75 | x = Scale(axis=bn_axis, name=scale_name_base + '2a')(x) 76 | x = Activation('relu', name=conv_name_base + '2a_relu')(x) 77 | 78 | x = ZeroPadding2D((1, 1), name=conv_name_base + '2b_zeropadding')(x) 79 | x = Convolution2D(nb_filter2, kernel_size, kernel_size, 80 | name=conv_name_base + '2b', bias=False)(x) 81 | x = BatchNormalization(epsilon=eps, axis=bn_axis, name=bn_name_base + '2b')(x) 82 | x = Scale(axis=bn_axis, name=scale_name_base + '2b')(x) 83 | x = Activation('relu', name=conv_name_base + '2b_relu')(x) 84 | 85 | x = Convolution2D(nb_filter3, 1, 1, name=conv_name_base + '2c', bias=False)(x) 86 | x = BatchNormalization(epsilon=eps, axis=bn_axis, name=bn_name_base + '2c')(x) 87 | x = Scale(axis=bn_axis, name=scale_name_base + '2c')(x) 88 | 89 | shortcut = Convolution2D(nb_filter3, 1, 1, subsample=strides, 90 | name=conv_name_base + '1', bias=False)(input_tensor) 91 | shortcut = BatchNormalization(epsilon=eps, axis=bn_axis, name=bn_name_base + '1')(shortcut) 92 | shortcut = Scale(axis=bn_axis, name=scale_name_base + '1')(shortcut) 93 | 94 | x = merge([x, shortcut], mode='sum', name='res' + str(stage) + block) 95 | x = Activation('relu', name='res' + str(stage) + block + '_relu')(x) 96 | return x 97 | 98 | 99 | def resnet152_model(img_rows, img_cols, color_type=1, num_classes=None): 100 | """ 101 | Resnet 152 Model for Keras 102 | 103 | Model Schema and layer naming follow that of the original Caffe implementation 104 | https://github.com/KaimingHe/deep-residual-networks 105 | 106 | ImageNet Pretrained Weights 107 | Theano: https://drive.google.com/file/d/0Byy2AcGyEVxfZHhUT3lWVWxRN28/view?usp=sharing 108 | TensorFlow: https://drive.google.com/file/d/0Byy2AcGyEVxfeXExMzNNOHpEODg/view?usp=sharing 109 | 110 | Parameters: 111 | img_rows, img_cols - resolution of inputs 112 | channel - 1 for grayscale, 3 for color 113 | num_classes - number of class labels for our classification task 114 | """ 115 | eps = 1.1e-5 116 | 117 | # Handle Dimension Ordering for different backends 118 | global bn_axis 119 | if K.image_dim_ordering() == 'tf': 120 | bn_axis = 3 121 | img_input = Input(shape=(img_rows, img_cols, color_type), name='data') 122 | else: 123 | bn_axis = 1 124 | img_input = Input(shape=(color_type, img_rows, img_cols), name='data') 125 | 126 | x = ZeroPadding2D((3, 3), name='conv1_zeropadding')(img_input) 127 | x = Convolution2D(64, 7, 7, subsample=(2, 2), name='conv1', bias=False)(x) 128 | x = BatchNormalization(epsilon=eps, axis=bn_axis, name='bn_conv1')(x) 129 | x = Scale(axis=bn_axis, name='scale_conv1')(x) 130 | x = Activation('relu', name='conv1_relu')(x) 131 | x = MaxPooling2D((3, 3), strides=(2, 2), name='pool1')(x) 132 | 133 | x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1)) 134 | x = identity_block(x, 3, [64, 64, 256], stage=2, block='b') 135 | x = identity_block(x, 3, [64, 64, 256], stage=2, block='c') 136 | 137 | x = conv_block(x, 3, [128, 128, 512], stage=3, block='a') 138 | for i in range(1,8): 139 | x = identity_block(x, 3, [128, 128, 512], stage=3, block='b'+str(i)) 140 | 141 | x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a') 142 | for i in range(1,36): 143 | x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b'+str(i)) 144 | 145 | x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a') 146 | x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b') 147 | x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c') 148 | 149 | x_fc = AveragePooling2D((7, 7), name='avg_pool')(x) 150 | x_fc = Flatten()(x_fc) 151 | x_fc = Dense(1000, activation='softmax', name='fc1000')(x_fc) 152 | 153 | model = Model(img_input, x_fc) 154 | 155 | if K.image_dim_ordering() == 'th': 156 | # Use pre-trained weights for Theano backend 157 | weights_path = WEIGHTS_PATH + 'resnet152_weights_th.h5' 158 | else: 159 | # Use pre-trained weights for Tensorflow backend 160 | weights_path = WEIGHTS_PATH + 'resnet152_weights_tf.h5' 161 | 162 | model.load_weights(weights_path, by_name=True) 163 | 164 | return model 165 | 166 | 167 | def get_resnet152(classes_number, dim_ordering='th'): 168 | # One Dense - 0.09372 169 | base_model = resnet152_model(224, 224, 3, 17) 170 | x = base_model.layers[-2].output 171 | del base_model.layers[-1:] 172 | x = Dense(4096, activation='relu')(x) 173 | x = Dropout(0.5)(x) 174 | x = Dense(4096, activation='relu')(x) 175 | x = Dropout(0.5)(x) 176 | x = Dense(classes_number, activation='sigmoid', name='predictions')(x) 177 | model = Model(input=base_model.input, output=x) 178 | print(model.summary()) 179 | return model 180 | -------------------------------------------------------------------------------- /a01_squeezenet.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | # Based on: https://github.com/rcmalli/keras-squeezenet 3 | __author__ = 'ZFTurbo: https://kaggle.com/zfturbo' 4 | 5 | from keras.layers import Input, merge 6 | from keras.layers.convolutional import Convolution2D, MaxPooling2D 7 | from keras.layers.core import Dropout, Activation 8 | from keras.layers.pooling import GlobalAveragePooling2D 9 | from keras.models import Model 10 | 11 | WEIGHTS_PATH = '../weights/' 12 | 13 | sq1x1 = "squeeze1x1" 14 | exp1x1 = "expand1x1" 15 | exp3x3 = "expand3x3" 16 | relu = "relu_" 17 | 18 | # Modular function for Fire Node 19 | 20 | def fire_module(x, fire_id, squeeze=16, expand=64, dim_ordering='th'): 21 | s_id = 'fire' + str(fire_id) + '/' 22 | if dim_ordering is 'tf': 23 | c_axis = 3 24 | else: 25 | c_axis = 1 26 | 27 | x = Convolution2D(squeeze, 1, 1, border_mode='valid', name=s_id + sq1x1)(x) 28 | x = Activation('relu', name=s_id + relu + sq1x1)(x) 29 | 30 | left = Convolution2D(expand, 1, 1, border_mode='valid', name=s_id + exp1x1)(x) 31 | left = Activation('relu', name=s_id + relu + exp1x1)(left) 32 | 33 | right = Convolution2D(expand, 3, 3, border_mode='same', name=s_id + exp3x3)(x) 34 | right = Activation('relu', name=s_id + relu + exp3x3)(right) 35 | 36 | x = merge([left, right], mode='concat', concat_axis=c_axis, name=s_id + 'concat') 37 | return x 38 | 39 | 40 | # Original SqueezeNet from paper. 41 | def get_squeezenet_top(dim_ordering='th'): 42 | if dim_ordering is 'th': 43 | input_img = Input(shape=(3, 227, 227)) 44 | elif dim_ordering is 'tf': 45 | input_img = Input(shape=(227, 227, 3)) 46 | else: 47 | raise NotImplementedError("Theano and Tensorflow are only available") 48 | x = Convolution2D(64, 3, 3, subsample=(2, 2), border_mode='valid', name='conv1')(input_img) 49 | x = Activation('relu', name='relu_conv1')(x) 50 | x = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), name='pool1')(x) 51 | 52 | x = fire_module(x, fire_id=2, squeeze=16, expand=64, dim_ordering=dim_ordering) 53 | x = fire_module(x, fire_id=3, squeeze=16, expand=64, dim_ordering=dim_ordering) 54 | x = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), name='pool3')(x) 55 | 56 | x = fire_module(x, fire_id=4, squeeze=32, expand=128, dim_ordering=dim_ordering) 57 | x = fire_module(x, fire_id=5, squeeze=32, expand=128, dim_ordering=dim_ordering) 58 | x = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), name='pool5')(x) 59 | 60 | x = fire_module(x, fire_id=6, squeeze=48, expand=192, dim_ordering=dim_ordering) 61 | x = fire_module(x, fire_id=7, squeeze=48, expand=192, dim_ordering=dim_ordering) 62 | x = fire_module(x, fire_id=8, squeeze=64, expand=256, dim_ordering=dim_ordering) 63 | x = fire_module(x, fire_id=9, squeeze=64, expand=256, dim_ordering=dim_ordering) 64 | x = Dropout(0.5, name='drop9')(x) 65 | model = Model(input=input_img, output=[x]) 66 | model.load_weights(WEIGHTS_PATH + 'squeezenet_weights_th_dim_ordering_th_kernels.h5', by_name=True) 67 | return model 68 | 69 | 70 | def get_squeezenet(nb_classes, dim_ordering='th'): 71 | import keras.backend as K 72 | 73 | base_model = get_squeezenet_top(K.image_dim_ordering()) 74 | x = base_model.layers[-1].output 75 | x = Convolution2D(nb_classes, 1, 1, border_mode='valid', name='conv10')(x) 76 | x = Activation('relu', name='relu_conv10')(x) 77 | x = GlobalAveragePooling2D()(x) 78 | out = Activation('sigmoid', name='loss')(x) 79 | model = Model(input=base_model.input, output=[out]) 80 | return model 81 | 82 | 83 | if __name__ == '__main__': 84 | model = get_squeezenet(1000, dim_ordering='th') 85 | # model.compile(loss="categorical_crossentropy", optimizer="adam") 86 | 87 | -------------------------------------------------------------------------------- /a30_create_keras_models.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | __author__ = 'ZFTurbo: https://kaggle.com/zfturbo' 3 | 4 | import platform 5 | import os 6 | import sys 7 | import glob 8 | import time 9 | import cv2 10 | import pandas as pd 11 | from sklearn.metrics import fbeta_score 12 | from sklearn.model_selection import KFold 13 | from collections import Counter, defaultdict 14 | 15 | GPU_TO_USE = 0 16 | USE_THEANO = 1 17 | 18 | # Uncomment if you need to calculate specific fold 19 | # FOLD_TO_CALC = [5] 20 | 21 | if USE_THEANO: 22 | os.environ["KERAS_BACKEND"] = "theano" 23 | os.environ["THEANO_FLAGS"] = "device=gpu{},lib.cnmem=0.81".format(GPU_TO_USE, GPU_TO_USE) 24 | else: 25 | os.environ["KERAS_BACKEND"] = "tensorflow" 26 | os.environ["CUDA_VISIBLE_DEVICES"] = "{}".format(GPU_TO_USE) 27 | 28 | 29 | import datetime 30 | import shutil 31 | import random 32 | from a02_zoo import * 33 | 34 | random.seed(2016) 35 | np.random.seed(2016) 36 | 37 | CLASSES_NUMBER = 17 38 | PATIENCE = 50 39 | NB_EPOCH = 1000 40 | MAX_IMAGES_FOR_INFERENCE = 12000 # Increase if you have much of memory 41 | RESTORE_FROM_LAST_CHECKPOINT = 0 42 | UPDATE_BEST_MODEL = 0 43 | RECREATE_MODELS = 0 44 | 45 | INPUT_PATH = "../input/" 46 | MODELS_PATH = '../models/' 47 | if not os.path.isdir(MODELS_PATH): 48 | os.mkdir(MODELS_PATH) 49 | OUTPUT_PATH = "../subm/" 50 | if not os.path.isdir(OUTPUT_PATH): 51 | os.mkdir(OUTPUT_PATH) 52 | CODE_COPY_FOLDER = "../models/code/" 53 | if not os.path.isdir(CODE_COPY_FOLDER): 54 | os.mkdir(CODE_COPY_FOLDER) 55 | HISTORY_FOLDER_PATH = "../models/history/" 56 | if not os.path.isdir(HISTORY_FOLDER_PATH): 57 | os.mkdir(HISTORY_FOLDER_PATH) 58 | 59 | 60 | FULL_IMAGE_ARRAY = dict() 61 | def prepread_images(): 62 | files = glob.glob(INPUT_PATH + "train-jpg/*.jpg") 63 | total = 0 64 | for f in files: 65 | FULL_IMAGE_ARRAY[os.path.basename(f)] = cv2.imread(f) 66 | total += 1 67 | if total % 5000 == 0: 68 | print('Read {} files from {}...'.format(total, len(files))) 69 | 70 | 71 | def random_intensity_change(img, max_change): 72 | for j in range(3): 73 | mn = img[:, :, j].min() 74 | mx = img[:, :, j].max() 75 | new_min = mn + random.randint(-max_change, max_change) 76 | if new_min < 0: 77 | new_min = 0 78 | new_max = mx + random.randint(-max_change, max_change) 79 | if new_max > 255: 80 | new_max = 255 81 | # Rescale channel 82 | img[:, :, j] = np.round(new_min + (new_max - new_min) * (img[:, :, j].astype(np.float32) - mn) / (mx - mn)) 83 | return img.astype(np.uint8) 84 | 85 | 86 | def batch_generator_train(cnn_type, files, labels, augment=False): 87 | import keras.backend as K 88 | global FULL_IMAGE_ARRAY 89 | 90 | dim_ordering = K.image_dim_ordering() 91 | in_shape = get_input_shape(cnn_type) 92 | batch_size = get_batch_size(cnn_type) 93 | if len(FULL_IMAGE_ARRAY) == 0: 94 | prepread_images() 95 | 96 | while True: 97 | index = random.sample(range(len(files)), batch_size) 98 | batch_files = files[index] 99 | batch_labels = labels[index] 100 | 101 | image_list = [] 102 | mask_list = [] 103 | for i in range(len(batch_files)): 104 | # image = cv2.imread(batch_files[i]) 105 | image = FULL_IMAGE_ARRAY[os.path.basename(batch_files[i])] 106 | 107 | if cnn_type == 'INCEPTION_V3' or cnn_type == 'INCEPTION_V4' or cnn_type == 'XCEPTION': 108 | random_border = 20 109 | start0 = random.randint(0, random_border) 110 | start1 = random.randint(0, random_border) 111 | end0 = random.randint(0, random_border) 112 | end1 = random.randint(0, random_border) 113 | image = image[start0:image.shape[0] - end0, start1:image.shape[1] - end1] 114 | image = cv2.resize(image, (299, 299), cv2.INTER_LANCZOS4) 115 | else: 116 | box_size = random.randint(200, 256) 117 | start0 = random.randint(0, image.shape[0] - box_size) 118 | start1 = random.randint(0, image.shape[1] - box_size) 119 | image = image[start0:start0 + box_size, start1:start1 + box_size] 120 | image = cv2.resize(image, in_shape, cv2.INTER_LANCZOS4) 121 | 122 | if augment: 123 | # all possible mirroring and flips 124 | # (in total there are only 8 possible configurations) 125 | mirror = random.randint(0, 1) 126 | if mirror == 1: 127 | # flipud 128 | image = image[::-1, :, :] 129 | angle = random.randint(0, 3) 130 | if angle != 0: 131 | image = np.rot90(image, k=angle) 132 | 133 | # image = random_intensity_change(image, 10) 134 | 135 | mask = batch_labels[i] 136 | image_list.append(image.astype(np.float32)) 137 | mask_list.append(mask) 138 | image_list = np.array(image_list) 139 | image_list = image_list.transpose((0, 3, 1, 2)) 140 | image_list = preprocess_input_overall(cnn_type, image_list) 141 | if dim_ordering == 'tf': 142 | image_list = image_list.transpose((0, 2, 3, 1)) 143 | mask_list = np.array(mask_list) 144 | yield image_list, mask_list 145 | 146 | 147 | def train_single_model(num_fold, cnn_type, train_files, valid_files, train_labels, valid_labels): 148 | from keras.callbacks import EarlyStopping, ModelCheckpoint 149 | 150 | print('Creating and compiling model [{}]...'.format(cnn_type)) 151 | model = get_pretrained_model(cnn_type, CLASSES_NUMBER) 152 | 153 | final_model_path = MODELS_PATH + '{}_fold_{}.h5'.format(cnn_type, num_fold) 154 | cache_model_path = MODELS_PATH + '{}_temp_fold_{}.h5'.format(cnn_type, num_fold) 155 | if os.path.isfile(final_model_path) and RECREATE_MODELS == 0: 156 | print('Model {} already exists. Skip it'.format(final_model_path)) 157 | return 0.0 158 | if os.path.isfile(cache_model_path) and RESTORE_FROM_LAST_CHECKPOINT: 159 | print('Load model from last point: ', cache_model_path) 160 | model.load_weights(cache_model_path) 161 | elif os.path.isfile(final_model_path) and UPDATE_BEST_MODEL: 162 | print('Load model from best point: ', final_model_path) 163 | model.load_weights(final_model_path) 164 | else: 165 | print('Start training from begining') 166 | 167 | print('Fitting model...') 168 | batch_size = get_batch_size(cnn_type) 169 | print('Batch size: {}'.format(batch_size)) 170 | print('Learning rate: {}'.format(get_learning_rate(cnn_type))) 171 | samples_train_per_epoch = batch_size * (1 + len(train_files) // (10 * batch_size)) 172 | samples_valid_per_epoch = samples_train_per_epoch 173 | print('Samples train: {}, Samples valid: {}'.format(samples_train_per_epoch, samples_valid_per_epoch)) 174 | 175 | callbacks = [ 176 | EarlyStopping(monitor='val_loss', patience=PATIENCE, verbose=0), 177 | ModelCheckpoint(cache_model_path, monitor='val_loss', save_best_only=True, verbose=0), 178 | ] 179 | 180 | history = model.fit_generator(generator=batch_generator_train(cnn_type, train_files, train_labels, True), 181 | nb_epoch=NB_EPOCH, 182 | samples_per_epoch=samples_train_per_epoch, 183 | validation_data=batch_generator_train(cnn_type, valid_files, valid_labels, True), 184 | nb_val_samples=samples_valid_per_epoch, 185 | verbose=2, max_q_size=300, 186 | callbacks=callbacks) 187 | 188 | min_loss = min(history.history['val_loss']) 189 | print('Minimum loss for given fold: ', min_loss) 190 | model.load_weights(cache_model_path) 191 | model.save(final_model_path) 192 | now = datetime.datetime.now() 193 | filename = HISTORY_FOLDER_PATH + 'history_{}_{}_{:.4f}_lr_{}_{}.csv'.format(cnn_type, num_fold, min_loss, get_learning_rate(cnn_type), now.strftime("%Y-%m-%d-%H-%M")) 194 | pd.DataFrame(history.history).to_csv(filename, index=False) 195 | return min_loss 196 | 197 | 198 | def run_cross_validation_create_models(nfolds, cnn_type): 199 | global FOLD_TO_CALC 200 | tbl = pd.read_csv(INPUT_PATH + "train_v2.csv") 201 | labels = tbl['tags'].apply(lambda x: x.split(' ')) 202 | counts = defaultdict(int) 203 | for l in labels: 204 | for l2 in l: 205 | counts[l2] += 1 206 | indexes = sorted(list(counts.keys())) 207 | for i in range(len(indexes)): 208 | tbl['label_{}'.format(i)] = 0 209 | 210 | files = [] 211 | for id in tbl['image_name'].values: 212 | files.append(INPUT_PATH + "train-jpg/" + id + '.jpg') 213 | files = np.array(files) 214 | 215 | lbl = np.zeros((len(labels), len(indexes))) 216 | for j in range(len(labels)): 217 | l = labels[j] 218 | for i in range(len(indexes)): 219 | if indexes[i] in l: 220 | lbl[j][i] = 1 221 | 222 | # print(len(tbl)) 223 | # print(len(labels)) 224 | print('Labels shape:', lbl.shape) 225 | 226 | kf = KFold(n_splits=nfolds, shuffle=True, random_state=get_random_state(cnn_type)) 227 | num_fold = 0 228 | sum_score = 0 229 | for train_ids, valid_ids in kf.split(range(len(files))): 230 | num_fold += 1 231 | print('Start KFold number {} from {}'.format(num_fold, nfolds)) 232 | print('Split train: ', len(train_ids)) 233 | print('Split valid: ', len(valid_ids)) 234 | train_files = files[train_ids] 235 | valid_files = files[valid_ids] 236 | train_labels = lbl[train_ids] 237 | valid_labels = lbl[valid_ids] 238 | 239 | if 'FOLD_TO_CALC' in globals(): 240 | if num_fold not in FOLD_TO_CALC: 241 | continue 242 | 243 | score = train_single_model(num_fold, cnn_type, train_files, valid_files, train_labels, valid_labels) 244 | sum_score += score 245 | 246 | print('Avg loss: {}'.format(sum_score/nfolds)) 247 | return sum_score/nfolds 248 | 249 | # Note: 'RESNET152' and 'XCEPTION' can only be trained on Tensorflow backend! Other nets were trained on Theano backend 250 | # Check ../models/history/ folder for information about training process of already created models 251 | 252 | if __name__ == '__main__': 253 | num_folds = 5 254 | score1 = '' 255 | for cnn in ['INCEPTION_V3_DENSE_LAYERS', 'INCEPTION_V4', 'DENSENET_121', 'DENSENET_169', 'DENSENET_161', 256 | 'RESNET50_DENSE_LAYERS', 'RESNET101', 'VGG16', 'VGG19', 'RESNET152', 'XCEPTION']: 257 | score1 = run_cross_validation_create_models(num_folds, cnn) 258 | 259 | ''' 260 | Results of training: 261 | 262 | INCEPTION_V3_DENSE_LAYERS: 263 | Minimum loss for given fold: 0.0958579041777 264 | Minimum loss for given fold: 0.0930424076448 265 | Minimum loss for given fold: 0.0908006662158 266 | Minimum loss for given fold: 0.091942720989 267 | Minimum loss for given fold: 0.0911352778268 268 | Avg loss: 0.09255579537079658 269 | 270 | INCEPTION_V4 271 | Fold 1: 0.0877 272 | Fold 2: 0.0899 273 | Fold 3: 0.0873 274 | Fold 4: 0.0858307101844 275 | Fold 5: 0.0877363304435 276 | 277 | DENSENET_121 278 | Fold 1: 0.08790 279 | Fold 2: 0.08980 280 | Fold 3: 0.08564 281 | Fold 4: 0.08473 282 | Fold 5: 0.08758 283 | 284 | DENSENET_169 285 | Fold 1: 0.0877 286 | Fold 2: 0.0903 287 | Fold 3: 0.0878 288 | Fold 4: 0.0869 289 | Fold 5: 0.0863076225841 290 | 291 | DENSENET_161 292 | Fold 1: 0.0878 293 | Fold 2: 0.0897 294 | Fold 3: 0.0863 295 | Fold 4: 0.0865649292935 296 | Fold 5: 0.0866663176911 297 | 298 | RESNET50_DENSE_LAYERS 299 | Minimum loss for given fold: 0.088957440614 300 | Minimum loss for given fold: 0.0924427854127 301 | Minimum loss for given fold: 0.0871875942397 302 | Minimum loss for given fold: 0.0866941221426 303 | Minimum loss for given fold: 0.0883357608116 304 | Avg loss: 0.08872354064412694 305 | 306 | RESNET101 307 | Minimum loss for given fold: 0.0904679917903 308 | Minimum loss for given fold: 0.0917911478146 309 | Minimum loss for given fold: 0.0916355458077 310 | Minimum loss for given fold: 0.0891898434816 311 | Minimum loss for given fold: 0.0911772816706 312 | Avg loss: 0.09019579717912424 313 | 314 | XCEPTION 315 | Fold 1: 0.0867 316 | Fold 2: 0.0880 317 | Fold 3: 0.0846485815436 318 | Fold 4: 0.0856245925167 319 | Fold 5: 0.0850592969031 320 | 321 | RESNET152 322 | Fold 1: 0.0897 323 | Fold 2: 0.0911 324 | Fold 3: 0.0899847463562 325 | Fold 4: 0.0894 326 | Fold 5: 0.0890259628042 327 | 328 | VGG16 329 | Fold 1: 0.0885667443436 330 | Fold 2: 0.0911332166719 331 | Fold 3: 0.0852107660569 332 | Fold 4: 0.0856000116716 333 | Fold 5: 0.088289125765 334 | Avg loss: 0.0877599729018079 335 | 336 | VGG19 337 | Minimum loss for given fold: 0.0876369641887 338 | Minimum loss for given fold: 0.0916579335951 339 | Minimum loss for given fold: 0.0848985929617 340 | Minimum loss for given fold: 0.085898459866 341 | Minimum loss for given fold: 0.087161242456 342 | Avg loss: 0.08745063861349114 343 | ''' 344 | -------------------------------------------------------------------------------- /a30_create_keras_models_land.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | __author__ = 'ZFTurbo: https://kaggle.com/zfturbo' 3 | 4 | import os 5 | import glob 6 | import cv2 7 | import pandas as pd 8 | from sklearn.model_selection import KFold 9 | from a00_common_functions import get_train_label_matrix, get_indexes 10 | 11 | GPU_TO_USE = 0 12 | USE_THEANO = 1 13 | 14 | # Uncomment if you need to calculate specific fold 15 | # FOLD_TO_CALC = [5] 16 | 17 | if USE_THEANO: 18 | os.environ["KERAS_BACKEND"] = "theano" 19 | os.environ["THEANO_FLAGS"] = "device=gpu{},lib.cnmem=0.81".format(GPU_TO_USE, GPU_TO_USE) 20 | else: 21 | os.environ["KERAS_BACKEND"] = "tensorflow" 22 | os.environ["CUDA_VISIBLE_DEVICES"] = "{}".format(GPU_TO_USE) 23 | 24 | 25 | import datetime 26 | import random 27 | from a02_zoo import * 28 | 29 | random.seed(2016) 30 | np.random.seed(2016) 31 | 32 | PATIENCE = 50 33 | NB_EPOCH = 1000 34 | RESTORE_FROM_LAST_CHECKPOINT = 0 35 | UPDATE_BEST_MODEL = 0 36 | RECREATE_MODELS = 0 37 | 38 | 39 | INPUT_PATH = "../input/" 40 | MODELS_PATH = '../models/' 41 | if not os.path.isdir(MODELS_PATH): 42 | os.mkdir(MODELS_PATH) 43 | OUTPUT_PATH = "../subm/" 44 | if not os.path.isdir(OUTPUT_PATH): 45 | os.mkdir(OUTPUT_PATH) 46 | CODE_COPY_FOLDER = "../models/code/" 47 | if not os.path.isdir(CODE_COPY_FOLDER): 48 | os.mkdir(CODE_COPY_FOLDER) 49 | HISTORY_FOLDER_PATH = "../models/history/" 50 | if not os.path.isdir(HISTORY_FOLDER_PATH): 51 | os.mkdir(HISTORY_FOLDER_PATH) 52 | 53 | 54 | FULL_IMAGE_ARRAY = dict() 55 | def prepread_images(): 56 | files = glob.glob(INPUT_PATH + "train-jpg/*.jpg") 57 | total = 0 58 | for f in files: 59 | FULL_IMAGE_ARRAY[os.path.basename(f)] = cv2.imread(f) 60 | total += 1 61 | if total % 5000 == 0: 62 | print('Read {} files from {}...'.format(total, len(files))) 63 | 64 | 65 | def random_intensity_change(img, max_change): 66 | img = img.astype(np.int16) 67 | for j in range(3): 68 | delta = random.randint(-max_change, max_change) 69 | img[:, :, j] += delta 70 | img[img < 0] = 0 71 | img[img > 255] = 255 72 | return img.astype(np.uint8) 73 | 74 | 75 | def batch_generator_train(cnn_type, files, labels, augment=False): 76 | import keras.backend as K 77 | global FULL_IMAGE_ARRAY 78 | 79 | dim_ordering = K.image_dim_ordering() 80 | in_shape = get_input_shape(cnn_type) 81 | batch_size = get_batch_size(cnn_type) 82 | if 0: 83 | if len(FULL_IMAGE_ARRAY) == 0: 84 | prepread_images() 85 | 86 | while True: 87 | index = random.sample(range(len(files)), batch_size) 88 | batch_files = files[index] 89 | batch_labels = labels[index] 90 | 91 | image_list = [] 92 | mask_list = [] 93 | for i in range(len(batch_files)): 94 | image = cv2.imread(batch_files[i]) 95 | # image = FULL_IMAGE_ARRAY[os.path.basename(batch_files[i])] 96 | 97 | if cnn_type == 'INCEPTION_V3' or cnn_type == 'INCEPTION_V4' or cnn_type == 'XCEPTION': 98 | random_border = 20 99 | start0 = random.randint(0, random_border) 100 | start1 = random.randint(0, random_border) 101 | end0 = random.randint(0, random_border) 102 | end1 = random.randint(0, random_border) 103 | image = image[start0:image.shape[0] - end0, start1:image.shape[1] - end1] 104 | image = cv2.resize(image, (299, 299), cv2.INTER_LANCZOS4) 105 | else: 106 | box_size = random.randint(200, 256) 107 | start0 = random.randint(0, image.shape[0] - box_size) 108 | start1 = random.randint(0, image.shape[1] - box_size) 109 | image = image[start0:start0 + box_size, start1:start1 + box_size] 110 | image = cv2.resize(image, in_shape, cv2.INTER_LANCZOS4) 111 | 112 | if augment: 113 | # all possible mirroring and flips 114 | # (in total there are only 8 possible configurations) 115 | mirror = random.randint(0, 1) 116 | if mirror == 1: 117 | # flipud 118 | image = image[::-1, :, :] 119 | angle = random.randint(0, 3) 120 | if angle != 0: 121 | image = np.rot90(image, k=angle) 122 | image = random_intensity_change(image, 3) 123 | 124 | mask = batch_labels[i] 125 | image_list.append(image.astype(np.float32)) 126 | mask_list.append(mask) 127 | image_list = np.array(image_list) 128 | image_list = image_list.transpose((0, 3, 1, 2)) 129 | image_list = preprocess_input_overall(cnn_type, image_list) 130 | if dim_ordering == 'tf': 131 | image_list = image_list.transpose((0, 2, 3, 1)) 132 | mask_list = np.array(mask_list) 133 | yield image_list, mask_list 134 | 135 | 136 | def train_single_model(num_fold, cnn_type, train_files, valid_files, train_labels, valid_labels): 137 | from keras.callbacks import EarlyStopping, ModelCheckpoint 138 | 139 | print('Creating and compiling model [{}]...'.format(cnn_type)) 140 | model = get_pretrained_model(cnn_type, 13, final_layer_activation='sigmoid') 141 | 142 | final_model_path = MODELS_PATH + '{}_fold_{}_land.h5'.format(cnn_type, num_fold) 143 | cache_model_path = MODELS_PATH + '{}_temp_fold_{}_land.h5'.format(cnn_type, num_fold) 144 | if os.path.isfile(final_model_path) and RECREATE_MODELS == 0: 145 | print('Model {} already exists. Skip it'.format(final_model_path)) 146 | return 0.0 147 | if os.path.isfile(cache_model_path) and RESTORE_FROM_LAST_CHECKPOINT: 148 | print('Load model from last point: ', cache_model_path) 149 | model.load_weights(cache_model_path) 150 | elif os.path.isfile(final_model_path) and UPDATE_BEST_MODEL: 151 | print('Load model from best point: ', final_model_path) 152 | model.load_weights(final_model_path) 153 | else: 154 | print('Start training from begining') 155 | 156 | print('Fitting model...') 157 | batch_size = get_batch_size(cnn_type) 158 | print('Batch size: {}'.format(batch_size)) 159 | print('Learning rate: {}'.format(get_learning_rate(cnn_type))) 160 | samples_train_per_epoch = batch_size * (1 + len(train_files) // (10 * batch_size)) 161 | samples_valid_per_epoch = samples_train_per_epoch 162 | print('Samples train: {}, Samples valid: {}'.format(samples_train_per_epoch, samples_valid_per_epoch)) 163 | 164 | callbacks = [ 165 | EarlyStopping(monitor='val_loss', patience=PATIENCE, verbose=0), 166 | ModelCheckpoint(cache_model_path, monitor='val_loss', save_best_only=True, verbose=0), 167 | ] 168 | 169 | history = model.fit_generator(generator=batch_generator_train(cnn_type, train_files, train_labels, True), 170 | nb_epoch=NB_EPOCH, 171 | samples_per_epoch=samples_train_per_epoch, 172 | validation_data=batch_generator_train(cnn_type, valid_files, valid_labels, True), 173 | nb_val_samples=samples_valid_per_epoch, 174 | verbose=2, max_q_size=100, 175 | callbacks=callbacks) 176 | 177 | min_loss = min(history.history['val_loss']) 178 | print('Minimum loss for given fold: ', min_loss) 179 | model.load_weights(cache_model_path) 180 | model.save(final_model_path) 181 | now = datetime.datetime.now() 182 | filename = HISTORY_FOLDER_PATH + 'history_{}_{}_{:.4f}_lr_{}_{}_land.csv'.format(cnn_type, num_fold, min_loss, get_learning_rate(cnn_type), now.strftime("%Y-%m-%d-%H-%M")) 183 | pd.DataFrame(history.history).to_csv(filename, index=False) 184 | return min_loss 185 | 186 | 187 | def run_cross_validation_create_models(nfolds, cnn_type): 188 | global FOLD_TO_CALC 189 | tbl = pd.read_csv(INPUT_PATH + "train_v2.csv") 190 | lbl = get_train_label_matrix() 191 | indexes = get_indexes() 192 | 193 | # Only select land indexes 194 | choose = [0, 1, 2, 3, 4, 7, 8, 9, 12, 13, 14, 15, 16] 195 | print('Choose land indexes: {}'.format(np.array(indexes)[choose])) 196 | lbl = lbl[:, choose] 197 | 198 | files = [] 199 | for id in tbl['image_name'].values: 200 | files.append("../input/train-jpg/" + id + '.jpg') 201 | files = np.array(files) 202 | 203 | print('Label shape:', lbl.shape) 204 | print('Max labels in row:', max(lbl.sum(axis=1))) 205 | 206 | kf = KFold(n_splits=nfolds, shuffle=True, random_state=get_random_state(cnn_type)) 207 | num_fold = 0 208 | sum_score = 0 209 | for train_ids, valid_ids in kf.split(range(len(files))): 210 | num_fold += 1 211 | print('Start KFold number {} from {}'.format(num_fold, nfolds)) 212 | print('Split train: ', len(train_ids)) 213 | print('Split valid: ', len(valid_ids)) 214 | train_files = files[train_ids] 215 | valid_files = files[valid_ids] 216 | train_labels = lbl[train_ids] 217 | valid_labels = lbl[valid_ids] 218 | 219 | if 'FOLD_TO_CALC' in globals(): 220 | if num_fold not in FOLD_TO_CALC: 221 | continue 222 | 223 | score = train_single_model(num_fold, cnn_type, train_files, valid_files, train_labels, valid_labels) 224 | sum_score += score 225 | 226 | print('Avg loss: {}'.format(sum_score/nfolds)) 227 | return sum_score/nfolds 228 | 229 | 230 | if __name__ == '__main__': 231 | num_folds = 5 232 | score1 = '' 233 | for cnn in ['DENSENET_121']: 234 | score1 = run_cross_validation_create_models(num_folds, cnn) 235 | 236 | ''' 237 | Training history: 238 | 239 | Fold 1: 0.0860362012675 240 | Fold 2: 0.0878 241 | Fold 3: 0.086705402459 242 | Fold 4: 0.0843 243 | Fold 5: 0.0844426939011 244 | ''' 245 | -------------------------------------------------------------------------------- /a30_create_keras_models_single_class.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | __author__ = 'ZFTurbo: https://kaggle.com/zfturbo' 3 | 4 | import os 5 | import glob 6 | import cv2 7 | import pandas as pd 8 | from sklearn.model_selection import KFold 9 | from collections import defaultdict 10 | 11 | GPU_TO_USE = 0 12 | USE_THEANO = 1 13 | 14 | # Uncomment if you need to calculate specific fold 15 | # FOLD_TO_CALC = [5] 16 | 17 | if USE_THEANO: 18 | os.environ["KERAS_BACKEND"] = "theano" 19 | os.environ["THEANO_FLAGS"] = "device=gpu{},lib.cnmem=0.81".format(GPU_TO_USE, GPU_TO_USE) 20 | else: 21 | os.environ["KERAS_BACKEND"] = "tensorflow" 22 | os.environ["CUDA_VISIBLE_DEVICES"] = "{}".format(GPU_TO_USE) 23 | 24 | 25 | import datetime 26 | import random 27 | from a02_zoo import * 28 | 29 | random.seed(2016) 30 | np.random.seed(2016) 31 | 32 | PATIENCE = 50 33 | NB_EPOCH = 1000 34 | RESTORE_FROM_LAST_CHECKPOINT = 0 35 | UPDATE_BEST_MODEL = 0 36 | RECREATE_MODELS = 0 37 | 38 | 39 | INPUT_PATH = "../input/" 40 | MODELS_PATH = '../models/' 41 | if not os.path.isdir(MODELS_PATH): 42 | os.mkdir(MODELS_PATH) 43 | OUTPUT_PATH = "../subm/" 44 | if not os.path.isdir(OUTPUT_PATH): 45 | os.mkdir(OUTPUT_PATH) 46 | CODE_COPY_FOLDER = "../models/code/" 47 | if not os.path.isdir(CODE_COPY_FOLDER): 48 | os.mkdir(CODE_COPY_FOLDER) 49 | HISTORY_FOLDER_PATH = "../models/history/" 50 | if not os.path.isdir(HISTORY_FOLDER_PATH): 51 | os.mkdir(HISTORY_FOLDER_PATH) 52 | 53 | 54 | FULL_IMAGE_ARRAY = dict() 55 | def prepread_images(): 56 | files = glob.glob(INPUT_PATH + "train-jpg/*.jpg") 57 | total = 0 58 | for f in files: 59 | FULL_IMAGE_ARRAY[os.path.basename(f)] = cv2.imread(f) 60 | total += 1 61 | if total % 5000 == 0: 62 | print('Read {} files from {}...'.format(total, len(files))) 63 | 64 | 65 | def random_intensity_change(img, max_change): 66 | img = img.astype(np.int16) 67 | for j in range(3): 68 | delta = random.randint(-max_change, max_change) 69 | img[:, :, j] += delta 70 | img[img < 0] = 0 71 | img[img > 255] = 255 72 | return img.astype(np.uint8) 73 | 74 | 75 | def batch_generator_train(cnn_type, files, labels, augment=False): 76 | import keras.backend as K 77 | global FULL_IMAGE_ARRAY 78 | 79 | dim_ordering = K.image_dim_ordering() 80 | in_shape = get_input_shape(cnn_type) 81 | batch_size = get_batch_size(cnn_type) 82 | if len(FULL_IMAGE_ARRAY) == 0: 83 | prepread_images() 84 | 85 | files_no_class = files[labels == 0].copy() 86 | files_have_class = files[labels == 1].copy() 87 | 88 | while True: 89 | b_no = batch_size // 2 90 | b_have = batch_size - b_no 91 | index1 = random.sample(range(len(files_no_class)), b_no) 92 | index2 = random.sample(range(len(files_have_class)), b_have) 93 | batch_files = np.concatenate((files_no_class[index1], files_have_class[index2])) 94 | batch_labels = [0] * b_no + [1] * b_have 95 | 96 | image_list = [] 97 | mask_list = [] 98 | for i in range(len(batch_files)): 99 | # image = cv2.imread(batch_files[i]) 100 | image = FULL_IMAGE_ARRAY[os.path.basename(batch_files[i])] 101 | 102 | if cnn_type == 'INCEPTION_V3' or cnn_type == 'INCEPTION_V4' or cnn_type == 'XCEPTION': 103 | random_border = 30 104 | start0 = random.randint(0, random_border) 105 | start1 = random.randint(0, random_border) 106 | end0 = random.randint(0, random_border) 107 | end1 = random.randint(0, random_border) 108 | image = image[start0:image.shape[0] - end0, start1:image.shape[1] - end1] 109 | image = cv2.resize(image, (299, 299), cv2.INTER_LANCZOS4) 110 | else: 111 | box_size = random.randint(180, 256) 112 | start0 = random.randint(0, image.shape[0] - box_size) 113 | start1 = random.randint(0, image.shape[1] - box_size) 114 | image = image[start0:start0 + box_size, start1:start1 + box_size] 115 | image = cv2.resize(image, in_shape, cv2.INTER_LANCZOS4) 116 | 117 | if augment: 118 | # all possible mirroring and flips 119 | # (in total there are only 8 possible configurations) 120 | mirror = random.randint(0, 1) 121 | if mirror == 1: 122 | # flipud 123 | image = image[::-1, :, :] 124 | angle = random.randint(0, 3) 125 | if angle != 0: 126 | image = np.rot90(image, k=angle) 127 | image = random_intensity_change(image, 5) 128 | 129 | mask = batch_labels[i] 130 | image_list.append(image.astype(np.float32)) 131 | mask_list.append(mask) 132 | image_list = np.array(image_list) 133 | image_list = image_list.transpose((0, 3, 1, 2)) 134 | image_list = preprocess_input_overall(cnn_type, image_list) 135 | if dim_ordering == 'tf': 136 | image_list = image_list.transpose((0, 2, 3, 1)) 137 | mask_list = np.array(mask_list) 138 | yield image_list, mask_list 139 | 140 | 141 | def train_single_model(num_fold, cnn_type, class_id, train_files, valid_files, train_labels, valid_labels): 142 | from keras.callbacks import EarlyStopping, ModelCheckpoint 143 | 144 | tl1 = train_labels[:, class_id] 145 | tl2 = valid_labels[:, class_id] 146 | print('Class present in {} train cases'.format(len(tl1[tl1 == 1]))) 147 | print('Class present in {} valid cases'.format(len(tl2[tl2 == 1]))) 148 | 149 | if class_id == 1: 150 | lr = 0.000003 151 | elif class_id == 2: 152 | lr = 0.000003 153 | elif class_id == 3: 154 | lr = 0.000003 155 | elif class_id == 0: 156 | lr = 0.00003 157 | elif class_id == 5: 158 | lr = 0.00003 159 | elif class_id == 8: 160 | lr = 0.00003 161 | elif class_id == 9: 162 | lr = 0.000003 163 | elif class_id == 13: 164 | lr = 0.00003 165 | elif class_id == 11: 166 | lr = 0.00003 167 | elif class_id == 12: 168 | lr = 0.00003 169 | elif class_id == 14: 170 | lr = 0.000003 171 | elif class_id == 15: 172 | lr = 0.000003 173 | elif class_id == 16: 174 | lr = 0.00003 175 | else: 176 | lr = 0.000003 177 | 178 | print('Creating and compiling model [{}]...'.format(cnn_type)) 179 | model = get_pretrained_model(cnn_type, 1, learning_rate=lr, final_layer_activation='sigmoid') 180 | 181 | final_model_path = MODELS_PATH + '{}_fold_{}_single_class_{}.h5'.format(cnn_type, num_fold, class_id) 182 | cache_model_path = MODELS_PATH + '{}_temp_fold_{}_single_class_{}.h5'.format(cnn_type, num_fold, class_id) 183 | if os.path.isfile(final_model_path) and RECREATE_MODELS == 0: 184 | print('Model {} already exists. Skip it'.format(final_model_path)) 185 | return 0.0 186 | if os.path.isfile(cache_model_path) and RESTORE_FROM_LAST_CHECKPOINT: 187 | print('Load model from last point: ', cache_model_path) 188 | model.load_weights(cache_model_path) 189 | elif os.path.isfile(final_model_path) and UPDATE_BEST_MODEL: 190 | print('Load model from best point: ', final_model_path) 191 | model.load_weights(final_model_path) 192 | else: 193 | print('Start training from begining') 194 | 195 | print('Fitting model...') 196 | batch_size = get_batch_size(cnn_type) 197 | print('Batch size: {}'.format(batch_size)) 198 | print('Learning rate: {}'.format(lr)) 199 | samples_train_per_epoch = batch_size * (1 + len(train_files) // (10 * batch_size)) 200 | samples_valid_per_epoch = samples_train_per_epoch 201 | print('Samples train: {}, Samples valid: {}'.format(samples_train_per_epoch, samples_valid_per_epoch)) 202 | 203 | callbacks = [ 204 | EarlyStopping(monitor='val_loss', patience=PATIENCE, verbose=0), 205 | ModelCheckpoint(cache_model_path, monitor='val_loss', save_best_only=True, verbose=0), 206 | ] 207 | 208 | history = model.fit_generator(generator=batch_generator_train(cnn_type, train_files, train_labels[:, class_id], True), 209 | nb_epoch=NB_EPOCH, 210 | samples_per_epoch=samples_train_per_epoch, 211 | validation_data=batch_generator_train(cnn_type, valid_files, valid_labels[:, class_id], True), 212 | nb_val_samples=samples_valid_per_epoch, 213 | verbose=2, max_q_size=300, 214 | callbacks=callbacks) 215 | 216 | min_loss = min(history.history['val_loss']) 217 | print('Minimum loss for given fold: ', min_loss) 218 | model.load_weights(cache_model_path) 219 | model.save(final_model_path) 220 | now = datetime.datetime.now() 221 | filename = HISTORY_FOLDER_PATH + 'history_{}_sc_{}_{}_{:.4f}_lr_{}_{}.csv'.format(cnn_type, class_id, num_fold, min_loss, get_learning_rate(cnn_type), now.strftime("%Y-%m-%d-%H-%M")) 222 | pd.DataFrame(history.history).to_csv(filename, index=False) 223 | return min_loss 224 | 225 | 226 | def run_cross_validation_create_models_single_class(nfolds, cnn_type, class_id): 227 | global FOLD_TO_CALC 228 | tbl = pd.read_csv(INPUT_PATH + "train_v2.csv") 229 | labels = tbl['tags'].apply(lambda x: x.split(' ')) 230 | counts = defaultdict(int) 231 | for l in labels: 232 | for l2 in l: 233 | counts[l2] += 1 234 | indexes = sorted(list(counts.keys())) 235 | for i in range(len(indexes)): 236 | tbl['label_{}'.format(i)] = 0 237 | 238 | files = [] 239 | for id in tbl['image_name'].values: 240 | files.append(INPUT_PATH + "train-jpg/" + id + '.jpg') 241 | files = np.array(files) 242 | 243 | lbl = np.zeros((len(labels), len(indexes))) 244 | for j in range(len(labels)): 245 | l = labels[j] 246 | for i in range(len(indexes)): 247 | if indexes[i] in l: 248 | lbl[j][i] = 1 249 | 250 | # print(len(tbl)) 251 | # print(len(labels)) 252 | print(lbl.shape) 253 | 254 | kf = KFold(n_splits=nfolds, shuffle=True, random_state=get_random_state(cnn_type)) 255 | num_fold = 0 256 | sum_score = 0 257 | for train_ids, valid_ids in kf.split(range(len(files))): 258 | num_fold += 1 259 | print('Start KFold number {} from {}'.format(num_fold, nfolds)) 260 | print('Split train: ', len(train_ids)) 261 | print('Split valid: ', len(valid_ids)) 262 | train_files = files[train_ids] 263 | valid_files = files[valid_ids] 264 | train_labels = lbl[train_ids] 265 | valid_labels = lbl[valid_ids] 266 | 267 | if 'FOLD_TO_CALC' in globals(): 268 | if num_fold not in FOLD_TO_CALC: 269 | continue 270 | 271 | score = train_single_model(num_fold, cnn_type, class_id, train_files, valid_files, train_labels, valid_labels) 272 | sum_score += score 273 | 274 | print('Avg loss: {}'.format(sum_score/nfolds)) 275 | return sum_score/nfolds 276 | 277 | 278 | if __name__ == '__main__': 279 | num_folds = 5 280 | score1 = '' 281 | for class_id in range(0, 17): 282 | print('Class: {}'.format(class_id)) 283 | for cnn in ['DENSENET_121']: 284 | score1 = run_cross_validation_create_models_single_class(num_folds, cnn, class_id) 285 | 286 | 287 | ''' 288 | Training history: 289 | 290 | Class 0 (lr = 0.00003): 291 | Minimum loss for given fold: 0.253413404281 292 | Minimum loss for given fold: 0.248380952066 293 | Minimum loss for given fold: 0.257372181771 294 | Minimum loss for given fold: 0.26305075751 295 | Minimum loss for given fold: 0.249311803058 296 | Avg loss: 0.2543058197370466 297 | 298 | Class 1 (lr = 0.000003): 299 | Minimum loss for given fold: 0.108140268237 300 | Minimum loss for given fold: 0.0683494081152 301 | Minimum loss for given fold: 0.127105548575 302 | Minimum loss for given fold: 0.0354281800977 303 | Minimum loss for given fold: 0.0873878211542 304 | Avg loss: 0.08528224523581658 305 | 306 | Class 2 (lr = 0.00003): 307 | Minimum loss for given fold: 0.3910083831 308 | Minimum loss for given fold: 0.385183117686 309 | 310 | Class 2 (lr = 0.000003): 311 | Minimum loss for given fold: 0.388385605113 312 | Minimum loss for given fold: 0.379616589366 313 | Minimum loss for given fold: 0.341731107299 314 | Minimum loss for given fold: 0.340404011907 315 | Minimum loss for given fold: 0.331169278349 316 | Avg loss: 0.3562613184069409 317 | 318 | Class 3 (lr = 0.000003): 319 | Minimum loss for given fold: 0.330049385029 320 | Minimum loss for given fold: 0.342264363059 321 | Minimum loss for given fold: 0.191731132314 322 | Minimum loss for given fold: 0.288646133939 323 | Minimum loss for given fold: 0.391212643481 324 | Avg loss: 0.3087807315643187 325 | 326 | Class 4 (lr = 0.000003): 327 | Minimum loss for given fold: 0.413155568419 328 | Minimum loss for given fold: 0.346210029666 329 | Minimum loss for given fold: 0.389502455216 330 | Minimum loss for given fold: 0.365005755866 331 | Minimum loss for given fold: 0.422119773005 332 | Avg loss: 0.3856122146004144 333 | 334 | Class 5 (lr = 0.00003): 335 | Minimum loss for given fold: 0.14555569979 336 | Minimum loss for given fold: 0.166418199968 337 | Minimum loss for given fold: 0.153322775457 338 | Minimum loss for given fold: 0.145887596717 339 | Minimum loss for given fold: 0.144087674372 340 | 341 | Class 6 (lr = 0.000003): 342 | Minimum loss for given fold: 0.0762309606049 343 | Minimum loss for given fold: 0.0769969969614 344 | Minimum loss for given fold: 0.0780014535268 345 | Minimum loss for given fold: 0.0684322627767 346 | Minimum loss for given fold: 0.0662103267042 347 | Avg loss: 0.07317440011481076 348 | 349 | Class 7 (lr = 0.000003): 350 | Minimum loss for given fold: 0.168240847641 351 | Minimum loss for given fold: 0.13196111221 352 | Minimum loss for given fold: 0.202992705743 353 | Minimum loss for given fold: 0.218001743572 354 | Minimum loss for given fold: 0.111677368884 355 | Avg loss: 0.16657475560995533 356 | 357 | Class 8 (lr = 0.00003) 358 | Minimum loss for given fold: 0.366216352094 359 | Minimum loss for given fold: 0.372726423689 360 | Minimum loss for given fold: 0.378883124962 361 | Minimum loss for given fold: 0.358972315343 362 | Minimum loss for given fold: 0.366272835774 363 | Avg loss: 0.3686142103723538 364 | 365 | Class 9 () 366 | Minimum loss for given fold: 0.249355089172 367 | Minimum loss for given fold: 0.244136921579 368 | Minimum loss for given fold: 0.235164790115 369 | Minimum loss for given fold: 0.254924793532 370 | Minimum loss for given fold: 0.240773150398 371 | Avg loss: 0.24487094895937195 372 | 373 | Class 10: 374 | Minimum loss for given fold: 0.211632032154 375 | Minimum loss for given fold: 0.206836078017 376 | Minimum loss for given fold: 0.21180134523 377 | Minimum loss for given fold: 0.193908381799 378 | Minimum loss for given fold: 0.200499110361 379 | Avg loss: 0.20493538951225304 380 | 381 | Class 11: 382 | Minimum loss for given fold: 0.133019096482 383 | Minimum loss for given fold: 0.139944022527 384 | Minimum loss for given fold: 0.121319166928 385 | Minimum loss for given fold: 0.127049939878 386 | Minimum loss for given fold: 0.140708353002 387 | Avg loss: 0.13240811576359662 388 | 389 | Class 12 (): 390 | Minimum loss for given fold: 0.153227462225 391 | Minimum loss for given fold: 0.153025816772 392 | Minimum loss for given fold: 0.147686096728 393 | Minimum loss for given fold: 0.147378727081 394 | Minimum loss for given fold: 0.133062615749 395 | Avg loss: 0.14687614371097518 396 | 397 | Class 13: 398 | Minimum loss for given fold: 0.228606079165 399 | Minimum loss for given fold: 0.230709340768 400 | Minimum loss for given fold: 0.220355313731 401 | Minimum loss for given fold: 0.226218112549 402 | Minimum loss for given fold: 0.23019926849 403 | Avg loss: 0.227217622940647 404 | 405 | Class 14: 406 | Minimum loss for given fold: 0.306654619605 407 | Minimum loss for given fold: 0.355313857304 408 | Minimum loss for given fold: 0.288375787437 409 | Minimum loss for given fold: 0.249127054771 410 | Minimum loss for given fold: 0.290934943767 411 | Avg loss: 0.2980812525767603 412 | 413 | Class 15: 414 | Minimum loss for given fold: 0.373362306275 415 | Minimum loss for given fold: 0.396898513867 416 | Minimum loss for given fold: 0.311251565584 417 | Minimum loss for given fold: 0.275662233915 418 | Minimum loss for given fold: 0.300677734324 419 | Avg loss: 0.3315704707930117 420 | 421 | Class 16: 422 | Minimum loss for given fold: 0.288113219204 423 | Minimum loss for given fold: 0.270322405262 424 | Minimum loss for given fold: 0.276544707977 425 | Minimum loss for given fold: 0.27940388125 426 | Minimum loss for given fold: 0.289342449357 427 | Avg loss: 0.2807453326099081 428 | ''' -------------------------------------------------------------------------------- /a30_create_keras_models_weather.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | __author__ = 'ZFTurbo: https://kaggle.com/zfturbo' 3 | 4 | import os 5 | import glob 6 | import cv2 7 | import pandas as pd 8 | from sklearn.model_selection import KFold 9 | from a00_common_functions import get_train_label_matrix, get_indexes 10 | 11 | GPU_TO_USE = 0 12 | USE_THEANO = 1 13 | 14 | # Uncomment if you need to calculate specific fold 15 | # FOLD_TO_CALC = [5] 16 | 17 | if USE_THEANO: 18 | os.environ["KERAS_BACKEND"] = "theano" 19 | os.environ["THEANO_FLAGS"] = "device=gpu{},lib.cnmem=0.81".format(GPU_TO_USE, GPU_TO_USE) 20 | else: 21 | os.environ["KERAS_BACKEND"] = "tensorflow" 22 | os.environ["CUDA_VISIBLE_DEVICES"] = "{}".format(GPU_TO_USE) 23 | 24 | 25 | import datetime 26 | import random 27 | from a02_zoo import * 28 | 29 | random.seed(2016) 30 | np.random.seed(2016) 31 | 32 | PATIENCE = 50 33 | NB_EPOCH = 1000 34 | RESTORE_FROM_LAST_CHECKPOINT = 0 35 | UPDATE_BEST_MODEL = 0 36 | RECREATE_MODELS = 0 37 | 38 | 39 | INPUT_PATH = "../input/" 40 | MODELS_PATH = '../models/' 41 | if not os.path.isdir(MODELS_PATH): 42 | os.mkdir(MODELS_PATH) 43 | OUTPUT_PATH = "../subm/" 44 | if not os.path.isdir(OUTPUT_PATH): 45 | os.mkdir(OUTPUT_PATH) 46 | CODE_COPY_FOLDER = "../models/code/" 47 | if not os.path.isdir(CODE_COPY_FOLDER): 48 | os.mkdir(CODE_COPY_FOLDER) 49 | HISTORY_FOLDER_PATH = "../models/history/" 50 | if not os.path.isdir(HISTORY_FOLDER_PATH): 51 | os.mkdir(HISTORY_FOLDER_PATH) 52 | 53 | 54 | FULL_IMAGE_ARRAY = dict() 55 | def prepread_images(): 56 | files = glob.glob(INPUT_PATH + "train-jpg/*.jpg") 57 | total = 0 58 | for f in files: 59 | FULL_IMAGE_ARRAY[os.path.basename(f)] = cv2.imread(f) 60 | total += 1 61 | if total % 5000 == 0: 62 | print('Read {} files from {}...'.format(total, len(files))) 63 | 64 | 65 | def random_intensity_change(img, max_change): 66 | img = img.astype(np.int16) 67 | for j in range(3): 68 | delta = random.randint(-max_change, max_change) 69 | img[:, :, j] += delta 70 | img[img < 0] = 0 71 | img[img > 255] = 255 72 | return img.astype(np.uint8) 73 | 74 | 75 | def batch_generator_train(cnn_type, files, labels, augment=False): 76 | import keras.backend as K 77 | global FULL_IMAGE_ARRAY 78 | 79 | dim_ordering = K.image_dim_ordering() 80 | in_shape = get_input_shape(cnn_type) 81 | batch_size = get_batch_size(cnn_type) 82 | if len(FULL_IMAGE_ARRAY) == 0: 83 | prepread_images() 84 | 85 | while True: 86 | index = random.sample(range(len(files)), batch_size) 87 | batch_files = files[index] 88 | batch_labels = labels[index] 89 | 90 | image_list = [] 91 | mask_list = [] 92 | for i in range(len(batch_files)): 93 | # image = cv2.imread(batch_files[i]) 94 | image = FULL_IMAGE_ARRAY[os.path.basename(batch_files[i])] 95 | 96 | if cnn_type == 'INCEPTION_V3' or cnn_type == 'INCEPTION_V4' or cnn_type == 'XCEPTION': 97 | random_border = 20 98 | start0 = random.randint(0, random_border) 99 | start1 = random.randint(0, random_border) 100 | end0 = random.randint(0, random_border) 101 | end1 = random.randint(0, random_border) 102 | image = image[start0:image.shape[0] - end0, start1:image.shape[1] - end1] 103 | image = cv2.resize(image, (299, 299), cv2.INTER_LANCZOS4) 104 | else: 105 | box_size = random.randint(200, 256) 106 | start0 = random.randint(0, image.shape[0] - box_size) 107 | start1 = random.randint(0, image.shape[1] - box_size) 108 | image = image[start0:start0 + box_size, start1:start1 + box_size] 109 | image = cv2.resize(image, in_shape, cv2.INTER_LANCZOS4) 110 | 111 | if augment: 112 | # all possible mirroring and flips 113 | # (in total there are only 8 possible configurations) 114 | mirror = random.randint(0, 1) 115 | if mirror == 1: 116 | # flipud 117 | image = image[::-1, :, :] 118 | angle = random.randint(0, 3) 119 | if angle != 0: 120 | image = np.rot90(image, k=angle) 121 | image = random_intensity_change(image, 3) 122 | 123 | mask = batch_labels[i] 124 | image_list.append(image.astype(np.float32)) 125 | mask_list.append(mask) 126 | image_list = np.array(image_list) 127 | image_list = image_list.transpose((0, 3, 1, 2)) 128 | image_list = preprocess_input_overall(cnn_type, image_list) 129 | if dim_ordering == 'tf': 130 | image_list = image_list.transpose((0, 2, 3, 1)) 131 | mask_list = np.array(mask_list) 132 | yield image_list, mask_list 133 | 134 | 135 | def train_single_model(num_fold, cnn_type, train_files, valid_files, train_labels, valid_labels): 136 | from keras.callbacks import EarlyStopping, ModelCheckpoint 137 | 138 | print('Creating and compiling model [{}]...'.format(cnn_type)) 139 | model = get_pretrained_model(cnn_type, 4, final_layer_activation='softmax') 140 | 141 | final_model_path = MODELS_PATH + '{}_fold_{}_weather.h5'.format(cnn_type, num_fold) 142 | cache_model_path = MODELS_PATH + '{}_temp_fold_{}_weather.h5'.format(cnn_type, num_fold) 143 | if os.path.isfile(final_model_path) and RECREATE_MODELS == 0: 144 | print('Model {} already exists. Skip it'.format(final_model_path)) 145 | return 0.0 146 | if os.path.isfile(cache_model_path) and RESTORE_FROM_LAST_CHECKPOINT: 147 | print('Load model from last point: ', cache_model_path) 148 | model.load_weights(cache_model_path) 149 | elif os.path.isfile(final_model_path) and UPDATE_BEST_MODEL: 150 | print('Load model from best point: ', final_model_path) 151 | model.load_weights(final_model_path) 152 | else: 153 | print('Start training from begining') 154 | 155 | print('Fitting model...') 156 | batch_size = get_batch_size(cnn_type) 157 | print('Batch size: {}'.format(batch_size)) 158 | print('Learning rate: {}'.format(get_learning_rate(cnn_type))) 159 | samples_train_per_epoch = batch_size * (1 + len(train_files) // (10 * batch_size)) 160 | samples_valid_per_epoch = samples_train_per_epoch 161 | print('Samples train: {}, Samples valid: {}'.format(samples_train_per_epoch, samples_valid_per_epoch)) 162 | 163 | callbacks = [ 164 | EarlyStopping(monitor='val_loss', patience=PATIENCE, verbose=0), 165 | ModelCheckpoint(cache_model_path, monitor='val_loss', save_best_only=True, verbose=0), 166 | ] 167 | 168 | history = model.fit_generator(generator=batch_generator_train(cnn_type, train_files, train_labels, True), 169 | nb_epoch=NB_EPOCH, 170 | samples_per_epoch=samples_train_per_epoch, 171 | validation_data=batch_generator_train(cnn_type, valid_files, valid_labels, True), 172 | nb_val_samples=samples_valid_per_epoch, 173 | verbose=2, max_q_size=100, 174 | callbacks=callbacks) 175 | 176 | min_loss = min(history.history['val_loss']) 177 | print('Minimum loss for given fold: ', min_loss) 178 | model.load_weights(cache_model_path) 179 | model.save(final_model_path) 180 | now = datetime.datetime.now() 181 | filename = HISTORY_FOLDER_PATH + 'history_{}_{}_{:.4f}_lr_{}_{}_weather.csv'.format(cnn_type, num_fold, min_loss, get_learning_rate(cnn_type), now.strftime("%Y-%m-%d-%H-%M")) 182 | pd.DataFrame(history.history).to_csv(filename, index=False) 183 | return min_loss 184 | 185 | 186 | def run_cross_validation_create_models(nfolds, cnn_type): 187 | global FOLD_TO_CALC 188 | tbl = pd.read_csv(INPUT_PATH + "train_v2.csv") 189 | lbl = get_train_label_matrix() 190 | indexes = get_indexes() 191 | 192 | # Only select weather indexes 193 | # ['clear', 'partly_cloudy', 'haze', 'cloudy'] 194 | choose = [5, 6, 10, 11] 195 | print('Choose weather indexes: {}'.format(np.array(indexes)[choose])) 196 | lbl = lbl[:, choose] 197 | 198 | files = [] 199 | for id in tbl['image_name'].values: 200 | files.append("../input/train-jpg/" + id + '.jpg') 201 | files = np.array(files) 202 | 203 | print('Label shape:', lbl.shape) 204 | print('Max labels in row:', max(lbl.sum(axis=1))) 205 | 206 | kf = KFold(n_splits=nfolds, shuffle=True, random_state=get_random_state(cnn_type)) 207 | num_fold = 0 208 | sum_score = 0 209 | for train_ids, valid_ids in kf.split(range(len(files))): 210 | num_fold += 1 211 | print('Start KFold number {} from {}'.format(num_fold, nfolds)) 212 | print('Split train: ', len(train_ids)) 213 | print('Split valid: ', len(valid_ids)) 214 | train_files = files[train_ids] 215 | valid_files = files[valid_ids] 216 | train_labels = lbl[train_ids] 217 | valid_labels = lbl[valid_ids] 218 | 219 | if 'FOLD_TO_CALC' in globals(): 220 | if num_fold not in FOLD_TO_CALC: 221 | continue 222 | 223 | score = train_single_model(num_fold, cnn_type, train_files, valid_files, train_labels, valid_labels) 224 | sum_score += score 225 | 226 | print('Avg loss: {}'.format(sum_score/nfolds)) 227 | return sum_score/nfolds 228 | 229 | 230 | if __name__ == '__main__': 231 | num_folds = 5 232 | score1 = '' 233 | for cnn in ['DENSENET_121']: 234 | score1 = run_cross_validation_create_models(num_folds, cnn) 235 | 236 | 237 | ''' 238 | Training history: 239 | Fold 1: 0.185169663315 240 | Fold 2: 0.1752433345 241 | Fold 3: 0.175999773528 242 | Fold 4: 0.177833901902 243 | Fold 5: 0.168742229386 244 | ''' 245 | -------------------------------------------------------------------------------- /a31_create_cnn_features_basic.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | __author__ = 'ZFTurbo: https://kaggle.com/zfturbo' 3 | 4 | from a00_common_functions import * 5 | 6 | GPU_TO_USE = 0 7 | USE_THEANO = 1 8 | 9 | # Uncomment if you need to calculate specific fold 10 | # FOLD_TO_CALC = [5] 11 | 12 | if USE_THEANO: 13 | os.environ["KERAS_BACKEND"] = "theano" 14 | os.environ["THEANO_FLAGS"] = "device=gpu{},lib.cnmem=0.81".format(GPU_TO_USE, GPU_TO_USE) 15 | else: 16 | os.environ["KERAS_BACKEND"] = "tensorflow" 17 | os.environ["CUDA_VISIBLE_DEVICES"] = "{}".format(GPU_TO_USE) 18 | 19 | import random 20 | from a02_zoo import * 21 | 22 | random.seed(2016) 23 | np.random.seed(2016) 24 | 25 | RESTORE_FROM_LAST_CHECKPOINT = 0 26 | CLASSES_NUMBER = 17 27 | 28 | INPUT_PATH = "../input/" 29 | MODELS_PATH = '../models/' 30 | if not os.path.isdir(MODELS_PATH): 31 | os.mkdir(MODELS_PATH) 32 | OUTPUT_PATH = "../subm/" 33 | if not os.path.isdir(OUTPUT_PATH): 34 | os.mkdir(OUTPUT_PATH) 35 | FEATURES_PATH = "../features/" 36 | if not os.path.isdir(FEATURES_PATH): 37 | os.mkdir(FEATURES_PATH) 38 | CODE_COPY_FOLDER = "../models/code/" 39 | if not os.path.isdir(CODE_COPY_FOLDER): 40 | os.mkdir(CODE_COPY_FOLDER) 41 | HISTORY_FOLDER_PATH = "../models/history/" 42 | if not os.path.isdir(HISTORY_FOLDER_PATH): 43 | os.mkdir(HISTORY_FOLDER_PATH) 44 | CACHE_PATH = "../cache/" 45 | if not os.path.isdir(CACHE_PATH): 46 | os.mkdir(CACHE_PATH) 47 | 48 | 49 | def get_validation_score(nfolds, cnn_type): 50 | from keras.models import load_model 51 | from keras import backend as K 52 | 53 | if K.backend() == 'tensorflow': 54 | print('Update dim ordering to "tf"') 55 | K.set_image_dim_ordering('tf') 56 | 57 | restore_from_cache = 0 58 | 59 | tbl = pd.read_csv(INPUT_PATH + "train_v2.csv") 60 | labels = tbl['tags'].apply(lambda x: x.split(' ')) 61 | counts = defaultdict(int) 62 | for l in labels: 63 | for l2 in l: 64 | counts[l2] += 1 65 | indexes = sorted(list(counts.keys())) 66 | for i in range(len(indexes)): 67 | tbl['label_{}'.format(i)] = 0 68 | 69 | files = [] 70 | for id in tbl['image_name'].values: 71 | files.append(INPUT_PATH + "train-jpg/" + id + '.jpg') 72 | files = np.array(files) 73 | 74 | lbl = np.zeros((len(labels), len(indexes))) 75 | for j in range(len(labels)): 76 | l = labels[j] 77 | for i in range(len(indexes)): 78 | if indexes[i] in l: 79 | lbl[j][i] = 1 80 | 81 | # print(lbl) 82 | print(lbl.shape) 83 | stat = [] 84 | 85 | kf = KFold(n_splits=nfolds, shuffle=True, random_state=get_random_state(cnn_type)) 86 | num_fold = 0 87 | result = np.zeros((len(labels), len(indexes))) 88 | for train_ids, valid_ids in kf.split(range(len(files))): 89 | num_fold += 1 90 | start_time = time.time() 91 | cache_file = CACHE_PATH + '{}_valid_fold_{}.pklz'.format(cnn_type, num_fold) 92 | print('Start KFold number {} from {}'.format(num_fold, nfolds)) 93 | print('Split train: ', len(train_ids)) 94 | print('Split valid: ', len(valid_ids)) 95 | valid_files = files[valid_ids] 96 | valid_labels = lbl[valid_ids] 97 | if not (os.path.isfile(cache_file) and restore_from_cache): 98 | final_model_path = MODELS_PATH + '{}_fold_{}.h5'.format(cnn_type, num_fold) 99 | print('Loading model {}...'.format(final_model_path)) 100 | if cnn_type == 'RESNET101' or cnn_type == 'RESNET152' or 'DENSENET' in cnn_type: 101 | model = get_pretrained_model(cnn_type, CLASSES_NUMBER) 102 | model.load_weights(final_model_path) 103 | else: 104 | c = dict() 105 | c['f2beta_loss'] = f2beta_loss 106 | model = load_model(final_model_path, custom_objects=c) 107 | preds = get_raw_predictions_for_images_v3(model, cnn_type, valid_files) 108 | save_in_file(preds, cache_file) 109 | else: 110 | preds = load_from_file(cache_file) 111 | 112 | for i in range(len(valid_ids)): 113 | result[valid_ids[i], :] = preds[i] 114 | print(preds.shape) 115 | print(valid_labels.shape) 116 | 117 | best_score = -1 118 | best_thr = -1 119 | for thr1 in range(1, 100): 120 | p = preds.copy() 121 | thr = thr1 / 100 122 | p[p > thr] = 1 123 | p[p <= thr] = 0 124 | score = f2_score(valid_labels, p) 125 | print('THR: {} SCORE: {}'.format(thr, score)) 126 | if score > best_score: 127 | best_score = score 128 | best_thr = thr 129 | 130 | stat.append((best_score, best_thr)) 131 | print('Best score: {} THR: {}'.format(best_score, best_thr)) 132 | print('Fold time: {} seconds'.format(time.time() - start_time)) 133 | # if num_fold == 1: 134 | # exit() 135 | 136 | best_score = -1 137 | best_thr = -1 138 | for thr1 in range(1, 100): 139 | p = result.copy() 140 | thr = thr1 / 100 141 | p[p > thr] = 1 142 | p[p <= thr] = 0 143 | score = f2_score(lbl, p) 144 | print('THR: {} SCORE: {}'.format(thr, score)) 145 | if score > best_score: 146 | best_score = score 147 | best_thr = thr 148 | print('Best overall score: {} THR: {}'.format(best_score, best_thr)) 149 | for i in range(len(stat)): 150 | print('Best score fold {}: {} THR: {}'.format(i+1, stat[i][0], stat[i][1])) 151 | 152 | # Save validation file 153 | out = open(FEATURES_PATH + "valid_{}_score_{}_thr_{}.csv".format(cnn_type, best_score, best_thr), "w") 154 | # out = open(FEATURES_PATH + "valid_{}.csv".format(cnn_type), "w") 155 | out.write("image_name") 156 | for i in range(len(indexes)): 157 | out.write("," + indexes[i]) 158 | out.write("\n") 159 | ids = tbl['image_name'].values 160 | for i in range(len(result)): 161 | out.write(ids[i]) 162 | for j in range(len(indexes)): 163 | out.write("," + str(result[i][j])) 164 | out.write("\n") 165 | out.close() 166 | 167 | return best_score, best_thr 168 | 169 | 170 | def process_test(nfolds, cnn_type, score, thr): 171 | global FOLD_TO_CALC 172 | from keras.models import load_model 173 | from keras import backend as K 174 | 175 | if K.backend() == 'tensorflow': 176 | print('Update dim ordering to "tf"') 177 | K.set_image_dim_ordering('tf') 178 | 179 | restore_from_cache = 0 180 | 181 | tbl = pd.read_csv(INPUT_PATH + "sample_submission_v2.csv") 182 | indexes = get_indexes() 183 | ids = tbl['image_name'].values 184 | 185 | files = [] 186 | for id in ids: 187 | files.append(INPUT_PATH + "test-jpg/" + id + '.jpg') 188 | files = np.array(files) 189 | 190 | preds = [] 191 | for num_fold in range(1, nfolds+1): 192 | 193 | if 'FOLD_TO_CALC' in globals(): 194 | if num_fold not in FOLD_TO_CALC: 195 | continue 196 | 197 | cache_file = CACHE_PATH + '{}_test_fold_{}'.format(cnn_type, num_fold) 198 | print('Start KFold number {} from {}'.format(num_fold, nfolds)) 199 | if os.path.isfile(cache_file) and restore_from_cache: 200 | print('Restore from cache...') 201 | p = load_from_file(cache_file) 202 | else: 203 | final_model_path = MODELS_PATH + '{}_fold_{}.h5'.format(cnn_type, num_fold) 204 | print('Loading model {}...'.format(final_model_path)) 205 | if cnn_type == 'RESNET101' or cnn_type == 'RESNET152' or 'DENSENET' in cnn_type: 206 | model = get_pretrained_model(cnn_type, CLASSES_NUMBER) 207 | model.load_weights(final_model_path) 208 | else: 209 | c = dict() 210 | c['f2beta_loss'] = f2beta_loss 211 | model = load_model(final_model_path, custom_objects=c) 212 | p = get_raw_predictions_for_images_v3(model, cnn_type, files) 213 | save_in_file(p, cache_file) 214 | preds.append(p) 215 | preds = np.array(preds) 216 | print(preds.shape) 217 | preds = np.mean(preds, axis=0) 218 | 219 | # Save raw feature file 220 | out = open(FEATURES_PATH + "test_{}_score_{}_thr_{}.csv".format(cnn_type, score, thr), "w") 221 | out.write("image_name") 222 | for i in range(len(indexes)): 223 | out.write("," + indexes[i]) 224 | out.write("\n") 225 | ids = tbl['image_name'].values 226 | for i in range(len(preds)): 227 | out.write(ids[i]) 228 | for j in range(len(indexes)): 229 | out.write("," + str(preds[i][j])) 230 | out.write("\n") 231 | out.close() 232 | 233 | # Create submission 234 | out = open(OUTPUT_PATH + "subm_{}_score_{}_thr_{}.csv".format(cnn_type, score, thr), "w") 235 | out.write("image_name,tags\n") 236 | for i in range(len(files)): 237 | out.write(ids[i] + ',') 238 | for j in range(len(indexes)): 239 | if preds[i][j] > thr: 240 | out.write(indexes[j] + ' ') 241 | out.write("\n") 242 | out.close() 243 | 244 | 245 | if __name__ == '__main__': 246 | num_folds = 5 247 | score1 = '' 248 | for cnn in ['INCEPTION_V3_DENSE_LAYERS', 'INCEPTION_V4', 'DENSENET_121', 'DENSENET_169', 'DENSENET_161', 249 | 'RESNET50_DENSE_LAYERS', 'RESNET101', 'VGG16', 'VGG19', 'RESNET152', 'XCEPTION']: 250 | best_score, best_thr = get_validation_score(num_folds, cnn) 251 | process_test(num_folds, cnn, best_score, best_thr) 252 | 253 | 254 | ''' 255 | Validation history: 256 | 257 | INCEPTION_V3_DENSE_LAYERS 258 | Best overall score: 0.9248598060971008 THR: 0.2 259 | Best score fold 1: 0.9237842186349807 THR: 0.2 260 | Best score fold 2: 0.9243091765877415 THR: 0.21 261 | Best score fold 3: 0.9258338706397004 THR: 0.22 262 | Best score fold 4: 0.9253362287668513 THR: 0.2 263 | Best score fold 5: 0.9254969455343668 THR: 0.18 264 | 265 | DENSENET_121 266 | Best overall score: 0.9282839676880005 THR: 0.2 267 | Best score fold 1: 0.9259732549616995 THR: 0.18 268 | Best score fold 2: 0.927808702322814 THR: 0.18 269 | Best score fold 3: 0.9302367605882403 THR: 0.21 270 | Best score fold 4: 0.9289837762131035 THR: 0.17 271 | Best score fold 5: 0.9292442752504116 THR: 0.2 272 | 273 | DENSENET_169 274 | Best overall score: 0.9255917518310303 THR: 0.2 275 | Best score fold 1: 0.9263497963226633 THR: 0.22 276 | Best score fold 2: 0.9229784022084283 THR: 0.14 277 | Best score fold 3: 0.9273674053370875 THR: 0.24 278 | Best score fold 4: 0.9265559805626908 THR: 0.2 279 | Best score fold 5: 0.9262725947617672 THR: 0.22 280 | 281 | DENSENET_161 282 | Best overall score: 0.9269659592740706 THR: 0.21 283 | Best score fold 1: 0.9264609624386172 THR: 0.2 284 | Best score fold 2: 0.9248898149127779 THR: 0.22 285 | Best score fold 3: 0.9289309539548167 THR: 0.21 286 | Best score fold 4: 0.9275478928335408 THR: 0.21 287 | Best score fold 5: 0.9282206246774681 THR: 0.17 288 | 289 | INCEPTION_v4 290 | Best overall score: 0.9262189395255209 THR: 0.17 291 | Best score fold 1: 0.922717236909142 THR: 0.18 292 | Best score fold 2: 0.9251909182647715 THR: 0.17 293 | Best score fold 3: 0.9268627144887531 THR: 0.21 294 | Best score fold 4: 0.9280003580757566 THR: 0.17 295 | Best score fold 5: 0.9286064966963335 THR: 0.17 296 | 297 | RESNET50_DENSE_LAYERS 298 | Best overall score: 0.9263739674493671 THR: 0.18 299 | Best score fold 1: 0.9239484491111165 THR: 0.17 300 | Best score fold 2: 0.9235899447121498 THR: 0.18 301 | Best score fold 3: 0.9296567172921516 THR: 0.19 302 | Best score fold 4: 0.9282728191281 THR: 0.17 303 | Best score fold 5: 0.9271708073379697 THR: 0.19 304 | 305 | RESNET101 306 | Best overall score: 0.9253183157874696 THR: 0.2 307 | Best score fold 1: 0.924428816482149 THR: 0.25 308 | Best score fold 2: 0.9239647724658601 THR: 0.21 309 | Best score fold 3: 0.9280631565793689 THR: 0.24 310 | Best score fold 4: 0.926793153998684 THR: 0.19 311 | Best score fold 5: 0.9247433106035584 THR: 0.16 312 | 313 | XCEPTION 314 | Best overall score: 0.9259769103573405 THR: 0.18 315 | Best score fold 1: 0.9247594110616024 THR: 0.18 316 | Best score fold 2: 0.925430670838188 THR: 0.2 317 | Best score fold 3: 0.9297797702591243 THR: 0.22 318 | Best score fold 4: 0.925844795320704 THR: 0.16 319 | Best score fold 5: 0.9250094054431726 THR: 0.18 320 | 321 | RESNET152 322 | Best overall score: 0.9275937814558509 THR: 0.18 323 | Best score fold 1: 0.9268510953182838 THR: 0.18 324 | Best score fold 2: 0.9264972708020228 THR: 0.2 325 | Best score fold 3: 0.9283127362941068 THR: 0.16 326 | Best score fold 4: 0.9300323600597092 THR: 0.18 327 | Best score fold 5: 0.9276937670196796 THR: 0.2 328 | 329 | VGG16 330 | Best overall score: 0.927268456620429 THR: 0.19 331 | Best score fold 1: 0.925309640350592 THR: 0.19 332 | Best score fold 2: 0.9256756438246051 THR: 0.21 333 | Best score fold 3: 0.9301456100138225 THR: 0.19 334 | Best score fold 4: 0.9265122943092845 THR: 0.2 335 | Best score fold 5: 0.9291713665462222 THR: 0.19 336 | 337 | ''' -------------------------------------------------------------------------------- /a31_create_cnn_features_land.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | __author__ = 'ZFTurbo: https://kaggle.com/zfturbo' 3 | 4 | from a00_common_functions import * 5 | 6 | GPU_TO_USE = 0 7 | USE_THEANO = 1 8 | 9 | # Uncomment if you need to calculate specific fold 10 | # FOLD_TO_CALC = [5] 11 | 12 | if USE_THEANO: 13 | os.environ["KERAS_BACKEND"] = "theano" 14 | os.environ["THEANO_FLAGS"] = "device=gpu{},lib.cnmem=0.81".format(GPU_TO_USE, GPU_TO_USE) 15 | else: 16 | os.environ["KERAS_BACKEND"] = "tensorflow" 17 | os.environ["CUDA_VISIBLE_DEVICES"] = "{}".format(GPU_TO_USE) 18 | 19 | import random 20 | from a02_zoo import * 21 | 22 | random.seed(2016) 23 | np.random.seed(2016) 24 | 25 | RESTORE_FROM_LAST_CHECKPOINT = 0 26 | 27 | INPUT_PATH = "../input/" 28 | MODELS_PATH = '../models/' 29 | if not os.path.isdir(MODELS_PATH): 30 | os.mkdir(MODELS_PATH) 31 | OUTPUT_PATH = "../subm/" 32 | if not os.path.isdir(OUTPUT_PATH): 33 | os.mkdir(OUTPUT_PATH) 34 | FEATURES_PATH = "../features/" 35 | if not os.path.isdir(FEATURES_PATH): 36 | os.mkdir(FEATURES_PATH) 37 | CODE_COPY_FOLDER = "../models/code/" 38 | if not os.path.isdir(CODE_COPY_FOLDER): 39 | os.mkdir(CODE_COPY_FOLDER) 40 | HISTORY_FOLDER_PATH = "../models/history/" 41 | if not os.path.isdir(HISTORY_FOLDER_PATH): 42 | os.mkdir(HISTORY_FOLDER_PATH) 43 | CACHE_PATH = "../cache/" 44 | if not os.path.isdir(CACHE_PATH): 45 | os.mkdir(CACHE_PATH) 46 | 47 | 48 | def get_validation_score_land(nfolds, cnn_type): 49 | global FOLD_TO_CALC 50 | from keras.models import load_model 51 | from keras import backend as K 52 | 53 | if K.backend() == 'tensorflow': 54 | print('Update dim ordering to "tf"') 55 | K.set_image_dim_ordering('tf') 56 | 57 | restore_from_cache = 0 58 | 59 | choose = [0, 1, 2, 3, 4, 7, 8, 9, 12, 13, 14, 15, 16] 60 | 61 | tbl = pd.read_csv(INPUT_PATH + "train_v2.csv") 62 | labels = tbl['tags'].apply(lambda x: x.split(' ')) 63 | counts = defaultdict(int) 64 | for l in labels: 65 | for l2 in l: 66 | counts[l2] += 1 67 | indexes = sorted(list(counts.keys())) 68 | for i in range(len(indexes)): 69 | tbl['label_{}'.format(i)] = 0 70 | 71 | files = [] 72 | for id in tbl['image_name'].values: 73 | files.append(INPUT_PATH + "/train-jpg/" + id + '.jpg') 74 | files = np.array(files) 75 | 76 | lbl = np.zeros((len(labels), len(indexes))) 77 | for j in range(len(labels)): 78 | l = labels[j] 79 | for i in range(len(indexes)): 80 | if indexes[i] in l: 81 | lbl[j][i] = 1 82 | 83 | # print(lbl) 84 | print(lbl.shape) 85 | stat = [] 86 | 87 | kf = KFold(n_splits=nfolds, shuffle=True, random_state=get_random_state(cnn_type)) 88 | num_fold = 0 89 | result = np.zeros((len(labels), len(choose))) 90 | for train_ids, valid_ids in kf.split(range(len(files))): 91 | num_fold += 1 92 | 93 | if 'FOLD_TO_CALC' in globals(): 94 | if num_fold not in FOLD_TO_CALC: 95 | continue 96 | 97 | start_time = time.time() 98 | cache_file = CACHE_PATH + '{}_valid_fold_{}_land.pklz'.format(cnn_type, num_fold) 99 | print('Start KFold number {} from {}'.format(num_fold, nfolds)) 100 | print('Split train: ', len(train_ids)) 101 | print('Split valid: ', len(valid_ids)) 102 | valid_files = files[valid_ids] 103 | valid_labels = lbl[valid_ids][:, choose] 104 | if not (os.path.isfile(cache_file) and restore_from_cache): 105 | final_model_path = MODELS_PATH + '{}_fold_{}_land.h5'.format(cnn_type, num_fold) 106 | print('Loading model {}...'.format(final_model_path)) 107 | if cnn_type == 'RESNET101' or cnn_type == 'RESNET152' or 'DENSENET' in cnn_type: 108 | model = get_pretrained_model(cnn_type, len(choose), final_layer_activation='sigmoid') 109 | model.load_weights(final_model_path) 110 | weights = model.layers[-1].get_weights() 111 | print(weights[0].shape) 112 | print(weights[1].shape) 113 | else: 114 | c = dict() 115 | c['f2beta_loss'] = f2beta_loss 116 | model = load_model(final_model_path, custom_objects=c) 117 | preds = get_raw_predictions_for_images_v3(model, cnn_type, valid_files) 118 | save_in_file(preds, cache_file) 119 | else: 120 | preds = load_from_file(cache_file) 121 | 122 | for i in range(len(valid_ids)): 123 | result[valid_ids[i], :] = preds[i] 124 | print(preds.shape) 125 | print(valid_labels.shape) 126 | 127 | best_score = -1 128 | best_thr = -1 129 | for thr1 in range(1, 100): 130 | p = preds.copy() 131 | thr = thr1 / 100 132 | p[p > thr] = 1 133 | p[p <= thr] = 0 134 | score = f2_score(valid_labels, p) 135 | print('THR: {} SCORE: {}'.format(thr, score)) 136 | if score > best_score: 137 | best_score = score 138 | best_thr = thr 139 | 140 | stat.append((best_score, best_thr)) 141 | print('Best score: {} THR: {}'.format(best_score, best_thr)) 142 | print('Fold time: {} seconds'.format(time.time() - start_time)) 143 | # if num_fold == 1: 144 | # exit() 145 | 146 | best_score = -1 147 | best_thr = -1 148 | for thr1 in range(1, 100): 149 | p = result.copy() 150 | thr = thr1 / 100 151 | p[p > thr] = 1 152 | p[p <= thr] = 0 153 | score = f2_score(lbl[:, choose], p) 154 | print('THR: {} SCORE: {}'.format(thr, score)) 155 | if score > best_score: 156 | best_score = score 157 | best_thr = thr 158 | print('Best overall score: {} THR: {}'.format(best_score, best_thr)) 159 | for i in range(len(stat)): 160 | print('Best score fold {}: {} THR: {}'.format(i+1, stat[i][0], stat[i][1])) 161 | 162 | print(result.sum(axis=1)) 163 | 164 | # Save validation file 165 | out = open(FEATURES_PATH + "valid_{}_score_{}_thr_{}_land.csv".format(cnn_type, best_score, best_thr), "w") 166 | # out = open(FEATURES_PATH + "valid_{}_land.csv".format(cnn_type), "w") 167 | out.write("image_name") 168 | for i in choose: 169 | out.write("," + indexes[i]) 170 | out.write("\n") 171 | ids = tbl['image_name'].values 172 | for i in range(len(result)): 173 | out.write(ids[i]) 174 | for j in range(len(choose)): 175 | out.write("," + str(result[i][j])) 176 | out.write("\n") 177 | out.close() 178 | 179 | return best_score, best_thr 180 | 181 | 182 | def process_test_land(nfolds, cnn_type, score, thr): 183 | global FOLD_TO_CALC 184 | from keras.models import load_model 185 | from keras import backend as K 186 | 187 | if K.backend() == 'tensorflow': 188 | print('Update dim ordering to "tf"') 189 | K.set_image_dim_ordering('tf') 190 | 191 | restore_from_cache = 1 192 | choose = [0, 1, 2, 3, 4, 7, 8, 9, 12, 13, 14, 15, 16] 193 | 194 | tbl = pd.read_csv(INPUT_PATH + "sample_submission_v2.csv") 195 | indexes = get_indexes() 196 | ids = tbl['image_name'].values 197 | 198 | files = [] 199 | for id in ids: 200 | files.append("../input/test-jpg/" + id + '.jpg') 201 | files = np.array(files) 202 | # files = files[:100] 203 | 204 | preds = [] 205 | for num_fold in range(1, nfolds+1): 206 | 207 | if 'FOLD_TO_CALC' in globals(): 208 | if num_fold not in FOLD_TO_CALC: 209 | continue 210 | 211 | cache_file = CACHE_PATH + '{}_test_fold_{}_land'.format(cnn_type, num_fold) 212 | print('Start KFold number {} from {}'.format(num_fold, nfolds)) 213 | if os.path.isfile(cache_file) and restore_from_cache: 214 | print('Restore from cache...') 215 | p = load_from_file(cache_file) 216 | else: 217 | final_model_path = MODELS_PATH + '{}_fold_{}_land.h5'.format(cnn_type, num_fold) 218 | print('Loading model {}...'.format(final_model_path)) 219 | if cnn_type == 'RESNET101' or cnn_type == 'RESNET152' or 'DENSENET' in cnn_type: 220 | model = get_pretrained_model(cnn_type, len(choose), final_layer_activation='sigmoid') 221 | model.load_weights(final_model_path) 222 | else: 223 | c = dict() 224 | c['f2beta_loss'] = f2beta_loss 225 | model = load_model(final_model_path, custom_objects=c) 226 | p = get_raw_predictions_for_images_v3(model, cnn_type, files) 227 | save_in_file(p, cache_file) 228 | preds.append(p) 229 | preds = np.array(preds) 230 | print(preds.shape) 231 | preds = np.mean(preds, axis=0) 232 | 233 | # Save raw file 234 | out = open(FEATURES_PATH + "test_{}_score_{}_thr_{}_land.csv".format(cnn_type, score, thr), "w") 235 | out.write("image_name") 236 | for i in choose: 237 | out.write("," + indexes[i]) 238 | out.write("\n") 239 | ids = tbl['image_name'].values 240 | for i in range(len(preds)): 241 | out.write(ids[i]) 242 | for j in range(len(choose)): 243 | out.write("," + str(preds[i][j])) 244 | out.write("\n") 245 | out.close() 246 | 247 | # Create submission 248 | out = open(OUTPUT_PATH + "subm_{}_score_{}_thr_{}_land.csv".format(cnn_type, score, thr), "w") 249 | out.write("image_name,tags\n") 250 | for i in range(len(files)): 251 | out.write(ids[i] + ',') 252 | for j in range(len(choose)): 253 | if preds[i][j] > thr: 254 | out.write(indexes[j] + ' ') 255 | out.write("\n") 256 | out.close() 257 | 258 | 259 | if __name__ == '__main__': 260 | num_folds = 5 261 | for cnn in ['DENSENET_121']: 262 | best_score, best_thr = get_validation_score_land(num_folds, cnn) 263 | process_test_land(num_folds, cnn, best_score, best_thr) 264 | 265 | 266 | ''' 267 | Validation result: 268 | 269 | Best overall score: 0.8783776613969391 THR: 0.19 270 | Best score fold 1: 0.8717132327773861 THR: 0.21 271 | Best score fold 2: 0.8810646984891833 THR: 0.22 272 | Best score fold 3: 0.8828568139672164 THR: 0.19 273 | Best score fold 4: 0.8775218783366753 THR: 0.21 274 | Best score fold 5: 0.8796787233198925 THR: 0.19 275 | ''' -------------------------------------------------------------------------------- /a31_create_cnn_features_weather.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | __author__ = 'ZFTurbo: https://kaggle.com/zfturbo' 3 | 4 | from a00_common_functions import * 5 | 6 | GPU_TO_USE = 0 7 | USE_THEANO = 1 8 | 9 | # Uncomment if you need to calculate specific fold 10 | # FOLD_TO_CALC = [5] 11 | 12 | if USE_THEANO: 13 | os.environ["KERAS_BACKEND"] = "theano" 14 | os.environ["THEANO_FLAGS"] = "device=gpu{},lib.cnmem=0.81".format(GPU_TO_USE, GPU_TO_USE) 15 | else: 16 | os.environ["KERAS_BACKEND"] = "tensorflow" 17 | os.environ["CUDA_VISIBLE_DEVICES"] = "{}".format(GPU_TO_USE) 18 | 19 | import random 20 | from a02_zoo import * 21 | 22 | random.seed(2016) 23 | np.random.seed(2016) 24 | 25 | RESTORE_FROM_LAST_CHECKPOINT = 0 26 | 27 | INPUT_PATH = "../input/" 28 | MODELS_PATH = '../models/' 29 | if not os.path.isdir(MODELS_PATH): 30 | os.mkdir(MODELS_PATH) 31 | OUTPUT_PATH = "../subm/" 32 | if not os.path.isdir(OUTPUT_PATH): 33 | os.mkdir(OUTPUT_PATH) 34 | FEATURES_PATH = "../features/" 35 | if not os.path.isdir(FEATURES_PATH): 36 | os.mkdir(FEATURES_PATH) 37 | CODE_COPY_FOLDER = "../models/code/" 38 | if not os.path.isdir(CODE_COPY_FOLDER): 39 | os.mkdir(CODE_COPY_FOLDER) 40 | HISTORY_FOLDER_PATH = "../models/history/" 41 | if not os.path.isdir(HISTORY_FOLDER_PATH): 42 | os.mkdir(HISTORY_FOLDER_PATH) 43 | CACHE_PATH = "../cache/" 44 | if not os.path.isdir(CACHE_PATH): 45 | os.mkdir(CACHE_PATH) 46 | 47 | 48 | def get_validation_score_weather(nfolds, cnn_type): 49 | global FOLD_TO_CALC 50 | from keras.models import load_model 51 | from keras import backend as K 52 | 53 | if K.backend() == 'tensorflow': 54 | print('Update dim ordering to "tf"') 55 | K.set_image_dim_ordering('tf') 56 | 57 | restore_from_cache = 0 58 | 59 | choose = [5, 6, 10, 11] 60 | 61 | tbl = pd.read_csv(INPUT_PATH + "train_v2.csv") 62 | labels = tbl['tags'].apply(lambda x: x.split(' ')) 63 | counts = defaultdict(int) 64 | for l in labels: 65 | for l2 in l: 66 | counts[l2] += 1 67 | indexes = sorted(list(counts.keys())) 68 | for i in range(len(indexes)): 69 | tbl['label_{}'.format(i)] = 0 70 | 71 | files = [] 72 | for id in tbl['image_name'].values: 73 | files.append("../input/train-jpg/" + id + '.jpg') 74 | files = np.array(files) 75 | 76 | lbl = np.zeros((len(labels), len(indexes))) 77 | for j in range(len(labels)): 78 | l = labels[j] 79 | for i in range(len(indexes)): 80 | if indexes[i] in l: 81 | lbl[j][i] = 1 82 | 83 | # print(lbl) 84 | print(lbl.shape) 85 | stat = [] 86 | 87 | kf = KFold(n_splits=nfolds, shuffle=True, random_state=get_random_state(cnn_type)) 88 | num_fold = 0 89 | result = np.zeros((len(labels), len(choose))) 90 | for train_ids, valid_ids in kf.split(range(len(files))): 91 | num_fold += 1 92 | 93 | if 'FOLD_TO_CALC' in globals(): 94 | if num_fold not in FOLD_TO_CALC: 95 | continue 96 | 97 | start_time = time.time() 98 | cache_file = CACHE_PATH + '{}_valid_fold_{}_weather'.format(cnn_type, num_fold) 99 | print('Start KFold number {} from {}'.format(num_fold, nfolds)) 100 | print('Split train: ', len(train_ids)) 101 | print('Split valid: ', len(valid_ids)) 102 | valid_files = files[valid_ids] 103 | valid_labels = lbl[valid_ids][:, choose] 104 | if not (os.path.isfile(cache_file) and restore_from_cache): 105 | final_model_path = MODELS_PATH + '{}_fold_{}_weather.h5'.format(cnn_type, num_fold) 106 | print('Loading model {}...'.format(final_model_path)) 107 | if cnn_type == 'RESNET101' or cnn_type == 'RESNET152' or 'DENSENET' in cnn_type: 108 | model = get_pretrained_model(cnn_type, len(choose), final_layer_activation='softmax') 109 | model.load_weights(final_model_path) 110 | weights = model.layers[-1].get_weights() 111 | print(weights[0].shape) 112 | print(weights[1].shape) 113 | else: 114 | c = dict() 115 | c['f2beta_loss'] = f2beta_loss 116 | model = load_model(final_model_path, custom_objects=c) 117 | preds = get_raw_predictions_for_images_v3(model, cnn_type, valid_files) 118 | save_in_file(preds, cache_file) 119 | else: 120 | preds = load_from_file(cache_file) 121 | 122 | for i in range(len(valid_ids)): 123 | result[valid_ids[i], :] = preds[i] 124 | print(preds.shape) 125 | print(valid_labels.shape) 126 | 127 | best_score = -1 128 | best_thr = -1 129 | for thr1 in range(1, 100): 130 | p = preds.copy() 131 | thr = thr1 / 100 132 | p[p > thr] = 1 133 | p[p <= thr] = 0 134 | score = f2_score(valid_labels, p) 135 | print('THR: {} SCORE: {}'.format(thr, score)) 136 | if score > best_score: 137 | best_score = score 138 | best_thr = thr 139 | 140 | stat.append((best_score, best_thr)) 141 | print('Best score: {} THR: {}'.format(best_score, best_thr)) 142 | print('Fold time: {} seconds'.format(time.time() - start_time)) 143 | # if num_fold == 1: 144 | # exit() 145 | 146 | best_score = -1 147 | best_thr = -1 148 | for thr1 in range(1, 100): 149 | p = result.copy() 150 | thr = thr1 / 100 151 | p[p > thr] = 1 152 | p[p <= thr] = 0 153 | score = f2_score(lbl[:, choose], p) 154 | print('THR: {} SCORE: {}'.format(thr, score)) 155 | if score > best_score: 156 | best_score = score 157 | best_thr = thr 158 | print('Best overall score: {} THR: {}'.format(best_score, best_thr)) 159 | for i in range(len(stat)): 160 | print('Best score fold {}: {} THR: {}'.format(i+1, stat[i][0], stat[i][1])) 161 | 162 | # Save validation file 163 | out = open(FEATURES_PATH + "valid_{}_score_{}_thr_{}_weather.csv".format(cnn_type, best_score, best_thr), "w") 164 | # out = open(FEATURES_PATH + "valid_{}_weather.csv".format(cnn_type), "w") 165 | out.write("image_name") 166 | for i in choose: 167 | out.write("," + indexes[i]) 168 | out.write("\n") 169 | ids = tbl['image_name'].values 170 | for i in range(len(result)): 171 | out.write(ids[i]) 172 | for j in range(len(choose)): 173 | out.write("," + str(result[i][j])) 174 | out.write("\n") 175 | out.close() 176 | 177 | return best_score, best_thr 178 | 179 | 180 | def process_test_weather(nfolds, cnn_type, score, thr): 181 | global FOLD_TO_CALC 182 | from keras.models import load_model 183 | from keras import backend as K 184 | 185 | if K.backend() == 'tensorflow': 186 | print('Update dim ordering to "tf"') 187 | K.set_image_dim_ordering('tf') 188 | 189 | restore_from_cache = 0 190 | choose = [5, 6, 10, 11] 191 | 192 | tbl = pd.read_csv(INPUT_PATH + "sample_submission_v2.csv") 193 | indexes = get_indexes() 194 | ids = tbl['image_name'].values 195 | 196 | files = [] 197 | for id in ids: 198 | files.append(INPUT_PATH + "test-jpg/" + id + '.jpg') 199 | files = np.array(files) 200 | 201 | preds = [] 202 | for num_fold in range(1, nfolds+1): 203 | 204 | if 'FOLD_TO_CALC' in globals(): 205 | if num_fold not in FOLD_TO_CALC: 206 | continue 207 | 208 | cache_file = CACHE_PATH + '{}_test_fold_{}_weather'.format(cnn_type, num_fold) 209 | print('Start KFold number {} from {}'.format(num_fold, nfolds)) 210 | if os.path.isfile(cache_file) and restore_from_cache: 211 | print('Restore from cache...') 212 | p = load_from_file(cache_file) 213 | else: 214 | final_model_path = MODELS_PATH + '{}_fold_{}_weather.h5'.format(cnn_type, num_fold) 215 | print('Loading model {}...'.format(final_model_path)) 216 | if cnn_type == 'RESNET101' or cnn_type == 'RESNET152' or 'DENSENET' in cnn_type: 217 | model = get_pretrained_model(cnn_type, len(choose), final_layer_activation='softmax') 218 | model.load_weights(final_model_path) 219 | else: 220 | c = dict() 221 | c['f2beta_loss'] = f2beta_loss 222 | model = load_model(final_model_path, custom_objects=c) 223 | p = get_raw_predictions_for_images_v3(model, cnn_type, files) 224 | save_in_file(p, cache_file) 225 | preds.append(p) 226 | preds = np.array(preds) 227 | print(preds.shape) 228 | preds = np.mean(preds, axis=0) 229 | 230 | # Save raw file 231 | out = open(FEATURES_PATH + "test_{}_score_{}_thr_{}_weather.csv".format(cnn_type, score, thr), "w") 232 | out.write("image_name") 233 | for i in choose: 234 | out.write("," + indexes[i]) 235 | out.write("\n") 236 | ids = tbl['image_name'].values 237 | for i in range(len(preds)): 238 | out.write(ids[i]) 239 | for j in range(len(choose)): 240 | out.write("," + str(preds[i][j])) 241 | out.write("\n") 242 | out.close() 243 | 244 | # Create submission 245 | out = open(OUTPUT_PATH + "subm_{}_score_{}_thr_{}_weather.csv".format(cnn_type, score, thr), "w") 246 | out.write("image_name,tags\n") 247 | for i in range(len(files)): 248 | out.write(ids[i] + ',') 249 | for j in range(len(choose)): 250 | if preds[i][j] > thr: 251 | out.write(indexes[j] + ' ') 252 | out.write("\n") 253 | out.close() 254 | 255 | 256 | if __name__ == '__main__': 257 | num_folds = 5 258 | for cnn in ['DENSENET_121']: 259 | best_score, best_thr = get_validation_score_weather(num_folds, cnn) 260 | process_test_weather(num_folds, cnn, best_score, best_thr) 261 | 262 | 263 | ''' 264 | Validation result: 265 | 266 | Best overall score: 0.9541445064401411 THR: 0.18 267 | Best score fold 1: 0.9531132364012799 THR: 0.16 268 | Best score fold 2: 0.9549042443064183 THR: 0.21 269 | Best score fold 3: 0.9564864483342744 THR: 0.22 270 | Best score fold 4: 0.953799201251647 THR: 0.18 271 | Best score fold 5: 0.9557163445983705 THR: 0.23 272 | ''' -------------------------------------------------------------------------------- /a32_create_cnn_features_single_class.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | __author__ = 'ZFTurbo: https://kaggle.com/zfturbo' 3 | 4 | from a00_common_functions import * 5 | from sklearn.metrics import log_loss, roc_auc_score 6 | 7 | GPU_TO_USE = 0 8 | USE_THEANO = 1 9 | 10 | if USE_THEANO: 11 | os.environ["KERAS_BACKEND"] = "theano" 12 | os.environ["THEANO_FLAGS"] = "device=gpu{},lib.cnmem=0.81".format(GPU_TO_USE, GPU_TO_USE) 13 | else: 14 | os.environ["KERAS_BACKEND"] = "tensorflow" 15 | os.environ["CUDA_VISIBLE_DEVICES"] = "{}".format(GPU_TO_USE) 16 | 17 | import random 18 | from a02_zoo import * 19 | 20 | random.seed(2016) 21 | np.random.seed(2016) 22 | 23 | RESTORE_FROM_LAST_CHECKPOINT = 0 24 | 25 | INPUT_PATH = "../input/" 26 | MODELS_PATH = '../models/' 27 | if not os.path.isdir(MODELS_PATH): 28 | os.mkdir(MODELS_PATH) 29 | OUTPUT_PATH = "../subm/" 30 | if not os.path.isdir(OUTPUT_PATH): 31 | os.mkdir(OUTPUT_PATH) 32 | FEATURES_PATH = "../features/" 33 | if not os.path.isdir(FEATURES_PATH): 34 | os.mkdir(FEATURES_PATH) 35 | CODE_COPY_FOLDER = "../models/code/" 36 | if not os.path.isdir(CODE_COPY_FOLDER): 37 | os.mkdir(CODE_COPY_FOLDER) 38 | HISTORY_FOLDER_PATH = "../models/history/" 39 | if not os.path.isdir(HISTORY_FOLDER_PATH): 40 | os.mkdir(HISTORY_FOLDER_PATH) 41 | CACHE_PATH = "../cache/" 42 | if not os.path.isdir(CACHE_PATH): 43 | os.mkdir(CACHE_PATH) 44 | 45 | 46 | def get_validation_score_single_class(nfolds, class_id, cnn_type): 47 | from keras.models import load_model 48 | from keras import backend as K 49 | 50 | if K.backend() == 'tensorflow': 51 | print('Update dim ordering to "tf"') 52 | K.set_image_dim_ordering('tf') 53 | 54 | restore_from_cache = 0 55 | 56 | tbl = pd.read_csv(INPUT_PATH + "train_v2.csv") 57 | labels = tbl['tags'].apply(lambda x: x.split(' ')) 58 | counts = defaultdict(int) 59 | for l in labels: 60 | for l2 in l: 61 | counts[l2] += 1 62 | indexes = sorted(list(counts.keys())) 63 | for i in range(len(indexes)): 64 | tbl['label_{}'.format(i)] = 0 65 | 66 | files = [] 67 | for id in tbl['image_name'].values: 68 | files.append("../input/train-jpg/" + id + '.jpg') 69 | files = np.array(files) 70 | 71 | lbl = np.zeros((len(labels), len(indexes))) 72 | for j in range(len(labels)): 73 | l = labels[j] 74 | for i in range(len(indexes)): 75 | if indexes[i] in l: 76 | lbl[j][i] = 1 77 | 78 | print(lbl.shape) 79 | stat = [] 80 | 81 | kf = KFold(n_splits=nfolds, shuffle=True, random_state=get_random_state(cnn_type)) 82 | num_fold = 0 83 | result = np.zeros(len(labels)) 84 | for train_ids, valid_ids in kf.split(range(len(files))): 85 | num_fold += 1 86 | start_time = time.time() 87 | cache_file = CACHE_PATH + '{}_valid_fold_{}_single_class_{}'.format(cnn_type, num_fold, class_id) 88 | print('Start KFold number {} from {}'.format(num_fold, nfolds)) 89 | print('Split train: ', len(train_ids)) 90 | print('Split valid: ', len(valid_ids)) 91 | valid_files = files[valid_ids] 92 | valid_labels = lbl[valid_ids, class_id] 93 | if not (os.path.isfile(cache_file) and restore_from_cache): 94 | final_model_path = MODELS_PATH + '{}_fold_{}_single_class_{}.h5'.format(cnn_type, num_fold, class_id) 95 | print('Loading model {}...'.format(final_model_path)) 96 | model = get_pretrained_model(cnn_type, 1, final_layer_activation='sigmoid') 97 | model.load_weights(final_model_path) 98 | preds = get_raw_predictions_for_images(model, cnn_type, valid_files, 8) 99 | save_in_file(preds, cache_file) 100 | else: 101 | preds = load_from_file(cache_file) 102 | 103 | if len(preds[np.isnan(preds)]) > 0: 104 | print('There are {} NANs in pred! Fix them with mean'.format(len(preds[np.isnan(preds)]))) 105 | mean = preds[~np.isnan(preds)].mean() 106 | preds[np.isnan(preds)] = mean 107 | result[valid_ids] = preds 108 | print(preds.shape) 109 | print(valid_labels.shape) 110 | 111 | best_score = log_loss(valid_labels, preds) 112 | print('Best log loss: {}'.format(best_score)) 113 | best_score = roc_auc_score(valid_labels, preds) 114 | print('Best AUC: {}'.format(best_score)) 115 | print('Fold time: {} seconds'.format(time.time() - start_time)) 116 | 117 | best_score = log_loss(lbl[:, class_id], result) 118 | print('Best log loss score: {}'.format(best_score)) 119 | best_score = roc_auc_score(lbl[:, class_id], result) 120 | print('Best AUC score: {}'.format(best_score)) 121 | 122 | # Save validation file 123 | out = open(FEATURES_PATH + "valid_{}_single_class_{}_score_{}.csv".format(cnn_type, class_id, best_score), "w") 124 | # out = open(FEATURES_PATH + "valid_{}_single_class_{}.csv".format(cnn_type, class_id), "w") 125 | out.write("image_name") 126 | out.write("," + indexes[class_id]) 127 | out.write("\n") 128 | ids = tbl['image_name'].values 129 | for i in range(len(result)): 130 | out.write(ids[i]) 131 | out.write("," + str(result[i])) 132 | out.write("\n") 133 | out.close() 134 | 135 | return best_score 136 | 137 | 138 | def process_test_single_class(nfolds, cnn_type, class_id, score): 139 | global FOLD_TO_CALC 140 | from keras.models import load_model 141 | from keras import backend as K 142 | 143 | if K.backend() == 'tensorflow': 144 | print('Update dim ordering to "tf"') 145 | K.set_image_dim_ordering('tf') 146 | 147 | restore_from_cache = 1 148 | 149 | tbl = pd.read_csv(INPUT_PATH + "sample_submission_v2.csv") 150 | indexes = get_indexes() 151 | ids = tbl['image_name'].values 152 | 153 | files = [] 154 | for id in ids: 155 | files.append("../input/test-jpg/" + id + '.jpg') 156 | files = np.array(files) 157 | # files = files[:100] 158 | 159 | preds = [] 160 | for num_fold in range(1, nfolds+1): 161 | start_time = time.time() 162 | 163 | if 'FOLD_TO_CALC' in globals(): 164 | if num_fold not in FOLD_TO_CALC: 165 | continue 166 | 167 | cache_file = CACHE_PATH + '{}_test_fold_{}_single_class_{}'.format(cnn_type, num_fold, class_id) 168 | print('Start KFold number {} from {}'.format(num_fold, nfolds)) 169 | if os.path.isfile(cache_file) and restore_from_cache: 170 | print('Restore from cache...') 171 | p = load_from_file(cache_file) 172 | else: 173 | final_model_path = MODELS_PATH + '{}_fold_{}_single_class_{}.h5'.format(cnn_type, num_fold, class_id) 174 | print('Loading model {}...'.format(final_model_path)) 175 | model = get_pretrained_model(cnn_type, 1, final_layer_activation='sigmoid') 176 | model.load_weights(final_model_path) 177 | p = get_raw_predictions_for_images(model, cnn_type, files, 8) 178 | save_in_file(p, cache_file) 179 | preds.append(p) 180 | preds = np.array(preds) 181 | print(preds.shape) 182 | preds = np.mean(preds, axis=0) 183 | 184 | # Save raw file 185 | out = open(FEATURES_PATH + "test_{}_single_class_{}_score_{}.csv".format(cnn_type, class_id, score), "w") 186 | out.write("image_name") 187 | out.write("," + indexes[class_id]) 188 | out.write("\n") 189 | ids = tbl['image_name'].values 190 | for i in range(len(preds)): 191 | out.write(ids[i]) 192 | out.write("," + str(preds[i][0])) 193 | out.write("\n") 194 | out.close() 195 | 196 | 197 | if __name__ == '__main__': 198 | num_folds = 5 199 | for class_id in range(1, 17): 200 | for cnn in ['DENSENET_121']: 201 | best_score = get_validation_score_single_class(num_folds, class_id, cnn) 202 | process_test_single_class(num_folds, cnn, class_id, best_score) 203 | 204 | 205 | ''' 206 | Validation results: 207 | 208 | Class 0 (24 augment): 209 | Fold 1: 210 | Best log loss: 0.22772516145361263 211 | Best AUC: 0.9681611676805534 212 | Fold 2: 213 | Best log loss: 0.24578336946242255 214 | Best AUC: 0.9651904325699159 215 | Fold 3: 216 | Best log loss: 0.24523281635632213 217 | Best AUC: 0.9648608595328034 218 | Fold 4: 219 | Best log loss: 0.2655470102476548 220 | Best AUC: 0.9635775854593888 221 | Fold 5: 222 | Best log loss: 0.24445596844166212 223 | Best AUC: 0.9635897933788119 224 | Best log loss score: 0.24574889719927384 225 | Best AUC score: 0.9643620758364253 226 | 227 | New validation (8 augment): 228 | 229 | Class 4: 230 | Best log loss: 0.19271503292580192 231 | Best AUC: 0.9154468633958038 232 | Best log loss: 0.20040894672624915 233 | Best AUC: 0.954901232680224 234 | Best log loss: 0.15068588557553242 235 | Best AUC: 0.938265249604139 236 | Best log loss: 0.2462618427549412 237 | Best AUC: 0.9301456936283611 238 | Best log loss: 0.23098790161785845 239 | Best AUC: 0.9130177596192206 240 | Best log loss score: 0.2042112602743943 241 | Best AUC score: 0.9274156010934623 242 | 243 | Class 1: 244 | Best log loss: 0.03811951119234441 245 | Best AUC: 0.9951909464526258 246 | Best log loss: 0.048754306974963635 247 | Best AUC: 0.9984422903143837 248 | Best log loss: 0.06174141446384239 249 | Best AUC: 0.9921879692151934 250 | Best log loss: 0.030328301687537473 251 | Best AUC: 0.9989694385216773 252 | Best log loss: 0.04053232867061804 253 | Best AUC: 0.9966355784605692 254 | Fold time: 728.634740114212 seconds 255 | Best log loss score: 0.04389525566509881 256 | Best AUC score: 0.9967844843931196 257 | 258 | Class 5: 259 | Best log loss: 0.13754383867745576 260 | Best AUC: 0.9898741534977363 261 | Best log loss: 0.1407097081301429 262 | Best AUC: 0.9884055385718593 263 | Best log loss: 0.14993728358783198 264 | Best AUC: 0.9891054665926355 265 | Best log loss: 0.1479958416072093 266 | Best AUC: 0.9901369790230676 267 | Best log loss: 0.13113664577419135 268 | Best AUC: 0.9887792855989967 269 | Fold time: 803.063668012619 seconds 270 | Best log loss score: 0.14146491865284122 271 | Best AUC score: 0.9891344529494603 272 | 273 | Class 6: 274 | Best log loss: 0.1013346719765783 275 | Best AUC: 0.9932364226284082 276 | Best log loss: 0.11845386073228246 277 | Best AUC: 0.9939338697510687 278 | Best log loss: 0.10418842342835029 279 | Best AUC: 0.9923096942759018 280 | Best log loss: 0.0936938813620891 281 | Best AUC: 0.9953129506560875 282 | Best log loss: 0.0887781298560271 283 | Best AUC: 0.9955662606053148 284 | Fold time: 690.0189571380615 seconds 285 | Best log loss score: 0.10129010250387299 286 | Best AUC score: 0.9939780821432701 287 | 288 | Class 2: 289 | Best log loss: 0.3193028522943171 290 | Best AUC: 0.9168173985473191 291 | Best log loss: 0.29456970135771376 292 | Best AUC: 0.9214497759336571 293 | Best log loss: 0.2815473452201511 294 | Best AUC: 0.9249727894921321 295 | Best log loss: 0.29912306827751317 296 | Best AUC: 0.9347493342197668 297 | Best log loss: 0.29637484576337675 298 | Best AUC: 0.938331896766066 299 | Fold time: 617.7306578159332 seconds 300 | Best log loss score: 0.2981836069756931 301 | Best AUC score: 0.9272408748804607 302 | 303 | Class 3: 304 | Best log loss: 0.2915650288312323 305 | Best AUC: 0.9327988659329712 306 | Best log loss: 0.29132380419460807 307 | Best AUC: 0.9346460885945589 308 | Best log loss: 0.2261809924541458 309 | Best AUC: 0.9743947653799936 310 | Best log loss: 0.2695016681096027 311 | Best AUC: 0.9528340647768844 312 | Best log loss: 0.3026091752105957 313 | Best AUC: 0.9159740381023533 314 | Fold time: 607.7529680728912 seconds 315 | Best log loss score: 0.27623548212075444 316 | Best AUC score: 0.9445849004906965 317 | 318 | Class 7: 319 | Best log loss: 0.10770234185427274 320 | Best AUC: 0.9885339753014611 321 | Best log loss: 0.08665580671766984 322 | Best AUC: 0.9938273514851486 323 | Best log loss: 0.10820953582724398 324 | Best AUC: 0.9902734810048262 325 | Best log loss: 0.06758468670264607 326 | Best AUC: 0.9873491347971499 327 | Best log loss: 0.040493334328091525 328 | Best AUC: 0.9953300728228552 329 | Fold time: 738.5424618721008 seconds 330 | Best log loss score: 0.08213016964691523 331 | Best AUC score: 0.9898691151341044 332 | 333 | Class 12: 334 | Best log loss: 0.1762134442716055 335 | Best AUC: 0.9853955942900918 336 | Best log loss: 0.17752719105933168 337 | Best AUC: 0.9843507889270774 338 | Best log loss: 0.17138769259719164 339 | Best AUC: 0.9830966247941769 340 | Best log loss: 0.1590568591193237 341 | Best AUC: 0.9874772203614748 342 | Best log loss: 0.13316565771373035 343 | Best AUC: 0.986358030102203 344 | Fold time: 619.8796629905701 seconds 345 | Best log loss score: 0.16347091746764283 346 | Best AUC score: 0.9845900937304197 347 | 348 | Class 10: 349 | Best log loss: 0.21512476057973134 350 | Best AUC: 0.9713505690415105 351 | Best log loss: 0.20589928443105426 352 | Best AUC: 0.973395825689021 353 | Best log loss: 0.2008245884089307 354 | Best AUC: 0.9709878360815531 355 | Best log loss: 0.21708911002298573 356 | Best AUC: 0.9738312645732162 357 | Best log loss: 0.2076587643386939 358 | Best AUC: 0.973662198972203 359 | Fold time: 598.0848441123962 seconds 360 | Best log loss score: 0.2093193424692287 361 | Best AUC score: 0.9725594023611087 362 | 363 | Class 11: 364 | Best log loss: 0.09679505293445312 365 | Best AUC: 0.9934267337278585 366 | Best log loss: 0.10177069504633214 367 | Best AUC: 0.9938068291132358 368 | Best log loss: 0.09597090466566603 369 | Best AUC: 0.9936317460069493 370 | Best log loss: 0.1074384845116223 371 | Best AUC: 0.9936855283653945 372 | Best log loss: 0.13955825222181115 373 | Best AUC: 0.9909820947620634 374 | Fold time: 682.8519070148468 seconds 375 | Best log loss score: 0.10830590562199588 376 | Best AUC score: 0.9930041927993318 377 | 378 | Class 14: 379 | Best log loss: 0.18723967045451514 380 | Best AUC: 0.9522946870213888 381 | Best log loss: 0.2737099454922427 382 | Best AUC: 0.9381282085033567 383 | Best log loss: 0.26238505813365587 384 | Best AUC: 0.9538068585643212 385 | Best log loss: 0.23004463341357106 386 | Best AUC: 0.9678736277445109 387 | Best log loss: 0.22609733187008046 388 | Best AUC: 0.9602489110143124 389 | Fold time: 705.9609940052032 seconds 390 | Best log loss score: 0.23589556996675196 391 | Best AUC score: 0.953874770466746 392 | 393 | Class 8: 394 | Best log loss: 0.42778759389878684 395 | Best AUC: 0.9231615100013267 396 | Best log loss: 0.3170926169958293 397 | Best AUC: 0.9254201946965472 398 | Best log loss: 0.39044749693707115 399 | Best AUC: 0.9225318030973451 400 | Best log loss: 0.3933927935731148 401 | Best AUC: 0.9361093947609957 402 | Best log loss: 0.318176923364811 403 | Best AUC: 0.9225128061653657 404 | Fold time: 802.30384516716 seconds 405 | Best log loss score: 0.3693807498092545 406 | Best AUC score: 0.9239076007702497 407 | 408 | Class 9: 409 | Best log loss: 0.19298712914387076 410 | Best AUC: 0.9671239105721865 411 | Best log loss: 0.21576608148377555 412 | Best AUC: 0.969628411966739 413 | Best log loss: 0.17453910143247506 414 | Best AUC: 0.9733059192056234 415 | Best log loss: 0.18491012337760004 416 | Best AUC: 0.9690910716472565 417 | Best log loss: 0.19258769059213018 418 | Best AUC: 0.9689034194174346 419 | Fold time: 668.7590019702911 seconds 420 | Best log loss score: 0.19215801456555895 421 | Best AUC score: 0.9693925215613167 422 | 423 | Class 13: 424 | Best log loss: 0.18746864205839933 425 | Best AUC: 0.9768615485494193 426 | Best log loss: 0.20173310505902728 427 | Best AUC: 0.9749429994128367 428 | Best log loss: 0.2061426603520332 429 | Best AUC: 0.9790132993037375 430 | Best log loss: 0.19074217667342347 431 | Best AUC: 0.9743706430198277 432 | Best log loss: 0.2011499023623359 433 | Best AUC: 0.9761030213325245 434 | Fold time: 670.308308839798 seconds 435 | Best log loss score: 0.1974472058461548 436 | Best AUC score: 0.97613310930843 437 | 438 | Class 15: 439 | Best log loss: 0.23077802701389574 440 | Best AUC: 0.9445833250004999 441 | Best log loss: 0.22811947313801614 442 | Best AUC: 0.9491929005852426 443 | Best log loss: 0.24251130014276662 444 | Best AUC: 0.9489624751491053 445 | Best log loss: 0.26617685795212054 446 | Best AUC: 0.9587954435605022 447 | Best log loss: 0.19164812884391486 448 | Best AUC: 0.9563842333954066 449 | Fold time: 606.2063620090485 seconds 450 | Best log loss score: 0.23184775060083418 451 | Best AUC score: 0.9507165151970609 452 | 453 | Class 16: 454 | Best log loss: 0.256431130564825 455 | Best AUC: 0.9644717785585375 456 | Best log loss: 0.22385005951884268 457 | Best AUC: 0.9635620465641866 458 | Best log loss: 0.25767038116700974 459 | Best AUC: 0.9654633224892433 460 | Best log loss: 0.22202993941841004 461 | Best AUC: 0.964412043352981 462 | Best log loss: 0.2630350409576729 463 | Best AUC: 0.9581374413632833 464 | Fold time: 603.998811006546 seconds 465 | Best log loss score: 0.24460285483104532 466 | Best AUC score: 0.9628241034772262 467 | ''' -------------------------------------------------------------------------------- /a32_find_neighbours_features.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | __author__ = 'ZFTurbo: https://kaggle.com/zfturbo' 3 | 4 | import os 5 | import shutil 6 | os.environ["THEANO_FLAGS"] = "device=cpu" 7 | from a00_common_functions import * 8 | from multiprocessing import Process, Manager 9 | 10 | 11 | INPUT_PATH = "../input/" 12 | OUTPUT_PATH = "../modified_data/" 13 | FEATURES_PATH = "../features/" 14 | if not os.path.isdir(FEATURES_PATH): 15 | os.mkdir(FEATURES_PATH) 16 | 17 | 18 | def get_list_of_CNN_features(): 19 | files = glob.glob(FEATURES_PATH + 'valid_*.csv') 20 | train_files = [] 21 | test_files = [] 22 | for f in files: 23 | if '_land' in f: 24 | continue 25 | if '_weather' in f: 26 | continue 27 | if '_single_class' in f: 28 | continue 29 | train_files.append(f) 30 | test_files.append(f.replace('valid_', 'test_')) 31 | # print(train_files[-1]) 32 | # print(test_files[-1]) 33 | return train_files, test_files 34 | 35 | 36 | def get_features_for_table(t): 37 | features = list(set(t.columns.values) - {'image_name'}) 38 | return features 39 | 40 | 41 | def create_4_neighbours_features(prefix): 42 | indexes = get_indexes() 43 | train = pd.read_csv("../input/train_v2.csv")[['image_name']] 44 | test = pd.read_csv("../input/sample_submission_v2.csv")[['image_name']] 45 | for i in indexes: 46 | train[i] = -1 47 | test[i] = -1 48 | ne = pd.read_csv('../modified_data/' + prefix + '.csv') 49 | median = [-1]*len(indexes) 50 | 51 | train_files, test_files = get_list_of_CNN_features() 52 | 53 | print('Reading data...') 54 | train_s = pd.read_csv(train_files[0]) 55 | for i in range(1, len(train_files)): 56 | s = pd.read_csv(train_files[i]) 57 | train_s[indexes] += s[indexes] 58 | train_s[indexes] /= len(train_files) 59 | 60 | test_s = pd.read_csv(test_files[0]) 61 | for i in range(1, len(test_files)): 62 | s = pd.read_csv(test_files[i]) 63 | test_s[indexes] += s[indexes] 64 | test_s[indexes] /= len(test_files) 65 | 66 | full = pd.concat((train_s, test_s), axis=0) 67 | full.reset_index(drop=True, inplace=True) 68 | 69 | for i in range(len(indexes)): 70 | nm = indexes[i] 71 | median[i] = np.float64(full[[nm]].median()) 72 | 73 | for index, row in ne.iterrows(): 74 | id = row['id'] 75 | val = np.zeros(len(indexes)) 76 | count = 0 77 | for nm in ['ntop', 'nbottom', 'nleft', 'nright']: 78 | if row[nm] != 'fake': 79 | v = full[full['image_name'] == row[nm]][indexes].as_matrix() 80 | val += v[0] 81 | count += 1 82 | 83 | # print(id, count) 84 | if count == 0: 85 | if 'train_' in id: 86 | train.loc[train['image_name'] == id, indexes] = median 87 | else: 88 | test.loc[test['image_name'] == id, indexes] = median 89 | else: 90 | val /= count 91 | if 'train_' in id: 92 | train.loc[train['image_name'] == id, indexes] = val 93 | else: 94 | test.loc[test['image_name'] == id, indexes] = val 95 | 96 | if (index + 1) % 1000 == 0: 97 | print('Completed {} from {}'.format(index + 1, len(full))) 98 | 99 | train.to_csv(FEATURES_PATH + "feature_neighbours_4_mean_" + prefix + "_train.csv", index=False) 100 | test.to_csv(FEATURES_PATH + "feature_neighbours_4_mean_" + prefix + "_test.csv", index=False) 101 | 102 | 103 | def create_8_neighbours_features(prefix): 104 | indexes = get_indexes() 105 | train = pd.read_csv("../input/train_v2.csv")[['image_name']] 106 | test = pd.read_csv("../input/sample_submission_v2.csv")[['image_name']] 107 | for i in indexes: 108 | train[i] = -1 109 | test[i] = -1 110 | ne = pd.read_csv('../modified_data/' + prefix + '_8.csv') 111 | median = [-1]*len(indexes) 112 | 113 | train_files, test_files = get_list_of_CNN_features() 114 | 115 | print('Reading data...') 116 | train_s = pd.read_csv(train_files[0]) 117 | for i in range(1, len(train_files)): 118 | s = pd.read_csv(train_files[i]) 119 | train_s[indexes] += s[indexes] 120 | train_s[indexes] /= len(train_files) 121 | 122 | test_s = pd.read_csv(test_files[0]) 123 | for i in range(1, len(test_files)): 124 | s = pd.read_csv(test_files[i]) 125 | test_s[indexes] += s[indexes] 126 | test_s[indexes] /= len(test_files) 127 | 128 | full = pd.concat((train_s, test_s), axis=0) 129 | full.reset_index(drop=True, inplace=True) 130 | 131 | for i in range(len(indexes)): 132 | nm = indexes[i] 133 | median[i] = np.float64(full[[nm]].median()) 134 | 135 | for index, row in ne.iterrows(): 136 | id = row['id'] 137 | val = np.zeros(len(indexes)) 138 | count = 0 139 | for nm in ['ntop', 'nbottom', 'nleft', 'nright', 'ntl', 'ntr', 'nbl', 'nbr']: 140 | if row[nm] != 'fake': 141 | v = full[full['image_name'] == row[nm]][indexes].as_matrix() 142 | val += v[0] 143 | count += 1 144 | 145 | # print(id, count) 146 | if count == 0: 147 | if 'train_' in id: 148 | train.loc[train['image_name'] == id, indexes] = median 149 | else: 150 | test.loc[test['image_name'] == id, indexes] = median 151 | else: 152 | val /= count 153 | if 'train_' in id: 154 | train.loc[train['image_name'] == id, indexes] = val 155 | else: 156 | test.loc[test['image_name'] == id, indexes] = val 157 | 158 | if (index + 1) % 1000 == 0: 159 | print('Completed {} from {}'.format(index + 1, len(full))) 160 | 161 | train.to_csv(FEATURES_PATH + "feature_neighbours_8_mean_" + prefix + "_train.csv", index=False) 162 | test.to_csv(FEATURES_PATH + "feature_neighbours_8_mean_" + prefix + "_test.csv", index=False) 163 | 164 | 165 | def create_group_average_features(prefix): 166 | indexes = get_indexes() 167 | train = pd.read_csv("../input/train_v2.csv")[['image_name']] 168 | test = pd.read_csv("../input/sample_submission_v2.csv")[['image_name']] 169 | ne = pd.read_csv(OUTPUT_PATH + prefix + '.csv')[['id', 'group_id']] 170 | 171 | train_files, test_files = get_list_of_CNN_features() 172 | 173 | train_s = pd.read_csv(train_files[0]) 174 | for i in range(1, len(train_files)): 175 | s = pd.read_csv(train_files[i]) 176 | train_s[indexes] += s[indexes] 177 | train_s[indexes] /= len(train_files) 178 | 179 | test_s = pd.read_csv(test_files[0]) 180 | for i in range(1, len(test_files)): 181 | s = pd.read_csv(test_files[i]) 182 | test_s[indexes] += s[indexes] 183 | test_s[indexes] /= len(test_files) 184 | 185 | full = pd.concat((train_s, test_s), axis=0) 186 | full.reset_index(drop=True, inplace=True) 187 | 188 | ne['image_name'] = ne['id'] 189 | ne = pd.merge(ne, full, on='image_name', left_index=True) 190 | ne.reset_index(drop=True, inplace=True) 191 | 192 | gr = ne.groupby(['group_id']).median().reset_index() 193 | final_table = pd.merge(ne[['image_name', 'group_id']], gr, on='group_id', left_index=True) 194 | final_table.reset_index(drop=True, inplace=True) 195 | final_table.drop('group_id', axis=1, inplace=True) 196 | 197 | train = pd.merge(train, final_table, on='image_name', left_index=True) 198 | test = pd.merge(test, final_table, on='image_name', left_index=True) 199 | 200 | train.to_csv(FEATURES_PATH + "feature_neighbours_g_mean_" + prefix + "_train.csv", index=False) 201 | test.to_csv(FEATURES_PATH + "feature_neighbours_g_mean_" + prefix + "_test.csv", index=False) 202 | 203 | 204 | if __name__ == '__main__': 205 | start_time = time.time() 206 | create_4_neighbours_features('neighbours_merged') 207 | create_8_neighbours_features('neighbours_merged') 208 | create_4_neighbours_features('neighbours_merged_cleaned') 209 | create_8_neighbours_features('neighbours_merged_cleaned') 210 | create_group_average_features('neighbours_merged_with_groups') 211 | create_group_average_features('neighbours_merged_cleaned') 212 | print("Elapsed time overall: %s seconds" % (time.time() - start_time)) 213 | 214 | 215 | ''' 216 | JPG: Fakes in 251668 positions out of 406680 (61.88%) 217 | TIFF: Fakes in 247976 positions out of 406680 (60.98%) 218 | Merge: Fakes in 181335 positions out of 406680 (44.59%) 219 | Merge cleaned: Fakes in 259874 positions out of 406680 (63.9%) 220 | ''' 221 | 222 | 223 | -------------------------------------------------------------------------------- /a42_gbm_blender.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | __author__ = 'ZFTurbo: https://kaggle.com/zfturbo' 3 | 4 | import os 5 | # os.environ["THEANO_FLAGS"] = "device=cpu" 6 | import glob 7 | import time 8 | import cv2 9 | import pandas as pd 10 | import datetime 11 | from a00_common_functions import * 12 | from a00_common_functions_scores import * 13 | from sklearn.metrics import accuracy_score 14 | from scipy.optimize import minimize 15 | from sklearn.model_selection import KFold 16 | from scipy.stats.mstats import gmean 17 | import xgboost as xgb 18 | import shutil 19 | 20 | random.seed(2017) 21 | np.random.seed(2017) 22 | 23 | INPUT_PATH = "../input/" 24 | OUTPUT_PATH = "../subm/" 25 | if not os.path.isdir(OUTPUT_PATH): 26 | os.mkdir(OUTPUT_PATH) 27 | CACHE_PATH = "../cache/" 28 | if not os.path.isdir(CACHE_PATH): 29 | os.mkdir(CACHE_PATH) 30 | FEATURES_PATH = "../features/" 31 | 32 | 33 | def get_features_for_table(t): 34 | features = list(set(t.columns.values) - {'image_name'}) 35 | return features 36 | 37 | 38 | def xgboost_random_step(random_state, iter, lbl, indexes, train_files, train_s, test_s, matrix, X_test, additional_train, additional_test, add_features): 39 | start_time = time.time() 40 | rs = random_state + iter 41 | 42 | num_folds = random.randint(4, 10) 43 | eta = random.uniform(0.06, 0.45) 44 | max_depth = random.randint(2, 5) 45 | subsample = random.uniform(0.6, 0.99) 46 | colsample_bytree = random.uniform(0.6, 0.99) 47 | # eval_metric = random.choice(['auc', 'logloss']) 48 | eval_metric = 'logloss' 49 | # eta = 0.1 50 | # max_depth = 3 51 | # subsample = 0.95 52 | # colsample_bytree = 0.95 53 | log_str = 'XGBoost iter {}. FOLDS: {} METRIC: {} ETA: {}, MAX_DEPTH: {}, SUBSAMPLE: {}, COLSAMPLE_BY_TREE: {}'.format(iter, 54 | num_folds, 55 | eval_metric, 56 | eta, 57 | max_depth, 58 | subsample, 59 | colsample_bytree) 60 | print(log_str) 61 | params = { 62 | "objective": "binary:logistic", 63 | "booster": "gbtree", 64 | "eval_metric": eval_metric, 65 | "eta": eta, 66 | "tree_method": 'exact', 67 | "max_depth": max_depth, 68 | "subsample": subsample, 69 | "colsample_bytree": colsample_bytree, 70 | "silent": 1, 71 | "seed": rs, 72 | "nthread": 6, 73 | # 'gpu_id': 0, 74 | # 'updater': 'grow_gpu_hist', 75 | } 76 | num_boost_round = 1000 77 | early_stopping_rounds = 40 78 | 79 | validation_arr = np.zeros(lbl.shape) 80 | validation_arr[:, :] = -1 81 | 82 | test_preds = np.zeros((num_folds, len(test_s[0]), len(indexes))) 83 | # model_list = [[]]*len(indexes) 84 | kf = KFold(n_splits=num_folds, shuffle=True, random_state=rs) 85 | num_fold = 0 86 | for train_index, test_index in kf.split(list(range(len(train_s[0])))): 87 | num_fold += 1 88 | # print('Start fold {} from {}'.format(num_fold, num_folds)) 89 | feat = get_features_for_table(train_s[0]) 90 | X_train = train_s[0][feat].as_matrix()[train_index] 91 | X_valid = train_s[0][feat].as_matrix()[test_index] 92 | for i in range(1, len(train_files)): 93 | feat = get_features_for_table(train_s[i]) 94 | X_train = np.concatenate((X_train, train_s[i][feat].as_matrix()[train_index]), axis=1) 95 | X_valid = np.concatenate((X_valid, train_s[i][feat].as_matrix()[test_index]), axis=1) 96 | 97 | # Additional 98 | if additional_train is not None: 99 | X_train = np.concatenate((X_train, additional_train[add_features].as_matrix()[train_index]), axis=1) 100 | X_valid = np.concatenate((X_valid, additional_train[add_features].as_matrix()[test_index]), axis=1) 101 | 102 | y_train = lbl[train_index] 103 | y_valid = lbl[test_index] 104 | # print('Len train: ', X_train.shape) 105 | # print('Len valid: ', X_valid.shape) 106 | 107 | random_skip = random.sample(list(range(len(train_files))), 1) 108 | for i in range(len(indexes)): 109 | dtrain = xgb.DMatrix(X_train, y_train[:, i]) 110 | dvalid = xgb.DMatrix(X_valid, y_valid[:, i]) 111 | 112 | watchlist = [(dtrain, 'train'), (dvalid, 'eval')] 113 | gbm = xgb.train(params, dtrain, num_boost_round, evals=watchlist, 114 | early_stopping_rounds=early_stopping_rounds, verbose_eval=False) 115 | 116 | # print("Validating...") 117 | preds = gbm.predict(dvalid, ntree_limit=gbm.best_iteration + 1) 118 | test_preds[num_fold - 1, :, i] = gbm.predict(matrix, ntree_limit=gbm.best_iteration + 1) 119 | validation_arr[test_index, i] = preds 120 | # model_list[i].append((gbm, gbm.best_iteration)) 121 | print("Time XGBoost: %s sec" % (round(time.time() - start_time, 0))) 122 | return validation_arr, test_preds, log_str, params 123 | 124 | 125 | def get_feature_files(): 126 | valid_files = [] 127 | test_files = [] 128 | 129 | print('List of feature files: ') 130 | files = glob.glob(FEATURES_PATH + "valid_*.csv") 131 | for f in files: 132 | valid_files.append(f) 133 | test_files.append(f.replace('valid_', 'test_')) 134 | print(valid_files[-1]) 135 | print(test_files[-1]) 136 | 137 | files = glob.glob(FEATURES_PATH + "feature_*_train.csv") 138 | for f in files: 139 | valid_files.append(f) 140 | test_files.append(f.replace('_train', '_test')) 141 | print(valid_files[-1]) 142 | print(test_files[-1]) 143 | 144 | return valid_files, test_files 145 | 146 | 147 | def get_additional_features(train_s): 148 | # Additional features 149 | sample_subm = pd.read_csv(INPUT_PATH + "sample_submission_v2.csv") 150 | additional_data = pd.read_csv(FEATURES_PATH + "neighbours_merged_cleaned_fix_train.csv") 151 | additional_train = pd.merge(train_s[0][['image_name']], additional_data, on='image_name', left_index=True) 152 | additional_data = pd.read_csv(FEATURES_PATH + "neighbours_merged_with_groups_fix_train.csv") 153 | additional_train = pd.merge(additional_train, additional_data, on='image_name', left_index=True) 154 | additional_train.reset_index(drop=True, inplace=True) 155 | additional_data = pd.read_csv(FEATURES_PATH + "neighbours_merged_cleaned_fix_test.csv") 156 | additional_test = pd.merge(sample_subm[['image_name']], additional_data, on='image_name', left_index=True) 157 | additional_data = pd.read_csv(FEATURES_PATH + "neighbours_merged_with_groups_fix_test.csv") 158 | additional_test = pd.merge(additional_test, additional_data, on='image_name', left_index=True) 159 | additional_test.reset_index(drop=True, inplace=True) 160 | 161 | return additional_train, additional_test 162 | 163 | 164 | def multiple_gbm_runs(train_files, test_files, required_iterations): 165 | 166 | restore_from_cache_file = '' 167 | random_state = 2006 168 | random.seed(random_state) 169 | 170 | log_list = [] 171 | indexes = get_indexes() 172 | lbl = get_train_label_matrix() 173 | train_s = [0]*len(train_files) 174 | test_s = [0]*len(test_files) 175 | 176 | for i in range(len(train_files)): 177 | train_s[i] = pd.read_csv(train_files[i]) 178 | feat = get_features_for_table(train_s[i]) 179 | if (np.isnan(np.sum(train_s[i][feat].as_matrix())) == True): 180 | print('Some problem with ', train_files[i], 'check it!') 181 | exit() 182 | for i in range(len(test_files)): 183 | test_s[i] = pd.read_csv(test_files[i]) 184 | feat = get_features_for_table(test_s[i]) 185 | if (np.isnan(np.sum(test_s[i][feat].as_matrix())) == True): 186 | print('Some problem with ', test_files[i], 'check it!') 187 | exit() 188 | 189 | feat = get_features_for_table(test_s[0]) 190 | X_test = test_s[0][feat].as_matrix() 191 | for i in range(1, len(test_files)): 192 | feat = get_features_for_table(test_s[i]) 193 | X_test = np.concatenate((X_test, test_s[i][feat].as_matrix()), axis=1) 194 | print('Test shape:', X_test.shape) 195 | 196 | additional_train, additional_test = get_additional_features(train_s) 197 | 198 | add_features = list(additional_test.columns.values) 199 | add_features.remove('image_name') 200 | at = additional_test[add_features].as_matrix() 201 | X_test = np.concatenate((X_test, at), axis=1) 202 | 203 | matrix_xgboost = xgb.DMatrix(X_test) 204 | 205 | if not os.path.isfile(restore_from_cache_file): 206 | validation_full_arr = [] 207 | test_preds_full_arr = [] 208 | scores_full_arr = [] 209 | params_full_arr = [] 210 | thr_full_arr = [] 211 | for iter in range(required_iterations): 212 | validation_arr, test_preds, log_str, params = xgboost_random_step(random_state, iter, lbl, indexes, train_files, train_s, test_s, matrix_xgboost, X_test, additional_train, additional_test, add_features) 213 | validation_full_arr.append(validation_arr.copy()) 214 | test_preds_full_arr.append(test_preds.mean(axis=0)) 215 | log_list.append(log_str) 216 | 217 | best_score, searcher = get_optimal_score_very_fast(indexes, lbl, validation_arr, 7, 0.001, 1) 218 | log_str = 'Best score {} for THR array: {}'.format(best_score, list(searcher)) 219 | print(log_str) 220 | log_list.append(log_str) 221 | scores_full_arr.append(best_score) 222 | thr_full_arr.append(searcher) 223 | params_full_arr.append(params) 224 | # exit() 225 | if iter > 0 and iter % 10 == 0: 226 | avg_score = np.array(scores_full_arr).mean() 227 | save_in_file((validation_full_arr, test_preds_full_arr, scores_full_arr, params_full_arr, thr_full_arr), CACHE_PATH + "gbm_cache_iter_{}_from_{}_score_{}.pklz".format(iter, required_iterations, avg_score)) 228 | 229 | avg_score = np.array(scores_full_arr).mean() 230 | out_cache_file = CACHE_PATH + "gbm_cache_iter_{}_score_{}.pklz".format(required_iterations, avg_score) 231 | print('Write final cache file: {}'.format(out_cache_file)) 232 | save_in_file((validation_full_arr, test_preds_full_arr, scores_full_arr, params_full_arr, thr_full_arr), out_cache_file) 233 | else: 234 | print('Restore from cache file: {}'.format(restore_from_cache_file)) 235 | validation_full_arr, test_preds_full_arr, scores_full_arr, params_full_arr, thr_full_arr = load_from_file(restore_from_cache_file) 236 | 237 | scores_full_arr = np.array(scores_full_arr) 238 | validation_full_arr = np.array(validation_full_arr) 239 | test_preds_full_arr = np.array(test_preds_full_arr) 240 | condition = scores_full_arr > 0.931 241 | print('Left {} out of {} runs'.format(len(scores_full_arr[condition]), len(scores_full_arr))) 242 | validation_full_arr = validation_full_arr[condition] 243 | test_preds_full_arr = test_preds_full_arr[condition] 244 | 245 | # Experimental version 246 | if 0: 247 | best_score, searcher = get_optimal_score_very_fast_for_full_array(indexes, lbl, validation_full_arr, 7, 0.00001, 3) 248 | log_str = 'Best score {} for THR array: {}'.format(best_score, list(searcher)) 249 | print(log_str) 250 | log_list.append(log_str) 251 | 252 | # Create submission 253 | tbl = pd.read_csv(INPUT_PATH + "sample_submission_v2.csv") 254 | ids = tbl['image_name'].values 255 | files = [] 256 | for id in ids: 257 | files.append(INPUT_PATH + "test-jpg/" + id + '.jpg') 258 | files = np.array(files) 259 | 260 | preds = test_preds_full_arr 261 | 262 | sub_file = OUTPUT_PATH + "merger_xgboost_v2_{}.csv".format(best_score) 263 | out = open(sub_file, "w") 264 | out.write("image_name,tags\n") 265 | for i in range(len(files)): 266 | out.write(ids[i] + ',') 267 | for j in range(len(indexes)): 268 | cond = preds[:, i, j] > searcher[j] 269 | cond = 2 * np.sum(cond, axis=0) 270 | if cond >= preds.shape[0]: 271 | out.write(indexes[j] + ' ') 272 | out.write("\n") 273 | out.close() 274 | 275 | # Normal version 276 | else: 277 | if 1: 278 | validation_arr = np.mean(np.array(validation_full_arr), axis=0) 279 | test_preds = np.mean(np.array(test_preds_full_arr), axis=0) 280 | else: 281 | validation_arr = gmean(validation_full_arr, axis=0) 282 | test_preds = gmean(test_preds_full_arr, axis=0) 283 | 284 | if np.count_nonzero(validation_arr < 0) > 0: 285 | print('Some error here..') 286 | exit() 287 | 288 | # Check validation 289 | best_score, searcher = get_optimal_score_very_fast(indexes, lbl, validation_arr, 7, 0.00001, 3) 290 | # best_score, searcher = get_optimal_score_slow(indexes, lbl, validation_arr) 291 | log_str = 'Best score {} for THR array: {}'.format(best_score, list(searcher)) 292 | print(log_str) 293 | log_list.append(log_str) 294 | 295 | preds = test_preds 296 | 297 | # Create submission 298 | tbl = pd.read_csv(INPUT_PATH + "sample_submission_v2.csv") 299 | ids = tbl['image_name'].values 300 | files = [] 301 | for id in ids: 302 | files.append(INPUT_PATH + "test-jpg/" + id + '.jpg') 303 | files = np.array(files) 304 | 305 | sub_file = OUTPUT_PATH + "merger_xgboost_{}.csv".format(best_score) 306 | out = open(sub_file, "w") 307 | out.write("image_name,tags\n") 308 | for i in range(len(files)): 309 | out.write(ids[i] + ',') 310 | for j in range(len(indexes)): 311 | if preds[i][j] > searcher[j]: 312 | out.write(indexes[j] + ' ') 313 | out.write("\n") 314 | out.close() 315 | 316 | # Save raw test for analysis 317 | sub_file_raw = OUTPUT_PATH + "merger_xgboost_{}_raw_test.csv".format(best_score) 318 | out = open(sub_file_raw, "w") 319 | out.write("image_name") 320 | for i in indexes: 321 | out.write("," + str(i)) 322 | out.write("\n") 323 | 324 | for i in range(len(files)): 325 | out.write(ids[i]) 326 | for j in range(len(preds[i])): 327 | out.write(',' + str(preds[i][j])) 328 | out.write("\n") 329 | out.close() 330 | 331 | # Save raw validation for further analysis 332 | sub_file_raw = OUTPUT_PATH + "merger_xgboost_{}_raw_valid.csv".format(best_score) 333 | out = open(sub_file_raw, "w") 334 | out.write("image_name") 335 | for i in indexes: 336 | out.write("," + str(i)) 337 | out.write("\n") 338 | 339 | ids = list(train_s[0]['image_name'].values) 340 | for i in range(validation_arr.shape[0]): 341 | out.write(ids[i]) 342 | for j in range(len(validation_arr[i])): 343 | out.write(',' + str(validation_arr[i][j])) 344 | out.write("\n") 345 | out.close() 346 | 347 | # Copy code 348 | shutil.copy2(__file__, sub_file + ".py") 349 | 350 | # Save log 351 | out = open(sub_file + ".log", "w") 352 | for l in log_list: 353 | out.write(l + '\n') 354 | out.close() 355 | 356 | 357 | if __name__ == '__main__': 358 | start_time = time.time() 359 | required_iterations = 150 360 | valid_files, test_files = get_feature_files() 361 | multiple_gbm_runs(valid_files, test_files, required_iterations) 362 | print("Elapsed time overall: %s seconds" % (time.time() - start_time)) 363 | -------------------------------------------------------------------------------- /a42_keras_blender.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | __author__ = 'ZFTurbo: https://kaggle.com/zfturbo' 3 | 4 | import os 5 | import platform 6 | # os.environ["THEANO_FLAGS"] = "device=cpu" 7 | gpu_use = 0 8 | if platform.processor() == 'Intel64 Family 6 Model 63 Stepping 2, GenuineIntel': 9 | os.environ["THEANO_FLAGS"] = "device=gpu{},lib.cnmem=0.81,,base_compiledir='C:\\\\Users\\\\user\\\\AppData\\\\Local\\\\Theano{}'".format(gpu_use, gpu_use) 10 | 11 | import glob 12 | import time 13 | import cv2 14 | import pandas as pd 15 | import datetime 16 | import shutil 17 | from a00_common_functions import * 18 | from a00_common_functions_scores import * 19 | from sklearn.metrics import accuracy_score 20 | from scipy.stats.mstats import gmean 21 | from sklearn.model_selection import KFold 22 | from a42_gbm_blender import get_optimal_score_very_fast, get_optimal_score_very_fast_for_full_array 23 | 24 | INPUT_PATH = "../input/" 25 | OUTPUT_PATH = "../subm/" 26 | if not os.path.isdir(OUTPUT_PATH): 27 | os.mkdir(OUTPUT_PATH) 28 | MODELS_PATH = "../models/" 29 | CACHE_PATH = "../cache/" 30 | if not os.path.isdir(CACHE_PATH): 31 | os.mkdir(CACHE_PATH) 32 | FEATURES_PATH = "../features/" 33 | HISTORY_FOLDER_PATH = "../models/history2/" 34 | 35 | 36 | def Blender_CNN(num_of_res, num_of_inp, params): 37 | from keras.models import Sequential 38 | from keras.layers.core import Dense, Dropout, Flatten, Activation 39 | from keras.layers.normalization import BatchNormalization 40 | from keras.layers.advanced_activations import PReLU 41 | 42 | level1_size = 300 43 | if 'level1_size' in params: 44 | level1_size = params['level1_size'] 45 | level2_size = 300 46 | if 'level2_size' in params: 47 | level2_size = params['level2_size'] 48 | level3_size = 250 49 | if 'level3_size' in params: 50 | level3_size = params['level3_size'] 51 | 52 | dropout_val_1 = 0.1 53 | if 'dropout_val_1' in params: 54 | dropout_val_1 = params['dropout_val_1'] 55 | dropout_val_2 = 0.1 56 | if 'dropout_val_2' in params: 57 | dropout_val_2 = params['dropout_val_2'] 58 | dropout_val_3 = 0.1 59 | if 'dropout_val_3' in params: 60 | dropout_val_3 = params['dropout_val_3'] 61 | 62 | activation_1 = 'prelu' 63 | if 'activation_1' in params: 64 | activation_1 = params['activation_1'] 65 | activation_2 = 'prelu' 66 | if 'activation_2' in params: 67 | activation_2 = params['activation_2'] 68 | activation_3 = 'prelu' 69 | if 'activation_3' in params: 70 | activation_3 = params['activation_3'] 71 | 72 | use_3rd_level = 1 73 | if 'use_3rd_level' in params: 74 | use_3rd_level = params['use_3rd_level'] 75 | 76 | model = Sequential() 77 | model.add(Dense(level1_size, input_shape=(num_of_inp,))) 78 | if activation_1 == 'prelu': 79 | model.add(PReLU()) 80 | elif activation_1 == 'relu': 81 | model.add(Activation('relu')) 82 | else: 83 | model.add(Activation('elu')) 84 | model.add(Dropout(dropout_val_1)) 85 | model.add(Dense(level2_size)) 86 | if activation_2 == 'prelu': 87 | model.add(PReLU()) 88 | elif activation_2 == 'relu': 89 | model.add(Activation('relu')) 90 | else: 91 | model.add(Activation('elu')) 92 | model.add(Dropout(dropout_val_2)) 93 | if use_3rd_level == 1: 94 | model.add(Dense(level3_size)) 95 | if activation_3 == 'prelu': 96 | model.add(PReLU()) 97 | elif activation_3 == 'relu': 98 | model.add(Activation('relu')) 99 | else: 100 | model.add(Activation('elu')) 101 | model.add(Dropout(dropout_val_3)) 102 | model.add(Dense(num_of_res, activation='sigmoid')) 103 | return model 104 | 105 | 106 | def batch_generator_train(X, y, batch_size): 107 | rng = range(X.shape[0]) 108 | 109 | while True: 110 | index = random.sample(rng, batch_size) 111 | input = X[index, :] 112 | output = y[index, :] 113 | 114 | yield input, output 115 | 116 | 117 | def random_keras_step(random_state, iter, lbl, indexes, train_files, train_s, test_s, X_test_transform): 118 | from keras.callbacks import EarlyStopping, ModelCheckpoint 119 | from keras.optimizers import Adam, SGD, Adadelta 120 | 121 | start_time = time.time() 122 | rs = random_state + iter 123 | 124 | num_folds = random.randint(4, 10) 125 | batch_size = random.randint(200, 1000) 126 | 127 | patience = random.randint(50, 150) 128 | learning_rate = random.uniform(0.00001, 0.001) 129 | # roptim = random.choice(['Adam', 'SGD', 'Adadelta']) 130 | roptim = 'Adam' 131 | if 0: 132 | roptim = 'Adam' 133 | learning_rate = 0.001 134 | batch_size = 210 135 | 136 | # metric = random.choice(['binary_crossentropy', 'f2beta_loss']) 137 | metric = random.choice(['binary_crossentropy']) 138 | cnn_param = dict() 139 | cnn_param['dropout_val_1'] = random.uniform(0.05, 0.5) 140 | cnn_param['dropout_val_2'] = random.uniform(0.1, 0.5) 141 | cnn_param['dropout_val_3'] = random.uniform(0.1, 0.5) 142 | cnn_param['level1_size'] = random.randint(400, 700) 143 | cnn_param['level2_size'] = random.randint(350, 600) 144 | cnn_param['level3_size'] = random.randint(200, 500) 145 | cnn_param['activation_1'] = random.choice(['prelu', 'relu', 'elu']) 146 | cnn_param['activation_2'] = random.choice(['prelu', 'relu', 'elu']) 147 | cnn_param['activation_3'] = random.choice(['prelu', 'relu', 'elu']) 148 | # cnn_param['use_3rd_level'] = random.choice([0, 1]) 149 | # cnn_param['use_3rd_level'] = random.choice([0, 1]) 150 | # cnn_param['activation'] = 'relu' 151 | 152 | if 0: 153 | cnn_param['level1_size'] = 1000 154 | cnn_param['level2_size'] = 1000 155 | cnn_param['level3_size'] = 1000 156 | cnn_param['use_3rd_level'] = 0 157 | cnn_param['dropout_val_1'] = 0 158 | cnn_param['dropout_val_2'] = 0.5 159 | cnn_param['dropout_val_3'] = 0.5 160 | 161 | log_str = 'Keras iter {}. FOLDS: {} LR: {}, PATIENCE: {}, OPTIM: {}, METRIC: {}, BATCH: {}'.format( 162 | iter, 163 | num_folds, 164 | learning_rate, 165 | patience, 166 | roptim, 167 | metric, 168 | batch_size) 169 | print(log_str) 170 | print('CNN params: {}'.format(cnn_param)) 171 | 172 | validation_arr = np.zeros(lbl.shape) 173 | validation_arr[:, :] = -1 174 | 175 | test_preds = np.zeros((num_folds, len(test_s[0]), len(indexes))) 176 | kf = KFold(n_splits=num_folds, shuffle=True, random_state=rs) 177 | num_fold = 0 178 | for train_index, test_index in kf.split(list(range(len(train_s[0])))): 179 | num_fold += 1 180 | # print('Start fold {} from {}'.format(num_fold, num_folds)) 181 | feat = get_features_for_table(train_s[0]) 182 | X_train = train_s[0][feat].as_matrix()[train_index] 183 | X_valid = train_s[0][feat].as_matrix()[test_index] 184 | for i in range(1, len(train_files)): 185 | feat = get_features_for_table(train_s[i]) 186 | X_train = np.concatenate((X_train, train_s[i][feat].as_matrix()[train_index]), axis=1) 187 | X_valid = np.concatenate((X_valid, train_s[i][feat].as_matrix()[test_index]), axis=1) 188 | y_train = lbl[train_index] 189 | y_valid = lbl[test_index] 190 | # print('Shape train: ', X_train.shape) 191 | # print('Shape valid: ', X_valid.shape) 192 | 193 | final_model_path = MODELS_PATH + '{}_fold_{}.h5'.format("blender", num_fold) 194 | cache_model_path = MODELS_PATH + '{}_temp_fold_{}.h5'.format("blender", num_fold) 195 | 196 | model = Blender_CNN(len(indexes), X_train.shape[1], cnn_param) 197 | if roptim == 'Adam': 198 | optim = Adam(lr=learning_rate) 199 | elif roptim == 'SGD': 200 | optim = SGD(lr=learning_rate, decay=1e-6, momentum=0.9, nesterov=True) 201 | elif roptim == 'Adadelta': 202 | optim = Adadelta() 203 | else: 204 | print('Unknown optimizer {}!'.format(roptim)) 205 | exit() 206 | 207 | # optim = SGD(lr=learning_rate, decay=1e-6, momentum=0.9, nesterov=True) 208 | # model.compile(optimizer=optim, loss=f2beta_loss, metrics=['binary_crossentropy', 'accuracy']) 209 | # model.compile(optimizer=optim, loss=f2beta_loss, metrics=['binary_crossentropy']) 210 | if metric == 'binary_crossentropy': 211 | model.compile(optimizer=optim, loss='binary_crossentropy', metrics=[f2beta_loss]) 212 | else: 213 | model.compile(optimizer=optim, loss=f2beta_loss, metrics=['binary_crossentropy']) 214 | 215 | callbacks = [ 216 | EarlyStopping(monitor='val_loss', patience=patience, verbose=0), 217 | ModelCheckpoint(cache_model_path, monitor='val_loss', save_best_only=True, verbose=0), 218 | ] 219 | 220 | samples_per_epoch = batch_size * (1 + X_train.shape[0] // batch_size) 221 | nb_val_samples = batch_size * (1 + X_valid.shape[0] // batch_size) 222 | history = model.fit_generator(generator=batch_generator_train(X_train, y_train, batch_size), 223 | nb_epoch=10000, 224 | samples_per_epoch=samples_per_epoch, 225 | validation_data=batch_generator_train(X_valid, y_valid, batch_size), 226 | nb_val_samples=nb_val_samples, 227 | verbose=0, max_q_size=100, 228 | callbacks=callbacks) 229 | 230 | min_loss = min(history.history['val_loss']) 231 | print('Loss fold {}: {}'.format(num_fold, min_loss)) 232 | model.load_weights(cache_model_path) 233 | model.save(final_model_path) 234 | now = datetime.datetime.now() 235 | filename = HISTORY_FOLDER_PATH + 'history_{}_{}_{:.4f}_lr_{}_{}.csv'.format("blender", num_fold, min_loss, 236 | learning_rate, 237 | now.strftime("%Y-%m-%d-%H-%M")) 238 | # pd.DataFrame(history.history).to_csv(filename, index=False) 239 | 240 | preds = model.predict(X_valid) 241 | validation_arr[test_index] = preds 242 | test_preds[num_fold - 1] = model.predict(X_test_transform) 243 | 244 | print("Time Keras: %s sec" % (round(time.time() - start_time, 0))) 245 | return validation_arr, test_preds, log_str, cnn_param 246 | 247 | 248 | def get_features_for_table(t): 249 | features = list(set(t.columns.values) - {'image_name'}) 250 | return features 251 | 252 | 253 | def run_multiple_keras_blenders(train_files, test_files, required_iterations): 254 | 255 | restore_from_cache_file = [] 256 | random_state = 1002 257 | indexes = get_indexes() 258 | lbl = get_train_label_matrix() 259 | train_s = [0]*len(train_files) 260 | test_s = [0]*len(test_files) 261 | log_list = [] 262 | 263 | for i in range(len(train_files)): 264 | train_s[i] = pd.read_csv(train_files[i]) 265 | for i in range(len(test_files)): 266 | test_s[i] = pd.read_csv(test_files[i]) 267 | 268 | # Prepare test array 269 | feat = get_features_for_table(test_s[0]) 270 | X_test_transform = test_s[0][feat].as_matrix() 271 | for i in range(1, len(test_files)): 272 | feat = get_features_for_table(test_s[i]) 273 | X_test_transform = np.concatenate((X_test_transform, test_s[i][feat].as_matrix()), axis=1) 274 | print('Shape:', X_test_transform.shape) 275 | 276 | if len(restore_from_cache_file) == 0: 277 | validation_full_arr = [] 278 | test_preds_full_arr = [] 279 | scores_full_arr = [] 280 | params_full_arr = [] 281 | thr_full_arr = [] 282 | for iter in range(required_iterations): 283 | validation_arr, test_preds, log_str, cnn_param = random_keras_step(random_state, iter, lbl, indexes, train_files, train_s, test_s, X_test_transform) 284 | validation_full_arr.append(validation_arr.copy()) 285 | test_preds_full_arr.append(test_preds.mean(axis=0)) 286 | params_full_arr.append(cnn_param) 287 | log_list.append(log_str) 288 | 289 | best_score, searcher = get_optimal_score_very_fast(indexes, lbl, validation_arr, 7, 0.001, 1) 290 | log_str = 'Best score {} for THR array: {}'.format(best_score, list(searcher)) 291 | print(log_str) 292 | log_list.append(log_str) 293 | scores_full_arr.append(best_score) 294 | thr_full_arr.append(searcher) 295 | if iter > 0 and iter % 10 == 0: 296 | avg_score = np.array(scores_full_arr).mean() 297 | save_in_file((validation_full_arr, test_preds_full_arr, scores_full_arr, params_full_arr, thr_full_arr), 298 | CACHE_PATH + "keras_cache_iter_{}_from_{}_score_{}.pklz".format(iter, required_iterations, avg_score)) 299 | 300 | avg_score = np.array(scores_full_arr).mean() 301 | out_cache_file = CACHE_PATH + "keras_cache_iter_{}_score_{}.pklz".format(required_iterations, avg_score) 302 | print('Write final cache file: {}'.format(out_cache_file)) 303 | save_in_file((validation_full_arr, test_preds_full_arr, scores_full_arr, params_full_arr, thr_full_arr), out_cache_file) 304 | else: 305 | only_1_best_subm = 0 306 | print('Restore from cache file: {}'.format(restore_from_cache_file)) 307 | validation_full_arr, test_preds_full_arr, scores_full_arr, params_full_arr, thr_full_arr = load_from_file(restore_from_cache_file[0]) 308 | 309 | scores_full_arr = np.array(scores_full_arr) 310 | validation_full_arr = np.array(validation_full_arr) 311 | test_preds_full_arr = np.array(test_preds_full_arr) 312 | params_full_arr = np.array(params_full_arr) 313 | thr_full_arr = np.array(thr_full_arr) 314 | 315 | # Experiment with top 1 score: 316 | if only_1_best_subm: 317 | mx = scores_full_arr.max() 318 | cond = scores_full_arr >= mx 319 | scores_full_arr = scores_full_arr[cond] 320 | validation_full_arr = validation_full_arr[cond] 321 | test_preds_full_arr = test_preds_full_arr[cond] 322 | params_full_arr = params_full_arr[cond] 323 | thr_full_arr = thr_full_arr[cond] 324 | 325 | for i in range(1, len(restore_from_cache_file)): 326 | validation_full_arr1, test_preds_full_arr1, scores_full_arr1, params_full_arr1, thr_full_arr1 = load_from_file(restore_from_cache_file[i]) 327 | scores_full_arr1 = np.array(scores_full_arr1) 328 | validation_full_arr1 = np.array(validation_full_arr1) 329 | test_preds_full_arr1 = np.array(test_preds_full_arr1) 330 | params_full_arr1 = np.array(params_full_arr1) 331 | thr_full_arr1 = np.array(thr_full_arr1) 332 | 333 | # Experiment with top 1 score: 334 | if only_1_best_subm: 335 | mx = scores_full_arr1.max() 336 | cond = scores_full_arr1 >= mx 337 | scores_full_arr1 = scores_full_arr1[cond] 338 | validation_full_arr1 = validation_full_arr1[cond] 339 | test_preds_full_arr1 = test_preds_full_arr1[cond] 340 | params_full_arr1 = params_full_arr1[cond] 341 | thr_full_arr1 = thr_full_arr1[cond] 342 | 343 | validation_full_arr = np.concatenate((validation_full_arr, validation_full_arr1), axis=0) 344 | scores_full_arr = np.concatenate((scores_full_arr, scores_full_arr1), axis=0) 345 | test_preds_full_arr = np.concatenate((test_preds_full_arr, test_preds_full_arr1), axis=0) 346 | params_full_arr = np.concatenate((params_full_arr, params_full_arr1), axis=0) 347 | thr_full_arr = np.concatenate((thr_full_arr, thr_full_arr1), axis=0) 348 | 349 | print(validation_full_arr.shape) 350 | print(scores_full_arr.shape) 351 | print(test_preds_full_arr.shape) 352 | print(params_full_arr.shape) 353 | print(thr_full_arr.shape) 354 | 355 | 356 | scores_full_arr = np.array(scores_full_arr) 357 | validation_full_arr = np.array(validation_full_arr) 358 | test_preds_full_arr = np.array(test_preds_full_arr) 359 | condition = scores_full_arr > 0.931 360 | print('Left {} out of {} runs'.format(len(scores_full_arr[condition]), len(scores_full_arr))) 361 | validation_full_arr = validation_full_arr[condition] 362 | test_preds_full_arr = test_preds_full_arr[condition] 363 | 364 | if 1: 365 | validation_arr = np.mean(np.array(validation_full_arr), axis=0) 366 | test_preds = np.mean(np.array(test_preds_full_arr), axis=0) 367 | else: 368 | validation_arr = gmean(validation_full_arr, axis=0) 369 | test_preds = gmean(test_preds_full_arr, axis=0) 370 | 371 | if np.count_nonzero(validation_arr < 0) > 0: 372 | print('Some error here..') 373 | exit() 374 | 375 | # Check validation 376 | best_score, searcher = get_optimal_score_very_fast(indexes, lbl, validation_arr, 7, 0.00001, 3) 377 | # best_score, searcher = get_optimal_score_slow(indexes, lbl, validation_arr) 378 | log_str = 'Best score {} for THR array: {}'.format(best_score, list(searcher)) 379 | print(log_str) 380 | log_list.append(log_str) 381 | 382 | preds = test_preds 383 | 384 | # Create submission 385 | tbl = pd.read_csv(INPUT_PATH + "sample_submission_v2.csv") 386 | ids = tbl['image_name'].values 387 | files = [] 388 | for id in ids: 389 | files.append(INPUT_PATH + "test-jpg/" + id + '.jpg') 390 | files = np.array(files) 391 | 392 | sub_file = OUTPUT_PATH + "merger_keras_{}.csv".format(best_score) 393 | out = open(sub_file, "w") 394 | out.write("image_name,tags\n") 395 | for i in range(len(files)): 396 | out.write(ids[i] + ',') 397 | for j in range(len(indexes)): 398 | if preds[i][j] > searcher[j]: 399 | out.write(indexes[j] + ' ') 400 | out.write("\n") 401 | out.close() 402 | 403 | # Save raw test for analysis 404 | sub_file_raw = OUTPUT_PATH + "merger_keras_{}_raw_test.csv".format(best_score) 405 | out = open(sub_file_raw, "w") 406 | out.write("image_name") 407 | for i in indexes: 408 | out.write("," + str(i)) 409 | out.write("\n") 410 | 411 | for i in range(len(files)): 412 | out.write(ids[i]) 413 | for j in range(len(preds[i])): 414 | out.write(',' + str(preds[i][j])) 415 | out.write("\n") 416 | out.close() 417 | 418 | # Save raw validation for further analysis 419 | sub_file_raw = OUTPUT_PATH + "merger_keras_{}_raw_valid.csv".format(best_score) 420 | out = open(sub_file_raw, "w") 421 | out.write("image_name") 422 | for i in indexes: 423 | out.write("," + str(i)) 424 | out.write("\n") 425 | 426 | ids = list(train_s[0]['image_name'].values) 427 | for i in range(validation_arr.shape[0]): 428 | out.write(ids[i]) 429 | for j in range(len(validation_arr[i])): 430 | out.write(',' + str(validation_arr[i][j])) 431 | out.write("\n") 432 | out.close() 433 | 434 | 435 | def get_feature_files(): 436 | valid_files = [] 437 | test_files = [] 438 | 439 | print('List of feature files: ') 440 | files = glob.glob(FEATURES_PATH + "valid_*.csv") 441 | for f in files: 442 | valid_files.append(f) 443 | test_files.append(f.replace('valid_', 'test_')) 444 | print(valid_files[-1]) 445 | print(test_files[-1]) 446 | 447 | files = glob.glob(FEATURES_PATH + "feature_*_train.csv") 448 | for f in files: 449 | valid_files.append(f) 450 | test_files.append(f.replace('_train', '_test')) 451 | print(valid_files[-1]) 452 | print(test_files[-1]) 453 | 454 | return valid_files, test_files 455 | 456 | 457 | if __name__ == '__main__': 458 | start_time = time.time() 459 | required_iterations = 100 460 | valid_files, test_files = get_feature_files() 461 | run_multiple_keras_blenders(valid_files, test_files, required_iterations) 462 | print("Elapsed time overall: %s seconds" % (time.time() - start_time)) 463 | -------------------------------------------------------------------------------- /a50_ensemble_from_cache_v1.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | __author__ = 'ZFTurbo: https://kaggle.com/zfturbo' 3 | 4 | import shutil 5 | from a00_common_functions import * 6 | from a00_common_functions_scores import * 7 | from scipy.stats.mstats import gmean 8 | from a42_gbm_blender import get_optimal_score_very_fast, get_optimal_score_very_fast_for_full_array 9 | 10 | INPUT_PATH = "../input/" 11 | OUTPUT_PATH = "../subm/" 12 | CACHE_PATH = "../cache/" 13 | 14 | 15 | def ensemble_from_cache_v1(restore_from_cache_file): 16 | 17 | indexes = get_indexes() 18 | lbl = get_train_label_matrix() 19 | train = pd.read_csv(INPUT_PATH + 'train_v2.csv') 20 | log_list = [] 21 | 22 | only_1_best_subm = 0 23 | print('Restore from cache file: {}'.format(restore_from_cache_file)) 24 | validation_full_arr, test_preds_full_arr, scores_full_arr, params_full_arr, thr_full_arr = load_from_file(restore_from_cache_file[0]) 25 | 26 | scores_full_arr = np.array(scores_full_arr) 27 | validation_full_arr = np.array(validation_full_arr) 28 | test_preds_full_arr = np.array(test_preds_full_arr) 29 | params_full_arr = np.array(params_full_arr) 30 | thr_full_arr = np.array(thr_full_arr) 31 | 32 | # Experiment with top 1 score: 33 | if only_1_best_subm: 34 | scores_full_arr = scores_full_arr[0:only_1_best_subm] 35 | validation_full_arr = validation_full_arr[0:only_1_best_subm] 36 | test_preds_full_arr = test_preds_full_arr[0:only_1_best_subm] 37 | params_full_arr = params_full_arr[0:only_1_best_subm] 38 | thr_full_arr = thr_full_arr[0:only_1_best_subm] 39 | 40 | for i in range(1, len(restore_from_cache_file)): 41 | validation_full_arr1, test_preds_full_arr1, scores_full_arr1, params_full_arr1, thr_full_arr1 = load_from_file(restore_from_cache_file[i]) 42 | scores_full_arr1 = np.array(scores_full_arr1) 43 | validation_full_arr1 = np.array(validation_full_arr1) 44 | test_preds_full_arr1 = np.array(test_preds_full_arr1) 45 | params_full_arr1 = np.array(params_full_arr1) 46 | thr_full_arr1 = np.array(thr_full_arr1) 47 | 48 | # Experiment with top 1 score: 49 | if only_1_best_subm: 50 | scores_full_arr1 = scores_full_arr1[0:only_1_best_subm] 51 | validation_full_arr1 = validation_full_arr1[0:only_1_best_subm] 52 | test_preds_full_arr1 = test_preds_full_arr1[0:only_1_best_subm] 53 | params_full_arr1 = params_full_arr1[0:only_1_best_subm] 54 | thr_full_arr1 = thr_full_arr1[0:only_1_best_subm] 55 | 56 | validation_full_arr = np.concatenate((validation_full_arr, validation_full_arr1), axis=0) 57 | scores_full_arr = np.concatenate((scores_full_arr, scores_full_arr1), axis=0) 58 | test_preds_full_arr = np.concatenate((test_preds_full_arr, test_preds_full_arr1), axis=0) 59 | params_full_arr = np.concatenate((params_full_arr, params_full_arr1), axis=0) 60 | thr_full_arr = np.concatenate((thr_full_arr, thr_full_arr1), axis=0) 61 | 62 | print(validation_full_arr.shape) 63 | print(scores_full_arr.shape) 64 | print(test_preds_full_arr.shape) 65 | print(params_full_arr.shape) 66 | print(thr_full_arr.shape) 67 | 68 | scores_full_arr = np.array(scores_full_arr) 69 | validation_full_arr = np.array(validation_full_arr) 70 | test_preds_full_arr = np.array(test_preds_full_arr) 71 | condition = scores_full_arr > 0.931 72 | print('Left {} out of {} runs'.format(len(scores_full_arr[condition]), len(scores_full_arr))) 73 | validation_full_arr = validation_full_arr[condition] 74 | test_preds_full_arr = test_preds_full_arr[condition] 75 | 76 | 77 | if 1: 78 | validation_arr = np.mean(np.array(validation_full_arr), axis=0) 79 | test_preds = np.mean(np.array(test_preds_full_arr), axis=0) 80 | else: 81 | validation_arr = gmean(validation_full_arr, axis=0) 82 | test_preds = gmean(test_preds_full_arr, axis=0) 83 | 84 | if np.count_nonzero(validation_arr < 0) > 0: 85 | print('Some error here..') 86 | exit() 87 | 88 | # Check validation 89 | best_score, searcher = get_optimal_score_very_fast(indexes, lbl, validation_arr, 7, 0.000001, 3) 90 | # best_score, searcher = get_optimal_score_slow(indexes, lbl, validation_arr) 91 | log_str = 'Best score {} for THR array: {}'.format(best_score, list(searcher)) 92 | print(log_str) 93 | log_list.append(log_str) 94 | 95 | preds = test_preds 96 | 97 | # Create submission 98 | tbl = pd.read_csv(INPUT_PATH + "sample_submission_v2.csv") 99 | ids = tbl['image_name'].values 100 | files = [] 101 | for id in ids: 102 | files.append(INPUT_PATH + "test-jpg/" + id + '.jpg') 103 | files = np.array(files) 104 | 105 | sub_file = OUTPUT_PATH + "merger_final_{}.csv".format(best_score) 106 | out = open(sub_file, "w") 107 | out.write("image_name,tags\n") 108 | for i in range(len(files)): 109 | out.write(ids[i] + ',') 110 | for j in range(len(indexes)): 111 | if preds[i][j] > searcher[j]: 112 | out.write(indexes[j] + ' ') 113 | out.write("\n") 114 | out.close() 115 | print('File with predictions was written in {}'.format(sub_file)) 116 | 117 | # Save raw test for analysis 118 | sub_file_raw = OUTPUT_PATH + "merger_cache_{}_raw_test.csv".format(best_score) 119 | out = open(sub_file_raw, "w") 120 | out.write("image_name") 121 | for i in indexes: 122 | out.write("," + str(i)) 123 | out.write("\n") 124 | 125 | for i in range(len(files)): 126 | out.write(ids[i]) 127 | for j in range(len(preds[i])): 128 | out.write(',' + str(preds[i][j])) 129 | out.write("\n") 130 | out.close() 131 | 132 | # Save raw validation for further analysis 133 | sub_file_raw = OUTPUT_PATH + "merger_cache_{}_raw_valid.csv".format(best_score) 134 | out = open(sub_file_raw, "w") 135 | out.write("image_name") 136 | for i in indexes: 137 | out.write("," + str(i)) 138 | out.write("\n") 139 | 140 | ids = list(train['image_name'].values) 141 | for i in range(validation_arr.shape[0]): 142 | out.write(ids[i]) 143 | for j in range(len(validation_arr[i])): 144 | out.write(',' + str(validation_arr[i][j])) 145 | out.write("\n") 146 | out.close() 147 | 148 | 149 | if __name__ == '__main__': 150 | start_time = time.time() 151 | 152 | # Currently used 150 XGBoost + 100 Keras runs. But there can be more 153 | # effective proportions like 150 vs 150 or 150 vs 80 etc. 154 | 155 | cache_files = glob.glob(CACHE_PATH + 'gbm_cache_iter_150_score*.pklz') + \ 156 | glob.glob(CACHE_PATH + 'keras_cache_iter_100_score*.pklz') 157 | 158 | ensemble_from_cache_v1(cache_files) 159 | print("Elapsed time overall: %s seconds" % (time.time() - start_time)) 160 | -------------------------------------------------------------------------------- /images/Dataflow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZFTurbo/Kaggle-Planet-Understanding-the-Amazon-from-Space/af653700a627c9af1c07d2f379bc9dacd88034e0/images/Dataflow.png --------------------------------------------------------------------------------