├── .gitignore ├── util.py ├── losses.py ├── LICENSE ├── save_thresholds.py ├── README.md ├── paths.py ├── find_best_threshold.py ├── labels.py ├── submit_ensemble.py ├── submit_predictions.py ├── modules.py ├── datasets.py ├── callbacks.py ├── nn_finetune_resnet_50.py ├── nn_finetune_resnet_34.py ├── nn_finetune_resnet_101.py ├── nn_finetune_densenet_201.py ├── predict.py ├── nn_finetune_densenet_169.py ├── nn_finetune_densenet_121.py ├── nn_semisupervised_resnet_50.py ├── nn_semisupervised_resnet_101.py ├── nn_semisupervised_resnet_18.py ├── nn_semisupervised_resnet_34.py ├── ModuleTrainer.py ├── nn_semisupervised_densenet_121.py ├── nn_semisupervised_densenet_169.py ├── transforms.py └── model_tta_hyperopt.py /.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | 3 | !*.py 4 | !*.csv 5 | !.gitignore 6 | !logs 7 | !submissions 8 | !submissions/* 9 | !ensemble_weights/* -------------------------------------------------------------------------------- /util.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | 3 | import paths 4 | 5 | 6 | def find_epoch_val(model_name): 7 | print(model_name) 8 | log = pd.read_csv(paths.logs + model_name) 9 | 10 | best_epoch = 0 11 | best_val_loss = float('inf') 12 | for row in log.iterrows(): 13 | index, cols = row 14 | epoch = cols['epoch'] 15 | loss = cols['loss'] 16 | val_loss = cols['val_loss'] 17 | 18 | if val_loss < best_val_loss: 19 | best_val_loss = val_loss 20 | best_epoch = epoch 21 | 22 | return best_epoch -------------------------------------------------------------------------------- /losses.py: -------------------------------------------------------------------------------- 1 | from torch.nn import BCELoss 2 | 3 | 4 | def PseudoLabelingLoss(loss_fn=BCELoss()): 5 | def call(prediction, y, i, epoch): 6 | i = i.data 7 | i_supervised = (i <= 0).nonzero().squeeze() 8 | i_unsupervised = (i > 0).nonzero().squeeze() 9 | 10 | supervised_prediction = prediction[i_supervised] 11 | supervised_y = y[i_supervised] 12 | unsupervised_prediction = prediction[i_unsupervised] 13 | unsupervised_y = y[i_unsupervised] 14 | 15 | supervised_loss = loss_fn(supervised_prediction, supervised_y) 16 | unsupervised_loss = loss_fn(unsupervised_prediction, unsupervised_y) 17 | 18 | a = max((epoch-12), 0) / 35 19 | 20 | loss = (supervised_loss + a * unsupervised_loss) / (1 + a) 21 | 22 | return loss 23 | 24 | return call -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Tim Joseph 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /save_thresholds.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import sklearn.model_selection 3 | import multiprocessing 4 | from itertools import repeat, chain 5 | 6 | import paths 7 | import labels 8 | from datasets import mlb 9 | from find_best_threshold import optimise_f2_thresholds_fast 10 | 11 | labels_df = labels.get_labels_df() 12 | kf = sklearn.model_selection.KFold(n_splits=5, shuffle=True, random_state=1) 13 | split = list(kf.split(labels_df)) 14 | 15 | models = [ 16 | 'nn_semisupervised_densenet_121', 17 | ] 18 | 19 | 20 | def do(model_fold): 21 | model, i = model_fold 22 | net = np.load(paths.predictions + '{}-split_{}.npz'.format(model, i)) 23 | train_idx, val_idx = split[i] 24 | train_predictions = net['train'] 25 | train_true = mlb.transform(labels_df.ix[train_idx]['tags'].str.split()).astype(np.float32) 26 | 27 | thresholds = [] 28 | for train in train_predictions: 29 | threshold = optimise_f2_thresholds_fast(train_true, train, verbose=True) 30 | thresholds.append(threshold) 31 | 32 | thresholds = np.stack(thresholds, axis=1) 33 | np.save(paths.thresholds + '{}-split_{}'.format(model, i), thresholds) 34 | print('Saved {}-split_{}'.format(model, i)) 35 | 36 | 37 | def flatmap(f, items): 38 | return chain.from_iterable(map(f, items)) 39 | 40 | models = flatmap(lambda m: repeat(m, 5), models) 41 | 42 | 43 | tbd = zip(models, range(5)) 44 | 45 | p = multiprocessing.Pool(10) 46 | for i in enumerate(p.imap(do, tbd)): 47 | print(i) -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # KagglePlanetPytorch 2 | This repository contains the basic code of our 9th place submission. 3 | 4 | For questions refere to https://www.kaggle.com/c/planet-understanding-the-amazon-from-space/discussion/36887 or create an issue! 5 | 6 | ### Requirements 7 | Basic requirements are 8 | * Scitkit-Learn 9 | * Scikit-Image 10 | * Numpy, Scipy 11 | * Torchsample 12 | * Pytorch 13 | * XGBoost 14 | 15 | This list may not be exhaustive! 16 | 17 | ### Training a network 18 | Just run the nn_finetune-files. 19 | 20 | ### Create predictions for a network 21 | Choose the network in predict.py and run it. Predictions are then saved to /predictions. 22 | 23 | ### Calculate thresholds 24 | This step is only necessary because of the current implementation. Run save_thresholds.py for your model. The saved thresholds we be used in the next step to compare XGBoost to averaging. 25 | 26 | ### Make a submission from a single 5-fold model 27 | Specify the network in model_tta_hyperopt.py and run it. This will run hyper parameter optmization for XGBoost. The approach chosen in this file is probably not good at all, since this was the first time I used XGBoost and only had a week to the competition deadline. Please tell me if you can do better. Also if you can make the same basic approach work for model ensembling, tell me! :) 28 | Submission are saved to /submissions 29 | 30 | ### Make a weighted submissions from different submission files 31 | Just specify your submissions and weights in submit_ensemble.py 32 | -------------------------------------------------------------------------------- /paths.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | logs = './logs/' 4 | models = './models/' 5 | submissions = './submissions/' 6 | data = './data/' 7 | validations = './validations/' 8 | predictions = './predictions/' 9 | thresholds = './thresholds/' 10 | ensemble_weights = './ensemble_weights/' 11 | xgb_configurations = './xgb_configurations/' 12 | 13 | train_jpg = '../Planet/train-jpg/' 14 | train_tif = '../Planet/train-tif-v2/' 15 | test_jpg = '../Planet/test-jpg/' 16 | train_csv = '../Planet/train_v2.csv' 17 | 18 | dirs = [ 19 | logs, 20 | models, 21 | submissions, 22 | data, 23 | validations, 24 | predictions, 25 | thresholds, 26 | ensemble_weights, 27 | xgb_configurations 28 | ] 29 | 30 | data = [train_jpg, train_tif, test_jpg] 31 | 32 | files = [train_csv] 33 | 34 | for supplementary_dir in dirs: 35 | if os.path.isdir(supplementary_dir): 36 | continue 37 | 38 | if not os.path.isfile(supplementary_dir[:-1]): 39 | os.makedirs(supplementary_dir) 40 | print('Created directory', supplementary_dir) 41 | 42 | for data_dir in data: 43 | if os.path.isdir(data_dir): 44 | continue 45 | 46 | else: 47 | print('Directoy {} does not exists. Please either put the training/test data in the appropriate directories or ' 48 | 'change the path.'.format(data_dir)) 49 | 50 | 51 | for file in files: 52 | if os.path.isfile(file): 53 | continue 54 | 55 | else: 56 | print('File {} does not exists. Please either put the file in the appropriate directories or ' 57 | 'change the path.'.format(file)) 58 | -------------------------------------------------------------------------------- /find_best_threshold.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from sklearn.metrics import fbeta_score, make_scorer 3 | import itertools 4 | import pathos.multiprocessing 5 | 6 | 7 | def fbeta(true_label, prediction): 8 | return fbeta_score(true_label, prediction, beta=2, average='samples') 9 | 10 | 11 | def optimise_f2_thresholds_fast(y, p, iterations=100, verbose=True): 12 | best_threshold = [0.2]*17 13 | for t in range(17): 14 | best_fbeta = 0 15 | temp_threshhold = [0.2]*17 16 | for i in range(iterations): 17 | temp_value = i / float(iterations) 18 | temp_threshhold[t] = temp_value 19 | temp_fbeta = fbeta(y, p > temp_threshhold) 20 | if temp_fbeta > best_fbeta: 21 | best_fbeta = temp_fbeta 22 | best_threshold[t] = temp_value 23 | 24 | if verbose: 25 | print(t, best_fbeta, best_threshold[t]) 26 | 27 | return best_threshold 28 | 29 | 30 | def optimise_f2_thresholds(y, p, verbose=True, resolution=100): 31 | def mf(x): 32 | p2 = np.zeros_like(p) 33 | for i in range(17): 34 | p2[:, i] = (p[:, i] > x[i]).astype(np.int) 35 | score = fbeta_score(y, p2, beta=2, average='samples') 36 | return score 37 | 38 | x = [0.2] * 17 39 | for i in range(17): 40 | best_i2 = 0 41 | best_score = 0 42 | for i2 in range(resolution): 43 | i2 /= resolution 44 | x[i] = i2 45 | score = mf(x) 46 | if score > best_score: 47 | best_i2 = i2 48 | best_score = score 49 | x[i] = best_i2 50 | if verbose: 51 | print(i, best_i2, best_score) 52 | 53 | return x -------------------------------------------------------------------------------- /labels.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pandas as pd 3 | import sklearn.model_selection 4 | 5 | import paths 6 | 7 | 8 | def get_labels(test_size=0.2, random_state=32): 9 | tmp_df = pd.read_csv(paths.train_csv) 10 | assert tmp_df['image_name'].apply(lambda x: os.path.isfile(paths.train_jpg + x + '.jpg')).all(), \ 11 | "Some images referenced in the CSV file were not found" 12 | 13 | if test_size != 0: 14 | train, val = sklearn.model_selection.train_test_split(tmp_df, test_size=test_size, random_state=random_state) 15 | 16 | else: 17 | print('Skipping validation split!') 18 | _, val = sklearn.model_selection.train_test_split(tmp_df, test_size=0.2, random_state=random_state) 19 | train = tmp_df 20 | 21 | return train, val 22 | 23 | 24 | def get_labels_df(): 25 | tmp_df = pd.read_csv(paths.train_csv) 26 | assert tmp_df['image_name'].apply(lambda x: os.path.isfile(paths.train_jpg + x + '.jpg')).all(), \ 27 | "Some images referenced in the CSV file were not found" 28 | 29 | return tmp_df 30 | 31 | 32 | def get_labels_stratified(test_size=0.2, random_state=32): 33 | tmp_df = pd.read_csv(paths.train_csv) 34 | assert tmp_df['image_name'].apply(lambda x: os.path.isfile(paths.train_jpg + x + '.jpg')).all(), \ 35 | "Some images referenced in the CSV file were not found" 36 | 37 | if test_size != 0: 38 | train, val = sklearn.model_selection.train_test_split(tmp_df, test_size=test_size, random_state=random_state) 39 | 40 | else: 41 | print('Skipping validation split!') 42 | _, val = sklearn.model_selection.train_test_split(tmp_df, test_size=0.2, random_state=random_state) 43 | train = tmp_df 44 | 45 | return train, val -------------------------------------------------------------------------------- /submit_ensemble.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | import pandas as pd 4 | import numpy as np 5 | 6 | import os 7 | 8 | import paths 9 | from datasets import mlb, KaggleAmazonTestDataset 10 | 11 | # (submission, weight) 12 | submissions = [ 13 | ('nn_semisupervised_resnet_50_XGB_HYPEROPT', 1), 14 | ('nn_semisupervised_resnet_50_random_v1_XGB_HYPEROPT', 1), 15 | ('nn_semisupervised_resnet_101_XGB_HYPEROPT', 1), 16 | ] 17 | 18 | submission_name = 'WEIGHTED_VOTING_SUBMISSION' 19 | 20 | 21 | def voting_submission(): 22 | submissions_raw = [] 23 | weight_sum = 0 24 | for (submission, weight) in submissions: 25 | df = pd.read_csv(paths.submissions + submission) 26 | df = df.sort_values('image_name') 27 | print(df[:3]) 28 | raw = mlb.transform(df['tags'].str.split()).astype(np.float32) 29 | submissions_raw.append(raw*weight) 30 | weight_sum += weight 31 | 32 | stacked_submissions = np.stack(submissions_raw, 0) 33 | avg_submissions = np.sum(stacked_submissions, 0) / weight_sum 34 | 35 | avg_submissions[avg_submissions >= 0.5] = 1 36 | avg_submissions[avg_submissions < 0.5] = 0 37 | 38 | tags = mlb.inverse_transform(avg_submissions) 39 | 40 | test_images = sorted(list(map(lambda path: path[:-len('.jpg')], os.listdir(paths.test_jpg)))) 41 | 42 | print(test_images[:3]) 43 | 44 | with open(paths.ensemble_weights + submission_name, 'w') as fp: 45 | json.dump(submissions, fp, sort_keys=True, indent=4) 46 | 47 | with open(paths.submissions + submission_name, 'w') as submission: 48 | submission.write('image_name,tags\n') 49 | for labels, target in zip(tags, test_images): 50 | output = target + ',' + ' '.join(labels) 51 | submission.write("%s\n" % output) 52 | 53 | print('Submission ready!') 54 | 55 | voting_submission() -------------------------------------------------------------------------------- /submit_predictions.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | 4 | from scipy.stats.mstats import gmean 5 | import sklearn.model_selection 6 | 7 | import paths 8 | import labels 9 | from datasets import mlb 10 | import find_best_threshold 11 | 12 | np.set_printoptions(threshold=np.nan) 13 | np.set_printoptions(suppress=True) 14 | 15 | 16 | def submit_cv_ensemble(ensemble, output_file): 17 | thresholds = [] 18 | tests = [] 19 | for net, val, train in ensemble: 20 | y_val = mlb.transform(train['tags'].str.split()).astype(np.float32) 21 | threshold = find_best_threshold.optimise_f2_thresholds_fast(y_val, net['train']) 22 | thresholds.append(threshold) 23 | test = net['test'] 24 | tests.append(test) 25 | 26 | threshold_avg = np.average(np.stack(thresholds, axis=0), axis=0) 27 | test_avg = np.average(np.stack(tests, axis=0), axis=0) 28 | 29 | test_images = list(map(lambda path: path[:-len('.jpg')], os.listdir(paths.test_jpg))) 30 | test_avg[test_avg > threshold_avg] = 1 31 | test_avg[test_avg <= threshold_avg] = 0 32 | 33 | predictions = mlb.inverse_transform(test_avg) 34 | test_results = zip(predictions, test_images) 35 | 36 | with open(paths.submissions + output_file, 'w') as submission: 37 | submission.write('image_name,tags\n') 38 | for tags, target in test_results: 39 | output = target + ',' + ' '.join(tags) 40 | submission.write("%s\n" % output) 41 | 42 | print('Submission ready!') 43 | 44 | 45 | def load_cv_folds(model_name): 46 | models = [] 47 | for i in range(5): 48 | net = np.load(paths.predictions + model_name + '-split_{}.npz'.format(i)) 49 | net = { 50 | 'train': np.average(net['train'], axis=0), 51 | 'val': np.average(net['val'], axis=0), 52 | 'test': np.average(net['test'], axis=0) 53 | } 54 | models.append(net) 55 | 56 | labels_df = labels.get_labels_df() 57 | kf = sklearn.model_selection.KFold(n_splits=5, shuffle=True, random_state=1) 58 | split = kf.split(labels_df) 59 | folds = [] 60 | for i, ((train_idx, val_idx), net) in enumerate(zip(split, models)): 61 | val = labels_df.ix[val_idx] 62 | train = labels_df.ix[train_idx] 63 | folds.append((net, val, train)) 64 | print(i) 65 | 66 | return folds 67 | 68 | # load_cv_folds takes the model name 69 | folds = load_cv_folds('nn_finetune_resnet_50') 70 | # you can chose a name yourself here, but I name my ensembled model_fold_1+2+3+4+5 71 | submit_cv_ensemble(folds, 'nn_finetune_resnet_50_fold_1+2+3+4+5') -------------------------------------------------------------------------------- /modules.py: -------------------------------------------------------------------------------- 1 | from torch import nn 2 | import torch 3 | import torch.nn.functional as F 4 | 5 | 6 | class PreActivationBlock(nn.Module): 7 | def __init__(self, in_channels, channels, kernel_size=3, padding=1, dropout=None): 8 | super(PreActivationBlock, self).__init__() 9 | self.bn = nn.BatchNorm2d(in_channels) 10 | self.conv = nn.Conv2d(in_channels, channels, kernel_size=kernel_size, stride=1, padding=padding, bias=False) 11 | self.dropout = dropout 12 | if dropout is not None: 13 | self.drop = nn.Dropout2d(p=dropout) 14 | 15 | def forward(self, x): 16 | x = self.bn(x) 17 | x = F.elu(x, inplace=True) 18 | x = self.conv(x) 19 | 20 | if self.dropout is not None: 21 | x = self.drop(x) 22 | 23 | return x 24 | 25 | 26 | class ResidualBlock(nn.Module): 27 | def __init__(self, in_channels, channels): 28 | super(ResidualBlock, self).__init__() 29 | 30 | self.conv1 = PreActivationBlock(in_channels, channels) 31 | self.conv2 = PreActivationBlock(in_channels, channels) 32 | 33 | def forward(self, x): 34 | residual = x 35 | x = self.conv1(x) 36 | x = self.conv2(x) 37 | x += residual 38 | 39 | return x 40 | 41 | 42 | class ResidualModule(nn.Module): 43 | def __init__(self, depth, channels): 44 | super(ResidualModule, self).__init__() 45 | 46 | layers = [] 47 | for i in range(depth): 48 | layers.append(ResidualBlock(channels, channels)) 49 | 50 | self.layer = nn.Sequential(*layers) 51 | 52 | def forward(self, x): 53 | x = self.layer(x) 54 | 55 | return x 56 | 57 | 58 | class TransitionBlock(nn.Module): 59 | def __init__(self, in_channels, channels, dropout=None): 60 | super(TransitionBlock, self).__init__() 61 | self.layer = PreActivationBlock(in_channels, channels, kernel_size=1, padding=0, dropout=dropout) 62 | self.pool = nn.MaxPool2d(2) 63 | 64 | def forward(self, x): 65 | x = self.layer(x) 66 | x = self.pool(x) 67 | 68 | return x 69 | 70 | 71 | class BottleneckBlock(nn.Module): 72 | def __init__(self, in_channels, k, dropout=None): 73 | super(BottleneckBlock, self).__init__() 74 | 75 | self.bottleneck = PreActivationBlock(in_channels, 4*k, 1, 0, dropout) 76 | self.layer = PreActivationBlock(4*k, k, dropout=dropout) 77 | 78 | def forward(self, x): 79 | residual = x 80 | x = self.bottleneck(x) 81 | x = self.layer(x) 82 | 83 | return torch.cat([residual, x], 1) 84 | 85 | 86 | class DenseModule(nn.Module): 87 | def __init__(self, in_channels, depth, k, dropout=None): 88 | super(DenseModule, self).__init__() 89 | layers = [] 90 | channels = in_channels 91 | for i in range(depth): 92 | layers.append(BottleneckBlock(channels + i*k, k, dropout=dropout)) 93 | 94 | self.layer = nn.Sequential(*layers) 95 | 96 | def forward(self, x): 97 | x = self.layer(x) 98 | 99 | return x -------------------------------------------------------------------------------- /datasets.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | from torch import np 4 | from torch.utils.data.dataset import Dataset 5 | from sklearn.preprocessing import MultiLabelBinarizer 6 | import skimage.io 7 | 8 | tags = [ 9 | 'blooming', 10 | 'selective_logging', 11 | 'blow_down', 12 | 'conventional_mine', 13 | 'bare_ground', 14 | 'artisinal_mine', 15 | 'primary', 16 | 'agriculture', 17 | 'water', 18 | 'habitation', 19 | 'road', 20 | 'cultivation', 21 | 'slash_burn' 22 | ] 23 | 24 | tags_weather = [ 25 | 'cloudy', 26 | 'partly_cloudy', 27 | 'haze', 28 | 'clear' 29 | ] 30 | mlb = MultiLabelBinarizer() 31 | mlb = mlb.fit([tags, tags_weather]) 32 | 33 | 34 | class KaggleAmazonDataset(Dataset): 35 | def __init__(self, dataframe, img_path, transform): 36 | self.img_path = img_path 37 | self.transform = transform 38 | 39 | self.X_train = dataframe['image_name'].as_matrix() 40 | self.y_train = mlb.transform(dataframe['tags'].str.split()).astype(np.float32) 41 | 42 | def __len__(self): 43 | return len(self.X_train) 44 | 45 | 46 | class KaggleAmazonJPGDataset(KaggleAmazonDataset): 47 | def __init__(self, dataframe, img_path, transform, divide=True): 48 | super(KaggleAmazonJPGDataset, self).__init__(dataframe, img_path, transform) 49 | 50 | self.divide = divide 51 | 52 | def __getitem__(self, index): 53 | img = skimage.io.imread(self.img_path + self.X_train[index] + '.jpg') 54 | 55 | if self.divide: 56 | img = img / 255 57 | 58 | if self.transform: 59 | img = self.transform(img) 60 | 61 | label = torch.from_numpy(self.y_train[index]) 62 | return img, label 63 | 64 | 65 | class KaggleAmazonTestDataset(Dataset): 66 | 67 | def __init__(self, test_images, img_path, img_ext, transform, divide=True): 68 | self.img_path = img_path 69 | self.img_ext = img_ext 70 | self.transform = transform 71 | self.test_images = test_images 72 | self.divide = divide 73 | 74 | def __getitem__(self, index): 75 | img = skimage.io.imread(self.img_path + self.test_images[index] + self.img_ext) 76 | 77 | if self.divide: 78 | img = img / 255 79 | 80 | img = self.transform(img) 81 | 82 | return img, self.test_images[index] 83 | 84 | def __len__(self): 85 | return len(self.test_images) 86 | 87 | 88 | class KaggleAmazonUnsupervisedDataset(Dataset): 89 | def __init__(self, paths, img_path, img_ext, transform_train, transform_val, y_train): 90 | self.img_path = img_path 91 | self.img_ext = img_ext 92 | self.transform_train = transform_train 93 | self.transform = transform_train 94 | self.transform_val = transform_val 95 | self.X_train = paths 96 | self.y_train = y_train 97 | 98 | def __getitem__(self, index): 99 | img = skimage.io.imread(self.img_path + self.X_train[index] + self.img_ext) 100 | 101 | if self.transform: 102 | img = self.transform(img) 103 | 104 | label = torch.from_numpy(self.y_train[index]) 105 | 106 | return img, label 107 | 108 | def __len__(self): 109 | return len(self.X_train) 110 | 111 | 112 | class KaggleAmazonSemiSupervisedDataset(Dataset): 113 | def __init__(self, supervised, unsupervised, transform, indices=True): 114 | self.supervised = supervised 115 | self.unsupervised = unsupervised 116 | self.transform = transform 117 | self.indices = indices 118 | 119 | def __getitem__(self, index): 120 | if index < len(self.supervised): 121 | x, y = self.supervised[index] 122 | i = 0 123 | 124 | else: 125 | x, y = self.unsupervised[index-len(self.supervised)] 126 | i = 1 127 | 128 | if self.transform: 129 | x = self.transform(x) 130 | 131 | if self.indices: 132 | return x, y, i 133 | else: 134 | return x, y 135 | 136 | def __len__(self): 137 | return len(self.supervised) + len(self.unsupervised) 138 | 139 | 140 | if __name__ == "__main__": 141 | print(mlb.classes) 142 | -------------------------------------------------------------------------------- /callbacks.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | 4 | import os 5 | import time 6 | from torchsample import Callback 7 | import torch as th 8 | from torch.autograd import Variable 9 | from torch.utils.data import DataLoader 10 | 11 | 12 | class ModelCheckpoint(Callback): 13 | """ 14 | Model Checkpoint to save model weights during training 15 | """ 16 | 17 | def __init__(self, 18 | directory, 19 | name, 20 | monitor='val_loss', 21 | save_best_only=False, 22 | save_weights_only=True, 23 | saving_strategy=lambda epoch: True, 24 | mode=lambda loss_new, loss_old: loss_new < loss_old, 25 | verbose=0): 26 | 27 | self.directory = os.path.join(directory, name) 28 | self.file = os.path.join(self.directory, name) 29 | self.monitor = monitor 30 | self.save_best_only = save_best_only 31 | self.save_weights_only = save_weights_only 32 | self.mode = mode 33 | self.saving_strategy = saving_strategy 34 | self.verbose = verbose 35 | self.best_loss = None 36 | 37 | print('') 38 | 39 | os.makedirs(self.directory, exist_ok=True) 40 | 41 | super(ModelCheckpoint, self).__init__() 42 | 43 | def on_epoch_end(self, epoch, logs=None): 44 | if not self.saving_strategy(epoch): 45 | return 46 | 47 | if self.save_best_only: 48 | loss = logs.get(self.monitor) 49 | if loss is None: 50 | pass 51 | else: 52 | file = self.file + '-best_epoch_{}'.format(epoch) 53 | if self.best_loss is None: 54 | self.best_loss = loss 55 | if self.verbose > 0: 56 | print('\nEpoch {}: Saving model for the first time to {}'.format(epoch + 1, file)) 57 | self.model.save_state_dict(file) 58 | 59 | elif self.mode(self.best_loss, loss): 60 | if self.verbose > 0: 61 | print('\nEpoch {}: improved from {} to {} saving model to {}'.format(epoch+1, self.best_loss, loss, file)) 62 | self.best_loss = loss 63 | self.model.save_state_dict(file) 64 | 65 | else: 66 | file = self.file + '-epoch_{}'.format(epoch) 67 | if self.verbose > 0: 68 | print('\nEpoch %i: saving model to %s' % (epoch+1, file)) 69 | self.model.save_state_dict(file) 70 | 71 | 72 | class SemiSupervisedUpdater(Callback): 73 | def __init__(self, trainer, dataset, start_epoch=0, momentum=0, batch_size=96): 74 | super(SemiSupervisedUpdater, self).__init__() 75 | 76 | self.trainer = trainer 77 | self.dataset = dataset 78 | self.start_epoch = start_epoch 79 | self.loader = DataLoader(dataset, batch_size=batch_size, shuffle=False, num_workers=10, pin_memory=True) 80 | self.momentum = momentum 81 | 82 | def predict_loader(self, 83 | loader, 84 | cuda_device=-1): 85 | prediction_list = [] 86 | for batch_idx, batch_data in enumerate(loader): 87 | if not isinstance(batch_data, (tuple, list)): 88 | batch_data = [batch_data] 89 | input_batch = batch_data[0] 90 | if not isinstance(input_batch, (list, tuple)): 91 | input_batch = [input_batch] 92 | input_batch = [Variable(ins, volatile=True) for ins in input_batch] 93 | if cuda_device > -1: 94 | input_batch = [ins.cuda(cuda_device) for ins in input_batch] 95 | 96 | prediction_list.append(self.model.model(*input_batch)) 97 | 98 | # concatenate all outputs of the same type together (when there are multiple outputs) 99 | if len(prediction_list) > 0 and isinstance(prediction_list[0], (tuple, list)): 100 | nb_out = len(prediction_list[0]) 101 | out_list = [] 102 | for out_i in range(nb_out): 103 | precdiction_out_i = [prediction[out_i] for prediction in prediction_list] 104 | out_list.append(th.cat(precdiction_out_i, 0)) 105 | return out_list 106 | 107 | return th.cat(prediction_list, 0) 108 | 109 | def on_epoch_begin(self, epoch, logs=None): 110 | if epoch < self.start_epoch: 111 | return 112 | 113 | self.dataset.transform = self.dataset.transform_val 114 | predictions = self.predict_loader(self.loader, cuda_device=0).cpu().data.numpy() 115 | 116 | self.dataset.transform = self.dataset.transform_train 117 | self.dataset.y_train = (1 - self.momentum) * self.dataset.y_train + predictions * self.momentum 118 | 119 | -------------------------------------------------------------------------------- /nn_finetune_resnet_50.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | from itertools import chain 5 | 6 | import numpy as np 7 | 8 | import torchvision.models 9 | import torch.nn.functional as F 10 | import torch.optim as optim 11 | from torch import nn 12 | import torch.nn.init 13 | from torch.utils.data import DataLoader 14 | 15 | from torchsample.modules import ModuleTrainer 16 | from torchsample.callbacks import CSVLogger, LearningRateScheduler 17 | 18 | import sklearn.model_selection 19 | 20 | import paths 21 | import labels 22 | import transforms 23 | import callbacks 24 | from datasets import KaggleAmazonJPGDataset 25 | 26 | name = os.path.basename(sys.argv[0])[:-3] 27 | 28 | 29 | def generate_model(): 30 | class MyModel(nn.Module): 31 | def __init__(self, pretrained_model): 32 | super(MyModel, self).__init__() 33 | self.pretrained_model = pretrained_model 34 | self.layer1 = pretrained_model.layer1 35 | self.layer2 = pretrained_model.layer2 36 | self.layer3 = pretrained_model.layer3 37 | self.layer4 = pretrained_model.layer4 38 | 39 | pretrained_model.avgpool = nn.AvgPool2d(8) 40 | classifier = [ 41 | nn.Linear(pretrained_model.fc.in_features, 17), 42 | ] 43 | 44 | self.classifier = nn.Sequential(*classifier) 45 | pretrained_model.fc = self.classifier 46 | 47 | def forward(self, x): 48 | return F.sigmoid(self.pretrained_model(x)) 49 | 50 | return MyModel(torchvision.models.resnet50(pretrained=True)) 51 | 52 | random_state = 1 53 | labels_df = labels.get_labels_df() 54 | kf = sklearn.model_selection.KFold(n_splits=5, shuffle=True, random_state=random_state) 55 | split = kf.split(labels_df) 56 | 57 | 58 | def train_net(train, val, model, name): 59 | transformations_train = transforms.apply_chain([ 60 | transforms.random_fliplr(), 61 | transforms.random_flipud(), 62 | transforms.augment(), 63 | torchvision.transforms.ToTensor() 64 | ]) 65 | 66 | transformations_val = transforms.apply_chain([ 67 | torchvision.transforms.ToTensor(), 68 | ]) 69 | 70 | dset_train = KaggleAmazonJPGDataset(train, paths.train_jpg, transformations_train, divide=False) 71 | train_loader = DataLoader(dset_train, 72 | batch_size=64, 73 | shuffle=True, 74 | num_workers=10, 75 | pin_memory=True) 76 | 77 | dset_val = KaggleAmazonJPGDataset(val, paths.train_jpg, transformations_val, divide=False) 78 | val_loader = DataLoader(dset_val, 79 | batch_size=64, 80 | num_workers=10, 81 | pin_memory=True) 82 | 83 | ignored_params = list(map(id, chain( 84 | model.classifier.parameters(), 85 | model.layer1.parameters(), 86 | model.layer2.parameters(), 87 | model.layer3.parameters(), 88 | model.layer4.parameters() 89 | ))) 90 | base_params = filter(lambda p: id(p) not in ignored_params, 91 | model.parameters()) 92 | 93 | optimizer = optim.Adam([ 94 | {'params': base_params}, 95 | {'params': model.layer1.parameters()}, 96 | {'params': model.layer2.parameters()}, 97 | {'params': model.layer3.parameters()}, 98 | {'params': model.layer4.parameters()}, 99 | {'params': model.classifier.parameters()} 100 | ], lr=0, weight_decay=0.0005) 101 | 102 | trainer = ModuleTrainer(model) 103 | 104 | def schedule(current_epoch, current_lrs, **logs): 105 | lrs = [1e-3, 1e-4, 1e-5] 106 | epochs = [0, 2, 10] 107 | 108 | for lr, epoch in zip(lrs, epochs): 109 | if current_epoch >= epoch: 110 | current_lrs[5] = lr 111 | if current_epoch >= 1: 112 | current_lrs[4] = lr * 0.4 113 | current_lrs[3] = lr * 0.2 114 | current_lrs[2] = lr * 0.1 115 | current_lrs[1] = lr * 0.05 116 | current_lrs[0] = lr * 0.01 117 | 118 | return current_lrs 119 | 120 | trainer.set_callbacks([ 121 | callbacks.ModelCheckpoint( 122 | paths.models, 123 | name, 124 | save_best_only=False, 125 | saving_strategy=lambda epoch: True 126 | ), 127 | CSVLogger('./logs/' + name), 128 | LearningRateScheduler(schedule) 129 | ]) 130 | 131 | trainer.compile(loss=nn.BCELoss(), 132 | optimizer=optimizer) 133 | 134 | trainer.fit_loader(train_loader, 135 | val_loader, 136 | nb_epoch=35, 137 | verbose=1, 138 | cuda_device=0) 139 | 140 | 141 | if __name__ == "__main__": 142 | for i, (train_idx, val_idx) in enumerate(split): 143 | name = os.path.basename(sys.argv[0])[:-3] + '-split_' + str(i) 144 | train_net(labels_df.ix[train_idx], labels_df.ix[val_idx], generate_model(), name) 145 | -------------------------------------------------------------------------------- /nn_finetune_resnet_34.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | from itertools import chain 5 | 6 | import numpy as np 7 | 8 | import torchvision.models 9 | import torch.nn.functional as F 10 | import torch.optim as optim 11 | from torch import nn 12 | import torch.nn.init 13 | from torch.utils.data import DataLoader 14 | 15 | from torchsample.modules import ModuleTrainer 16 | from torchsample.callbacks import CSVLogger, LearningRateScheduler 17 | 18 | import sklearn.model_selection 19 | 20 | import paths 21 | import labels 22 | import transforms 23 | import callbacks 24 | from datasets import KaggleAmazonJPGDataset 25 | 26 | name = os.path.basename(sys.argv[0])[:-3] 27 | 28 | 29 | def generate_model(): 30 | class MyModel(nn.Module): 31 | def __init__(self, pretrained_model): 32 | super(MyModel, self).__init__() 33 | self.pretrained_model = pretrained_model 34 | self.layer1 = pretrained_model.layer1 35 | self.layer2 = pretrained_model.layer2 36 | self.layer3 = pretrained_model.layer3 37 | self.layer4 = pretrained_model.layer4 38 | 39 | pretrained_model.avgpool = nn.AvgPool2d(8) 40 | classifier = [ 41 | nn.Linear(pretrained_model.fc.in_features, 17), 42 | ] 43 | 44 | self.classifier = nn.Sequential(*classifier) 45 | pretrained_model.fc = self.classifier 46 | 47 | def forward(self, x): 48 | return F.sigmoid(self.pretrained_model(x)) 49 | 50 | return MyModel(torchvision.models.resnet34(pretrained=True)) 51 | 52 | random_state = 1 53 | labels_df = labels.get_labels_df() 54 | kf = sklearn.model_selection.KFold(n_splits=5, shuffle=True, random_state=random_state) 55 | split = kf.split(labels_df) 56 | 57 | 58 | def train_net(train, val, model, name): 59 | transformations_train = transforms.apply_chain([ 60 | transforms.to_float, 61 | transforms.augment_color(0.1), 62 | transforms.random_fliplr(), 63 | transforms.random_flipud(), 64 | transforms.augment(), 65 | torchvision.transforms.ToTensor() 66 | ]) 67 | 68 | transformations_val = transforms.apply_chain([ 69 | transforms.to_float, 70 | torchvision.transforms.ToTensor() 71 | ]) 72 | 73 | dset_train = KaggleAmazonJPGDataset(train, paths.train_jpg, transformations_train, divide=False) 74 | train_loader = DataLoader(dset_train, 75 | batch_size=128, 76 | shuffle=True, 77 | num_workers=10, 78 | pin_memory=True) 79 | 80 | dset_val = KaggleAmazonJPGDataset(val, paths.train_jpg, transformations_val, divide=False) 81 | val_loader = DataLoader(dset_val, 82 | batch_size=128, 83 | num_workers=10, 84 | pin_memory=True) 85 | 86 | ignored_params = list(map(id, chain( 87 | model.classifier.parameters(), 88 | model.layer1.parameters(), 89 | model.layer2.parameters(), 90 | model.layer3.parameters(), 91 | model.layer4.parameters() 92 | ))) 93 | base_params = filter(lambda p: id(p) not in ignored_params, 94 | model.parameters()) 95 | 96 | optimizer = optim.Adam([ 97 | {'params': base_params}, 98 | {'params': model.layer1.parameters()}, 99 | {'params': model.layer2.parameters()}, 100 | {'params': model.layer3.parameters()}, 101 | {'params': model.layer4.parameters()}, 102 | {'params': model.classifier.parameters()} 103 | ], lr=0, weight_decay=0.0005) 104 | 105 | trainer = ModuleTrainer(model) 106 | 107 | def schedule(current_epoch, current_lrs, **logs): 108 | lrs = [1e-3, 1e-4, 0.5e-4, 1e-5] 109 | epochs = [0, 2, 8, 14] 110 | 111 | for lr, epoch in zip(lrs, epochs): 112 | if current_epoch >= epoch: 113 | current_lrs[5] = lr 114 | if current_epoch >= 4: 115 | current_lrs[4] = lr * 1.0 116 | current_lrs[3] = lr * 1.0 117 | current_lrs[2] = lr * 1.0 118 | current_lrs[1] = lr * 0.1 119 | current_lrs[0] = lr * 0.05 120 | 121 | return current_lrs 122 | 123 | trainer.set_callbacks([ 124 | callbacks.ModelCheckpoint( 125 | paths.models, 126 | name, 127 | save_best_only=False, 128 | saving_strategy=lambda epoch: True 129 | ), 130 | CSVLogger('./logs/' + name), 131 | LearningRateScheduler(schedule) 132 | ]) 133 | 134 | trainer.compile(loss=nn.BCELoss(), 135 | optimizer=optimizer) 136 | 137 | trainer.fit_loader(train_loader, 138 | val_loader, 139 | nb_epoch=35, 140 | verbose=1, 141 | cuda_device=0) 142 | 143 | 144 | if __name__ == "__main__": 145 | for i, (train_idx, val_idx) in enumerate(split): 146 | name = os.path.basename(sys.argv[0])[:-3] + '-split_' + str(i) 147 | train_net(labels_df.ix[train_idx], labels_df.ix[val_idx], generate_model(), name) 148 | -------------------------------------------------------------------------------- /nn_finetune_resnet_101.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | from itertools import chain 5 | 6 | import numpy as np 7 | 8 | import torchvision.models 9 | import torch.nn.functional as F 10 | import torch.optim as optim 11 | from torch import nn 12 | import torch.nn.init 13 | from torch.utils.data import DataLoader 14 | 15 | from torchsample.modules import ModuleTrainer 16 | from torchsample.callbacks import CSVLogger, LearningRateScheduler 17 | 18 | import sklearn.model_selection 19 | 20 | import paths 21 | import labels 22 | import transforms 23 | import callbacks 24 | from datasets import KaggleAmazonJPGDataset 25 | 26 | name = os.path.basename(sys.argv[0])[:-3] 27 | 28 | 29 | def generate_model(): 30 | class MyModel(nn.Module): 31 | def __init__(self, pretrained_model): 32 | super(MyModel, self).__init__() 33 | self.pretrained_model = pretrained_model 34 | self.layer1 = pretrained_model.layer1 35 | self.layer2 = pretrained_model.layer2 36 | self.layer3 = pretrained_model.layer3 37 | self.layer4 = pretrained_model.layer4 38 | 39 | pretrained_model.avgpool = nn.AvgPool2d(8) 40 | classifier = [ 41 | nn.Linear(pretrained_model.fc.in_features, 17), 42 | ] 43 | 44 | self.classifier = nn.Sequential(*classifier) 45 | pretrained_model.fc = self.classifier 46 | 47 | def forward(self, x): 48 | return F.sigmoid(self.pretrained_model(x)) 49 | 50 | return MyModel(torchvision.models.resnet101(pretrained=True)) 51 | 52 | random_state = 1 53 | labels_df = labels.get_labels_df() 54 | kf = sklearn.model_selection.KFold(n_splits=5, shuffle=True, random_state=random_state) 55 | split = kf.split(labels_df) 56 | 57 | 58 | def train_net(train, val, model, name): 59 | transformations_train = transforms.apply_chain([ 60 | transforms.to_float, 61 | transforms.augment_color(0.1), 62 | transforms.random_fliplr(), 63 | transforms.random_flipud(), 64 | transforms.augment(), 65 | torchvision.transforms.ToTensor() 66 | ]) 67 | 68 | transformations_val = transforms.apply_chain([ 69 | transforms.to_float, 70 | torchvision.transforms.ToTensor() 71 | ]) 72 | 73 | dset_train = KaggleAmazonJPGDataset(train, paths.train_jpg, transformations_train, divide=False) 74 | train_loader = DataLoader(dset_train, 75 | batch_size=32, 76 | shuffle=True, 77 | num_workers=10, 78 | pin_memory=True) 79 | 80 | dset_val = KaggleAmazonJPGDataset(val, paths.train_jpg, transformations_val, divide=False) 81 | val_loader = DataLoader(dset_val, 82 | batch_size=32, 83 | num_workers=10, 84 | pin_memory=True) 85 | 86 | ignored_params = list(map(id, chain( 87 | model.classifier.parameters(), 88 | model.layer1.parameters(), 89 | model.layer2.parameters(), 90 | model.layer3.parameters(), 91 | model.layer4.parameters() 92 | ))) 93 | base_params = filter(lambda p: id(p) not in ignored_params, 94 | model.parameters()) 95 | 96 | optimizer = optim.Adam([ 97 | {'params': base_params}, 98 | {'params': model.layer1.parameters()}, 99 | {'params': model.layer2.parameters()}, 100 | {'params': model.layer3.parameters()}, 101 | {'params': model.layer4.parameters()}, 102 | {'params': model.classifier.parameters()} 103 | ], lr=0, weight_decay=0.0005) 104 | 105 | trainer = ModuleTrainer(model) 106 | 107 | def schedule(current_epoch, current_lrs, **logs): 108 | lrs = [1e-3, 1e-4, 0.5e-4, 1e-5, 0.5e-5] 109 | epochs = [0, 2, 12, 20, 25] 110 | 111 | for lr, epoch in zip(lrs, epochs): 112 | if current_epoch >= epoch: 113 | current_lrs[5] = lr 114 | if current_epoch >= 4: 115 | current_lrs[4] = lr * 1.0 116 | current_lrs[3] = lr * 1.0 117 | current_lrs[2] = lr * 1.0 118 | current_lrs[1] = lr * 0.1 119 | current_lrs[0] = lr * 0.05 120 | 121 | return current_lrs 122 | 123 | trainer.set_callbacks([ 124 | callbacks.ModelCheckpoint( 125 | paths.models, 126 | name, 127 | save_best_only=False, 128 | saving_strategy=lambda epoch: True 129 | ), 130 | CSVLogger('./logs/' + name), 131 | LearningRateScheduler(schedule) 132 | ]) 133 | 134 | trainer.compile(loss=nn.BCELoss(), 135 | optimizer=optimizer) 136 | 137 | trainer.fit_loader(train_loader, 138 | val_loader, 139 | nb_epoch=35, 140 | verbose=1, 141 | cuda_device=0) 142 | 143 | 144 | if __name__ == "__main__": 145 | for i, (train_idx, val_idx) in enumerate(split): 146 | name = os.path.basename(sys.argv[0])[:-3] + '-split_' + str(i) 147 | train_net(labels_df.ix[train_idx], labels_df.ix[val_idx], generate_model(), name) 148 | -------------------------------------------------------------------------------- /nn_finetune_densenet_201.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | from itertools import chain 5 | 6 | import numpy as np 7 | 8 | import torchvision.models 9 | import torch.nn.functional as F 10 | import torch.optim as optim 11 | from torch import nn 12 | import torch.nn.init 13 | from torch.utils.data import DataLoader 14 | 15 | from torchsample.modules import ModuleTrainer 16 | from torchsample.callbacks import CSVLogger, LearningRateScheduler 17 | 18 | import sklearn.model_selection 19 | 20 | import paths 21 | import labels 22 | import transforms 23 | import callbacks 24 | from datasets import KaggleAmazonJPGDataset 25 | 26 | name = os.path.basename(sys.argv[0])[:-3] 27 | 28 | 29 | def generate_model(): 30 | class DenseModel(nn.Module): 31 | def __init__(self, pretrained_model): 32 | super(DenseModel, self).__init__() 33 | self.classifier = nn.Linear(pretrained_model.classifier.in_features, 17) 34 | 35 | for m in self.modules(): 36 | if isinstance(m, nn.Conv2d): 37 | nn.init.kaiming_normal(m.weight) 38 | elif isinstance(m, nn.BatchNorm2d): 39 | m.weight.data.fill_(1) 40 | m.bias.data.zero_() 41 | elif isinstance(m, nn.Linear): 42 | m.bias.data.zero_() 43 | 44 | self.features = pretrained_model.features 45 | self.dense1 = pretrained_model.features._modules['denseblock1'] 46 | self.dense2 = pretrained_model.features._modules['denseblock2'] 47 | self.dense3 = pretrained_model.features._modules['denseblock3'] 48 | self.dense4 = pretrained_model.features._modules['denseblock4'] 49 | 50 | def forward(self, x): 51 | features = self.features(x) 52 | out = F.relu(features, inplace=True) 53 | out = F.avg_pool2d(out, kernel_size=8).view(features.size(0), -1) 54 | out = F.sigmoid(self.classifier(out)) 55 | return out 56 | 57 | return DenseModel(torchvision.models.densenet201(pretrained=True)) 58 | 59 | 60 | random_state = 1 61 | labels_df = labels.get_labels_df() 62 | kf = sklearn.model_selection.KFold(n_splits=5, shuffle=True, random_state=random_state) 63 | split = kf.split(labels_df) 64 | 65 | 66 | def train_net(train, val, model, name): 67 | transformations_train = transforms.apply_chain([ 68 | transforms.random_fliplr(), 69 | transforms.random_flipud(), 70 | transforms.augment(), 71 | torchvision.transforms.ToTensor() 72 | ]) 73 | 74 | transformations_val = transforms.apply_chain([ 75 | torchvision.transforms.ToTensor() 76 | ]) 77 | 78 | dset_train = KaggleAmazonJPGDataset(train, paths.train_jpg, transformations_train, divide=False) 79 | train_loader = DataLoader(dset_train, 80 | batch_size=32, 81 | shuffle=True, 82 | num_workers=10, 83 | pin_memory=True) 84 | 85 | dset_val = KaggleAmazonJPGDataset(val, paths.train_jpg, transformations_val, divide=False) 86 | val_loader = DataLoader(dset_val, 87 | batch_size=32, 88 | num_workers=10, 89 | pin_memory=True) 90 | 91 | ignored_params = list(map(id, chain( 92 | model.classifier.parameters(), 93 | model.dense1.parameters(), 94 | model.dense2.parameters(), 95 | model.dense3.parameters(), 96 | model.dense4.parameters() 97 | ))) 98 | base_params = filter(lambda p: id(p) not in ignored_params, 99 | model.parameters()) 100 | 101 | optimizer = optim.Adam([ 102 | {'params': base_params}, 103 | {'params': model.dense1.parameters()}, 104 | {'params': model.dense2.parameters()}, 105 | {'params': model.dense3.parameters()}, 106 | {'params': model.dense4.parameters()}, 107 | {'params': model.classifier.parameters()} 108 | ], lr=0, weight_decay=0.0005) 109 | 110 | trainer = ModuleTrainer(model) 111 | 112 | # New settings 113 | def schedule(current_epoch, current_lrs, **logs): 114 | lrs = [1e-3, 1e-4, 1e-5] 115 | epochs = [0, 4, 20] 116 | 117 | for lr, epoch in zip(lrs, epochs): 118 | if current_epoch >= epoch: 119 | current_lrs[5] = lr 120 | if current_epoch >= 2: 121 | current_lrs[4] = lr * 1.0 122 | current_lrs[3] = lr * 1.0 123 | current_lrs[2] = lr * 0.2 124 | current_lrs[1] = lr * 0.1 125 | current_lrs[0] = lr * 0.05 126 | 127 | return current_lrs 128 | 129 | trainer.set_callbacks([ 130 | callbacks.ModelCheckpoint( 131 | paths.models, 132 | name, 133 | save_best_only=False, 134 | saving_strategy=lambda epoch: True 135 | ), 136 | CSVLogger('./logs/' + name), 137 | LearningRateScheduler(schedule) 138 | ]) 139 | 140 | trainer.compile(loss=nn.BCELoss(), 141 | optimizer=optimizer) 142 | 143 | trainer.fit_loader(train_loader, 144 | val_loader, 145 | nb_epoch=35, 146 | verbose=1, 147 | cuda_device=0) 148 | 149 | 150 | if __name__ == "__main__": 151 | for i, (train_idx, val_idx) in enumerate(split): 152 | name = os.path.basename(sys.argv[0])[:-3] + '-split_' + str(i) 153 | train_net(labels_df.ix[train_idx], labels_df.ix[val_idx], generate_model(), name) -------------------------------------------------------------------------------- /predict.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | import itertools 4 | import torch 5 | import numpy as np 6 | import pandas as pd 7 | from pydoc import locate 8 | 9 | from torch.autograd import Variable 10 | import torchvision.transforms 11 | from torch.utils.data import DataLoader 12 | 13 | import sklearn.model_selection 14 | 15 | import paths 16 | import transforms 17 | import labels 18 | import util 19 | from datasets import KaggleAmazonJPGDataset, KaggleAmazonTestDataset, mlb 20 | 21 | batch_size = 128 22 | 23 | 24 | def predict(net, loaders): 25 | net.cuda() 26 | net.eval() 27 | 28 | def apply_transform(loader): 29 | predictions_acc = [] 30 | for batch_idx, (data, targets) in enumerate(loader): 31 | data = Variable(data.cuda(async=True), volatile=True) 32 | output = net(data) 33 | predictions = output.cpu().data.numpy() 34 | 35 | for prediction, target in zip(predictions, targets): 36 | predictions_acc.append(prediction) 37 | 38 | print('[{}/{} ({:.0f}%)]'.format(batch_idx * len(data), len(loader.dataset), 39 | 100. * batch_idx / len(loader)), end='\r') 40 | sys.stdout.flush() 41 | 42 | predictions_acc = np.array(predictions_acc) 43 | 44 | return predictions_acc 45 | 46 | results = [] 47 | 48 | for i, l in enumerate(loaders): 49 | print('TTA {}'.format(i)) 50 | r = apply_transform(l) 51 | results.append(r) 52 | 53 | return np.stack(results, axis=0) 54 | 55 | 56 | def get_loader(df, transformations): 57 | dset_val = KaggleAmazonJPGDataset(df, paths.train_jpg, transformations, divide=False) 58 | 59 | loader_val = DataLoader(dset_val, 60 | batch_size=batch_size, 61 | num_workers=12, 62 | pin_memory=True,) 63 | return loader_val 64 | 65 | 66 | def get_test_loader(test_images, transformations): 67 | dset_test = KaggleAmazonTestDataset(test_images, paths.test_jpg, '.jpg', transformations, divide=False) 68 | loader_val = DataLoader(dset_test, 69 | batch_size=batch_size, 70 | num_workers=12, 71 | pin_memory=True) 72 | return loader_val 73 | 74 | 75 | def predict_model(model, state, train, val, output_file, pre_transforms): 76 | model.load_state_dict(state) 77 | 78 | transformations = [ 79 | [transforms.rotate_90(0)], 80 | [transforms.rotate_90(1)], 81 | [transforms.rotate_90(2)], 82 | [transforms.rotate_90(3)], 83 | [transforms.fliplr()], 84 | [transforms.fliplr()], 85 | [transforms.augment_deterministic(scale_factor=1/1.1)], 86 | [transforms.augment_deterministic(scale_factor=1/1.2)], 87 | [transforms.augment_deterministic(scale_factor=1.1)], 88 | [transforms.augment_deterministic(scale_factor=1.2)], 89 | [transforms.augment_deterministic(translation=(20, 20))], 90 | [transforms.augment_deterministic(translation=(-20, 20))], 91 | [transforms.augment_deterministic(translation=(20, -20))], 92 | [transforms.augment_deterministic(translation=(-20, -20))], 93 | ] 94 | 95 | transformations = list(map(lambda t: transforms.apply_chain( 96 | pre_transforms + t + [ 97 | torchvision.transforms.ToTensor() 98 | ] 99 | ), transformations)) 100 | 101 | train_loaders = map(lambda t: get_loader(train, t), transformations) 102 | val_loaders = map(lambda t: get_loader(val, t), transformations) 103 | 104 | test_images = list(map(lambda path: path[:-len('.jpg')], os.listdir(paths.test_jpg))) 105 | test_loaders = map(lambda t: get_test_loader(test_images, t), transformations) 106 | 107 | print('Predicting val...') 108 | val_predictions = predict(model, val_loaders) 109 | print(val_predictions.shape) 110 | 111 | print('Predicting train...') 112 | train_predictions = predict(model, train_loaders) 113 | print(train_predictions.shape) 114 | 115 | print('Predicting test...') 116 | test_predictions = predict(model, test_loaders) 117 | print(test_predictions.shape) 118 | 119 | 120 | # Save train, val and test 121 | np.savez( 122 | paths.predictions + output_file, 123 | train=train_predictions, 124 | val=val_predictions, 125 | test=test_predictions, 126 | ) 127 | 128 | 129 | def predict_kfold(model_name, pre_transforms=[]): 130 | model = locate(model_name + '.generate_model')() 131 | random_state = locate(model_name + '.random_state') 132 | print('Random state: {}'.format(random_state)) 133 | 134 | labels_df = labels.get_labels_df() 135 | kf = sklearn.model_selection.KFold(n_splits=5, shuffle=True, random_state=random_state) 136 | split = kf.split(labels_df) 137 | 138 | for i, (train_idx, val_idx) in enumerate(split): 139 | split_name = model_name + '-split_' + str(i) 140 | best_epoch = util.find_epoch_val(split_name) 141 | print('Using epoch {} for predictions'.format(best_epoch)) 142 | epoch_name = split_name + '-epoch_' + str(best_epoch) 143 | train = labels_df.ix[train_idx] 144 | val = labels_df.ix[val_idx] 145 | state = torch.load(os.path.join(paths.models, split_name, epoch_name)) 146 | 147 | predict_model(model, state, train, val, output_file=split_name, pre_transforms=pre_transforms) 148 | 149 | if __name__ == "__main__": 150 | # Use the model name without -fold_x or -epoch_k here. Will automatically use the best epoch of every fold! 151 | predict_kfold('nn_semisupervised_densenet_121') 152 | 153 | -------------------------------------------------------------------------------- /nn_finetune_densenet_169.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | from itertools import chain 5 | 6 | 7 | import torchvision.models 8 | import torch.nn.functional as F 9 | import torch.optim as optim 10 | from torch import nn 11 | import torch.nn.init 12 | from torch.utils.data import DataLoader 13 | 14 | from torchsample.modules import ModuleTrainer 15 | from torchsample.callbacks import CSVLogger, LearningRateScheduler 16 | 17 | import sklearn.model_selection 18 | 19 | import paths 20 | import labels 21 | import transforms 22 | import callbacks 23 | from datasets import KaggleAmazonJPGDataset 24 | 25 | name = os.path.basename(sys.argv[0])[:-3] 26 | 27 | 28 | def generate_model(): 29 | class DenseModel(nn.Module): 30 | def __init__(self, pretrained_model): 31 | super(DenseModel, self).__init__() 32 | self.classifier = nn.Linear(pretrained_model.classifier.in_features, 17) 33 | 34 | for m in self.modules(): 35 | if isinstance(m, nn.Conv2d): 36 | nn.init.kaiming_normal(m.weight) 37 | elif isinstance(m, nn.BatchNorm2d): 38 | m.weight.data.fill_(1) 39 | m.bias.data.zero_() 40 | elif isinstance(m, nn.Linear): 41 | m.bias.data.zero_() 42 | 43 | self.features = pretrained_model.features 44 | self.dense1 = pretrained_model.features._modules['denseblock1'] 45 | self.dense2 = pretrained_model.features._modules['denseblock2'] 46 | self.dense3 = pretrained_model.features._modules['denseblock3'] 47 | self.dense4 = pretrained_model.features._modules['denseblock4'] 48 | 49 | def forward(self, x): 50 | features = self.features(x) 51 | out = F.relu(features, inplace=True) 52 | out = F.avg_pool2d(out, kernel_size=8).view(features.size(0), -1) 53 | out = F.sigmoid(self.classifier(out)) 54 | return out 55 | 56 | return DenseModel(torchvision.models.densenet169(pretrained=True)) 57 | 58 | 59 | random_state = 1 60 | labels_df = labels.get_labels_df() 61 | kf = sklearn.model_selection.KFold(n_splits=5, shuffle=True, random_state=random_state) 62 | split = kf.split(labels_df) 63 | 64 | 65 | def train_net(train, val, model, name): 66 | transformations_train = transforms.apply_chain([ 67 | transforms.to_float, 68 | transforms.random_fliplr(), 69 | transforms.random_flipud(), 70 | transforms.augment_color(0.1), 71 | transforms.augment(), 72 | torchvision.transforms.ToTensor() 73 | ]) 74 | 75 | transformations_val = transforms.apply_chain([ 76 | transforms.to_float, 77 | torchvision.transforms.ToTensor() 78 | ]) 79 | 80 | dset_train = KaggleAmazonJPGDataset(train, paths.train_jpg, transformations_train, divide=False) 81 | train_loader = DataLoader(dset_train, 82 | batch_size=32, 83 | shuffle=True, 84 | num_workers=10, 85 | pin_memory=True) 86 | 87 | dset_val = KaggleAmazonJPGDataset(val, paths.train_jpg, transformations_val, divide=False) 88 | val_loader = DataLoader(dset_val, 89 | batch_size=32, 90 | num_workers=10, 91 | pin_memory=True) 92 | 93 | ignored_params = list(map(id, chain( 94 | model.classifier.parameters(), 95 | model.dense1.parameters(), 96 | model.dense2.parameters(), 97 | model.dense3.parameters(), 98 | model.dense4.parameters() 99 | ))) 100 | base_params = filter(lambda p: id(p) not in ignored_params, 101 | model.parameters()) 102 | 103 | optimizer = optim.Adam([ 104 | {'params': base_params}, 105 | {'params': model.dense1.parameters()}, 106 | {'params': model.dense2.parameters()}, 107 | {'params': model.dense3.parameters()}, 108 | {'params': model.dense4.parameters()}, 109 | {'params': model.classifier.parameters()} 110 | ], lr=0, weight_decay=0.0005) 111 | 112 | trainer = ModuleTrainer(model) 113 | 114 | # New settings 115 | def schedule(current_epoch, current_lrs, **logs): 116 | lrs = [1e-3, 1e-4, 0.5e-4, 1e-5, 0.5e-5] 117 | epochs = [0, 4, 15, 20, 25] 118 | 119 | for lr, epoch in zip(lrs, epochs): 120 | if current_epoch >= epoch: 121 | current_lrs[5] = lr 122 | if current_epoch >= 2: 123 | current_lrs[4] = lr * 1 124 | current_lrs[3] = lr * 1 125 | current_lrs[2] = lr * 0.5 126 | current_lrs[1] = lr * 0.2 127 | current_lrs[0] = lr * 0.1 128 | 129 | return current_lrs 130 | 131 | 132 | trainer.set_callbacks([ 133 | callbacks.ModelCheckpoint( 134 | paths.models, 135 | name, 136 | save_best_only=False, 137 | saving_strategy=lambda epoch: True 138 | ), 139 | CSVLogger('./logs/' + name), 140 | LearningRateScheduler(schedule) 141 | ]) 142 | 143 | trainer.compile(loss=nn.BCELoss(), 144 | optimizer=optimizer) 145 | 146 | trainer.fit_loader(train_loader, 147 | val_loader, 148 | nb_epoch=30, 149 | verbose=1, 150 | cuda_device=0) 151 | 152 | 153 | if __name__ == "__main__": 154 | for i, (train_idx, val_idx) in enumerate(split): 155 | name = os.path.basename(sys.argv[0])[:-3] + '-split_' + str(i) 156 | train_net(labels_df.ix[train_idx], labels_df.ix[val_idx], generate_model(), name) -------------------------------------------------------------------------------- /nn_finetune_densenet_121.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | from itertools import chain 5 | 6 | import numpy as np 7 | 8 | import torchvision.models 9 | import torch.nn.functional as F 10 | import torch.optim as optim 11 | from torch import nn 12 | import torch.nn.init 13 | from torch.utils.data import DataLoader 14 | 15 | from torchsample.modules import ModuleTrainer 16 | from torchsample.callbacks import CSVLogger, LearningRateScheduler 17 | 18 | import sklearn.model_selection 19 | 20 | import paths 21 | import labels 22 | import transforms 23 | import callbacks 24 | from datasets import KaggleAmazonJPGDataset 25 | 26 | name = os.path.basename(sys.argv[0])[:-3] 27 | 28 | 29 | def generate_model(): 30 | class DenseModel(nn.Module): 31 | def __init__(self, pretrained_model): 32 | super(DenseModel, self).__init__() 33 | self.classifier = nn.Linear(pretrained_model.classifier.in_features, 17) 34 | 35 | for m in self.modules(): 36 | if isinstance(m, nn.Conv2d): 37 | nn.init.kaiming_normal(m.weight) 38 | elif isinstance(m, nn.BatchNorm2d): 39 | m.weight.data.fill_(1) 40 | m.bias.data.zero_() 41 | elif isinstance(m, nn.Linear): 42 | m.bias.data.zero_() 43 | 44 | self.features = pretrained_model.features 45 | self.dense1 = pretrained_model.features._modules['denseblock1'] 46 | self.dense2 = pretrained_model.features._modules['denseblock2'] 47 | self.dense3 = pretrained_model.features._modules['denseblock3'] 48 | self.dense4 = pretrained_model.features._modules['denseblock4'] 49 | 50 | def forward(self, x): 51 | features = self.features(x) 52 | out = F.relu(features, inplace=True) 53 | out = F.avg_pool2d(out, kernel_size=8).view(features.size(0), -1) 54 | out = F.sigmoid(self.classifier(out)) 55 | return out 56 | 57 | return DenseModel(torchvision.models.densenet121(pretrained=True)) 58 | 59 | 60 | random_state = 1 61 | labels_df = labels.get_labels_df() 62 | kf = sklearn.model_selection.KFold(n_splits=5, shuffle=True, random_state=random_state) 63 | split = kf.split(labels_df) 64 | 65 | 66 | def train_net(train, val, model, name): 67 | transformations_train = transforms.apply_chain([ 68 | transforms.to_float, 69 | transforms.augment_color(0.5), 70 | transforms.random_fliplr(), 71 | transforms.random_flipud(), 72 | transforms.augment(), 73 | torchvision.transforms.ToTensor() 74 | ]) 75 | 76 | transformations_val = transforms.apply_chain([ 77 | transforms.to_float, 78 | torchvision.transforms.ToTensor() 79 | ]) 80 | 81 | dset_train = KaggleAmazonJPGDataset(train, paths.train_jpg, transformations_train, divide=False) 82 | train_loader = DataLoader(dset_train, 83 | batch_size=32, 84 | shuffle=True, 85 | num_workers=10, 86 | pin_memory=True) 87 | 88 | dset_val = KaggleAmazonJPGDataset(val, paths.train_jpg, transformations_val, divide=False) 89 | val_loader = DataLoader(dset_val, 90 | batch_size=32, 91 | num_workers=10, 92 | pin_memory=True) 93 | 94 | ignored_params = list(map(id, chain( 95 | model.classifier.parameters(), 96 | model.dense1.parameters(), 97 | model.dense2.parameters(), 98 | model.dense3.parameters(), 99 | model.dense4.parameters() 100 | ))) 101 | base_params = filter(lambda p: id(p) not in ignored_params, 102 | model.parameters()) 103 | 104 | optimizer = optim.Adam([ 105 | {'params': base_params}, 106 | {'params': model.dense1.parameters()}, 107 | {'params': model.dense2.parameters()}, 108 | {'params': model.dense3.parameters()}, 109 | {'params': model.dense4.parameters()}, 110 | {'params': model.classifier.parameters()} 111 | ], lr=0, weight_decay=0.0005) 112 | 113 | trainer = ModuleTrainer(model) 114 | 115 | # New settings 116 | def schedule(current_epoch, current_lrs, **logs): 117 | lrs = [1e-3, 1e-4, 1e-5] 118 | epochs = [0, 4, 30] 119 | 120 | for lr, epoch in zip(lrs, epochs): 121 | if current_epoch >= epoch: 122 | current_lrs[5] = lr 123 | if current_epoch >= 2: 124 | current_lrs[4] = lr * 1.0 125 | current_lrs[3] = lr * 0.5 126 | current_lrs[2] = lr * 0.25 127 | current_lrs[1] = lr * 0.125 128 | current_lrs[0] = lr * 0.1 129 | 130 | return current_lrs 131 | 132 | 133 | trainer.set_callbacks([ 134 | callbacks.ModelCheckpoint( 135 | paths.models, 136 | name, 137 | save_best_only=False, 138 | saving_strategy=lambda epoch: True 139 | ), 140 | CSVLogger('./logs/' + name), 141 | LearningRateScheduler(schedule) 142 | ]) 143 | 144 | trainer.compile(loss=nn.BCELoss(), 145 | optimizer=optimizer) 146 | 147 | trainer.fit_loader(train_loader, 148 | val_loader, 149 | nb_epoch=35, 150 | verbose=1, 151 | cuda_device=0) 152 | 153 | 154 | if __name__ == "__main__": 155 | for i, (train_idx, val_idx) in enumerate(split): 156 | name = os.path.basename(sys.argv[0])[:-3] + '-split_' + str(i) 157 | train_net(labels_df.ix[train_idx], labels_df.ix[val_idx], generate_model(), name) -------------------------------------------------------------------------------- /nn_semisupervised_resnet_50.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | from itertools import chain 5 | 6 | import numpy as np 7 | import pandas as pd 8 | 9 | import torchvision.models 10 | import torch.nn.functional as F 11 | import torch.optim as optim 12 | from torch import nn 13 | from torch.utils.data import DataLoader 14 | 15 | from torchsample.callbacks import CSVLogger, LearningRateScheduler 16 | from callbacks import ModelCheckpoint, SemiSupervisedUpdater 17 | 18 | import sklearn.model_selection 19 | 20 | import paths 21 | import labels 22 | import transforms 23 | from datasets import KaggleAmazonUnsupervisedDataset, KaggleAmazonSemiSupervisedDataset, KaggleAmazonJPGDataset, mlb 24 | from ModuleTrainer import ModuleTrainer 25 | 26 | name = os.path.basename(sys.argv[0])[:-3] 27 | 28 | 29 | def generate_model(): 30 | class MyModel(nn.Module): 31 | def __init__(self, pretrained_model): 32 | super(MyModel, self).__init__() 33 | self.pretrained_model = pretrained_model 34 | self.layer1 = pretrained_model.layer1 35 | self.layer2 = pretrained_model.layer2 36 | self.layer3 = pretrained_model.layer3 37 | self.layer4 = pretrained_model.layer4 38 | 39 | pretrained_model.avgpool = nn.AvgPool2d(8) 40 | classifier = [ 41 | nn.Linear(pretrained_model.fc.in_features, 17), 42 | ] 43 | 44 | self.classifier = nn.Sequential(*classifier) 45 | pretrained_model.fc = self.classifier 46 | 47 | def forward(self, x): 48 | return F.sigmoid(self.pretrained_model(x)) 49 | 50 | return MyModel(torchvision.models.resnet50(pretrained=True)) 51 | 52 | 53 | random_state = 1 54 | labels_df = labels.get_labels_df() 55 | unsupervised_dataframe = pd.read_csv(paths.submissions + 'SOTA') 56 | 57 | kf = sklearn.model_selection.KFold(n_splits=5, shuffle=True, random_state=random_state) 58 | split = supervised_split = kf.split(labels_df) 59 | unsupervised_split = kf.split(unsupervised_dataframe) 60 | 61 | 62 | def train_net(train, val, unsupervised, model, name): 63 | unsupervised_initialization = mlb.transform(unsupervised['tags'].str.split()).astype(np.float32) 64 | unsupervised_samples = unsupervised['image_name'].as_matrix() 65 | 66 | unsupervised_initialization = unsupervised_initialization[:len(unsupervised_initialization)//2*3] 67 | unsupervised_samples = unsupervised_samples[:len(unsupervised_samples)//2*3] 68 | 69 | transformations_train = transforms.apply_chain([ 70 | transforms.random_fliplr(), 71 | transforms.random_flipud(), 72 | transforms.augment(), 73 | torchvision.transforms.ToTensor() 74 | ]) 75 | 76 | transformations_val = transforms.apply_chain([ 77 | torchvision.transforms.ToTensor() 78 | ]) 79 | 80 | dset_train_unsupervised = KaggleAmazonUnsupervisedDataset( 81 | unsupervised_samples, 82 | paths.test_jpg, 83 | '.jpg', 84 | transformations_train, 85 | transformations_val, 86 | unsupervised_initialization 87 | ) 88 | 89 | dset_train_supervised = KaggleAmazonJPGDataset(train, paths.train_jpg, transformations_train, divide=False) 90 | dset_train = KaggleAmazonSemiSupervisedDataset(dset_train_supervised, dset_train_unsupervised, None, indices=False) 91 | 92 | train_loader = DataLoader(dset_train, 93 | batch_size=64, 94 | shuffle=True, 95 | num_workers=10, 96 | pin_memory=True) 97 | 98 | dset_val = KaggleAmazonJPGDataset(val, paths.train_jpg, transformations_val, divide=False) 99 | val_loader = DataLoader(dset_val, 100 | batch_size=64, 101 | num_workers=10, 102 | pin_memory=True) 103 | 104 | ignored_params = list(map(id, chain( 105 | model.classifier.parameters(), 106 | model.layer1.parameters(), 107 | model.layer2.parameters(), 108 | model.layer3.parameters(), 109 | model.layer4.parameters() 110 | ))) 111 | base_params = filter(lambda p: id(p) not in ignored_params, 112 | model.parameters()) 113 | 114 | optimizer = optim.Adam([ 115 | {'params': base_params}, 116 | {'params': model.layer1.parameters()}, 117 | {'params': model.layer2.parameters()}, 118 | {'params': model.layer3.parameters()}, 119 | {'params': model.layer4.parameters()}, 120 | {'params': model.classifier.parameters()} 121 | ], lr=0, weight_decay=0.0001) 122 | 123 | trainer = ModuleTrainer(model) 124 | 125 | def schedule(current_epoch, current_lrs, **logs): 126 | lrs = [1e-3, 1e-4, 0.5e-4, 1e-5, 0.5e-5] 127 | epochs = [0, 1, 6, 8, 12] 128 | 129 | for lr, epoch in zip(lrs, epochs): 130 | if current_epoch >= epoch: 131 | current_lrs[5] = lr 132 | if current_epoch >= 2: 133 | current_lrs[4] = lr * 1 134 | current_lrs[3] = lr * 1 135 | current_lrs[2] = lr * 1 136 | current_lrs[1] = lr * 1 137 | current_lrs[0] = lr * 0.1 138 | 139 | return current_lrs 140 | 141 | trainer.set_callbacks([ 142 | ModelCheckpoint( 143 | paths.models, 144 | name, 145 | save_best_only=False, 146 | saving_strategy=lambda epoch: True 147 | ), 148 | CSVLogger(paths.logs + name), 149 | LearningRateScheduler(schedule), 150 | SemiSupervisedUpdater(trainer, dset_train_unsupervised, start_epoch=6, momentum=0.25) 151 | ]) 152 | 153 | trainer.compile(loss=nn.BCELoss(), 154 | optimizer=optimizer) 155 | 156 | trainer.fit_loader(train_loader, 157 | val_loader, 158 | nb_epoch=16, 159 | verbose=1, 160 | cuda_device=0) 161 | 162 | if __name__ == "__main__": 163 | for i, ((train_idx, val_idx), (train_idx_unsupervised, val_idx_unsupervised)) in enumerate(zip(supervised_split, unsupervised_split)): 164 | name = os.path.basename(sys.argv[0])[:-3] + '-split_' + str(i) 165 | train_net( 166 | labels_df.ix[train_idx], 167 | labels_df.ix[val_idx], 168 | unsupervised_dataframe.ix[train_idx_unsupervised][::4], 169 | generate_model(), 170 | name 171 | ) 172 | -------------------------------------------------------------------------------- /nn_semisupervised_resnet_101.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | from itertools import chain 5 | 6 | import numpy as np 7 | import pandas as pd 8 | 9 | import torchvision.models 10 | import torch.nn.functional as F 11 | import torch.optim as optim 12 | from torch import nn 13 | from torch.utils.data import DataLoader 14 | 15 | from torchsample.callbacks import CSVLogger, LearningRateScheduler 16 | from callbacks import ModelCheckpoint, SemiSupervisedUpdater 17 | 18 | import sklearn.model_selection 19 | 20 | import paths 21 | import labels 22 | import transforms 23 | from datasets import KaggleAmazonUnsupervisedDataset, KaggleAmazonSemiSupervisedDataset, KaggleAmazonJPGDataset, mlb 24 | from ModuleTrainer import ModuleTrainer 25 | 26 | name = os.path.basename(sys.argv[0])[:-3] 27 | 28 | 29 | def generate_model(): 30 | class MyModel(nn.Module): 31 | def __init__(self, pretrained_model): 32 | super(MyModel, self).__init__() 33 | self.pretrained_model = pretrained_model 34 | self.layer1 = pretrained_model.layer1 35 | self.layer2 = pretrained_model.layer2 36 | self.layer3 = pretrained_model.layer3 37 | self.layer4 = pretrained_model.layer4 38 | 39 | pretrained_model.avgpool = nn.AvgPool2d(8) 40 | classifier = [ 41 | nn.Linear(pretrained_model.fc.in_features, 17), 42 | ] 43 | 44 | self.classifier = nn.Sequential(*classifier) 45 | pretrained_model.fc = self.classifier 46 | 47 | def forward(self, x): 48 | return F.sigmoid(self.pretrained_model(x)) 49 | 50 | return MyModel(torchvision.models.resnet101(pretrained=True)) 51 | 52 | 53 | random_state = 1 54 | labels_df = labels.get_labels_df() 55 | unsupervised_dataframe = pd.read_csv(paths.submissions + 'SOTA') 56 | 57 | kf = sklearn.model_selection.KFold(n_splits=5, shuffle=True, random_state=random_state) 58 | split = supervised_split = kf.split(labels_df) 59 | unsupervised_split = kf.split(unsupervised_dataframe) 60 | 61 | 62 | def train_net(train, val, unsupervised, model, name): 63 | unsupervised_initialization = mlb.transform(unsupervised['tags'].str.split()).astype(np.float32) 64 | unsupervised_samples = unsupervised['image_name'].as_matrix() 65 | 66 | unsupervised_initialization = unsupervised_initialization[:len(unsupervised_initialization)//2*3] 67 | unsupervised_samples = unsupervised_samples[:len(unsupervised_samples)//2*3] 68 | 69 | transformations_train = transforms.apply_chain([ 70 | transforms.random_fliplr(), 71 | transforms.random_flipud(), 72 | transforms.augment(), 73 | torchvision.transforms.ToTensor() 74 | ]) 75 | 76 | transformations_val = transforms.apply_chain([ 77 | torchvision.transforms.ToTensor() 78 | ]) 79 | 80 | dset_train_unsupervised = KaggleAmazonUnsupervisedDataset( 81 | unsupervised_samples, 82 | paths.test_jpg, 83 | '.jpg', 84 | transformations_train, 85 | transformations_val, 86 | unsupervised_initialization 87 | ) 88 | 89 | dset_train_supervised = KaggleAmazonJPGDataset(train, paths.train_jpg, transformations_train, divide=False) 90 | dset_train = KaggleAmazonSemiSupervisedDataset(dset_train_supervised, dset_train_unsupervised, None, indices=False) 91 | 92 | train_loader = DataLoader(dset_train, 93 | batch_size=32, 94 | shuffle=True, 95 | num_workers=10, 96 | pin_memory=True) 97 | 98 | dset_val = KaggleAmazonJPGDataset(val, paths.train_jpg, transformations_val, divide=False) 99 | val_loader = DataLoader(dset_val, 100 | batch_size=32, 101 | num_workers=10, 102 | pin_memory=True) 103 | 104 | ignored_params = list(map(id, chain( 105 | model.classifier.parameters(), 106 | model.layer1.parameters(), 107 | model.layer2.parameters(), 108 | model.layer3.parameters(), 109 | model.layer4.parameters() 110 | ))) 111 | base_params = filter(lambda p: id(p) not in ignored_params, 112 | model.parameters()) 113 | 114 | optimizer = optim.Adam([ 115 | {'params': base_params}, 116 | {'params': model.layer1.parameters()}, 117 | {'params': model.layer2.parameters()}, 118 | {'params': model.layer3.parameters()}, 119 | {'params': model.layer4.parameters()}, 120 | {'params': model.classifier.parameters()} 121 | ], lr=0, weight_decay=0.0001) 122 | 123 | trainer = ModuleTrainer(model) 124 | 125 | def schedule(current_epoch, current_lrs, **logs): 126 | lrs = [1e-3, 1e-4, 0.5e-4, 1e-5, 0.5e-5] 127 | epochs = [0, 1, 6, 8, 12] 128 | 129 | for lr, epoch in zip(lrs, epochs): 130 | if current_epoch >= epoch: 131 | current_lrs[5] = lr 132 | if current_epoch >= 2: 133 | current_lrs[4] = lr * 1 134 | current_lrs[3] = lr * 1 135 | current_lrs[2] = lr * 1 136 | current_lrs[1] = lr * 1 137 | current_lrs[0] = lr * 0.1 138 | 139 | return current_lrs 140 | 141 | trainer.set_callbacks([ 142 | ModelCheckpoint( 143 | paths.models, 144 | name, 145 | save_best_only=False, 146 | saving_strategy=lambda epoch: True 147 | ), 148 | CSVLogger(paths.logs + name), 149 | LearningRateScheduler(schedule), 150 | SemiSupervisedUpdater(trainer, dset_train_unsupervised, start_epoch=6, momentum=0.25) 151 | ]) 152 | 153 | trainer.compile(loss=nn.BCELoss(), 154 | optimizer=optimizer) 155 | 156 | trainer.fit_loader(train_loader, 157 | val_loader, 158 | nb_epoch=16, 159 | verbose=1, 160 | cuda_device=0) 161 | 162 | if __name__ == "__main__": 163 | for i, ((train_idx, val_idx), (train_idx_unsupervised, val_idx_unsupervised)) in enumerate(zip(supervised_split, unsupervised_split)): 164 | name = os.path.basename(sys.argv[0])[:-3] + '-split_' + str(i) 165 | train_net( 166 | labels_df.ix[train_idx], 167 | labels_df.ix[val_idx], 168 | unsupervised_dataframe.ix[train_idx_unsupervised][::4], 169 | generate_model(), 170 | name 171 | ) 172 | -------------------------------------------------------------------------------- /nn_semisupervised_resnet_18.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | from itertools import chain 5 | 6 | import numpy as np 7 | import pandas as pd 8 | 9 | import torchvision.models 10 | import torch.nn.functional as F 11 | import torch.optim as optim 12 | from torch import nn 13 | from torch.utils.data import DataLoader 14 | 15 | from torchsample.callbacks import CSVLogger, LearningRateScheduler 16 | from callbacks import ModelCheckpoint, SemiSupervisedUpdater 17 | 18 | import sklearn.model_selection 19 | 20 | import paths 21 | import labels 22 | import transforms 23 | from datasets import KaggleAmazonUnsupervisedDataset, KaggleAmazonSemiSupervisedDataset, KaggleAmazonJPGDataset, mlb 24 | from ModuleTrainer import ModuleTrainer 25 | 26 | name = os.path.basename(sys.argv[0])[:-3] 27 | 28 | 29 | def generate_model(): 30 | class MyModel(nn.Module): 31 | def __init__(self, pretrained_model): 32 | super(MyModel, self).__init__() 33 | self.pretrained_model = pretrained_model 34 | self.layer1 = pretrained_model.layer1 35 | self.layer2 = pretrained_model.layer2 36 | self.layer3 = pretrained_model.layer3 37 | self.layer4 = pretrained_model.layer4 38 | 39 | pretrained_model.avgpool = nn.AvgPool2d(8) 40 | classifier = [ 41 | nn.Linear(pretrained_model.fc.in_features, 17), 42 | ] 43 | 44 | self.classifier = nn.Sequential(*classifier) 45 | pretrained_model.fc = self.classifier 46 | 47 | def forward(self, x): 48 | return self.pretrained_model(x) 49 | 50 | return MyModel(torchvision.models.resnet18(pretrained=True)) 51 | 52 | 53 | random_state = 1 54 | labels_df = labels.get_labels_df() 55 | unsupervised_dataframe = pd.read_csv(paths.submissions + 'SOTA') 56 | 57 | kf = sklearn.model_selection.KFold(n_splits=5, shuffle=True, random_state=random_state) 58 | split = supervised_split = kf.split(labels_df) 59 | unsupervised_split = kf.split(unsupervised_dataframe) 60 | 61 | 62 | def train_net(train, val, unsupervised, model, name): 63 | unsupervised_initialization = mlb.transform(unsupervised['tags'].str.split()).astype(np.float32) 64 | unsupervised_samples = unsupervised['image_name'].as_matrix() 65 | 66 | unsupervised_initialization = unsupervised_initialization[:len(unsupervised_initialization)//2*3] 67 | unsupervised_samples = unsupervised_samples[:len(unsupervised_samples)//2*3] 68 | 69 | transformations_train = transforms.apply_chain([ 70 | transforms.to_float, 71 | transforms.augment_color(0.1), 72 | transforms.random_fliplr(), 73 | transforms.random_flipud(), 74 | transforms.augment(), 75 | torchvision.transforms.ToTensor() 76 | ]) 77 | 78 | transformations_val = transforms.apply_chain([ 79 | transforms.to_float, 80 | torchvision.transforms.ToTensor() 81 | ]) 82 | 83 | dset_train_unsupervised = KaggleAmazonUnsupervisedDataset( 84 | unsupervised_samples, 85 | paths.test_jpg, 86 | '.jpg', 87 | transformations_train, 88 | transformations_val, 89 | unsupervised_initialization 90 | ) 91 | 92 | dset_train_supervised = KaggleAmazonJPGDataset(train, paths.train_jpg, transformations_train, divide=False) 93 | dset_train = KaggleAmazonSemiSupervisedDataset(dset_train_supervised, dset_train_unsupervised, None, indices=False) 94 | 95 | train_loader = DataLoader(dset_train, 96 | batch_size=128, 97 | shuffle=True, 98 | num_workers=10, 99 | pin_memory=True) 100 | 101 | dset_val = KaggleAmazonJPGDataset(val, paths.train_jpg, transformations_val, divide=False) 102 | val_loader = DataLoader(dset_val, 103 | batch_size=128, 104 | num_workers=10, 105 | pin_memory=True) 106 | 107 | ignored_params = list(map(id, chain( 108 | model.classifier.parameters(), 109 | model.layer1.parameters(), 110 | model.layer2.parameters(), 111 | model.layer3.parameters(), 112 | model.layer4.parameters() 113 | ))) 114 | base_params = filter(lambda p: id(p) not in ignored_params, 115 | model.parameters()) 116 | 117 | optimizer = optim.Adam([ 118 | {'params': base_params}, 119 | {'params': model.layer1.parameters()}, 120 | {'params': model.layer2.parameters()}, 121 | {'params': model.layer3.parameters()}, 122 | {'params': model.layer4.parameters()}, 123 | {'params': model.classifier.parameters()} 124 | ], lr=0, weight_decay=0.0005) 125 | 126 | trainer = ModuleTrainer(model) 127 | 128 | def schedule(current_epoch, current_lrs, **logs): 129 | lrs = [1e-3, 1e-4, 0.5e-4, 1e-5, 0.5e-5] 130 | epochs = [0, 1, 8, 12, 20] 131 | 132 | for lr, epoch in zip(lrs, epochs): 133 | if current_epoch >= epoch: 134 | current_lrs[5] = lr 135 | if current_epoch >= 2: 136 | current_lrs[4] = lr * 1.0 137 | current_lrs[3] = lr * 1.0 138 | current_lrs[2] = lr * 1.0 139 | current_lrs[1] = lr * 0.1 140 | current_lrs[0] = lr * 0.05 141 | 142 | return current_lrs 143 | 144 | trainer.set_callbacks([ 145 | ModelCheckpoint( 146 | paths.models, 147 | name, 148 | save_best_only=False, 149 | saving_strategy=lambda epoch: True 150 | ), 151 | CSVLogger(paths.logs + name), 152 | LearningRateScheduler(schedule), 153 | SemiSupervisedUpdater(trainer, dset_train_unsupervised, start_epoch=10, momentum=0.25) 154 | ]) 155 | 156 | trainer.compile(loss=nn.BCELoss(), 157 | optimizer=optimizer) 158 | 159 | trainer.fit_loader(train_loader, 160 | val_loader, 161 | nb_epoch=35, 162 | verbose=1, 163 | cuda_device=0) 164 | 165 | if __name__ == "__main__": 166 | for i, ((train_idx, val_idx), (train_idx_unsupervised, val_idx_unsupervised)) in enumerate(zip(supervised_split, unsupervised_split)): 167 | name = os.path.basename(sys.argv[0])[:-3] + '-split_' + str(i) 168 | train_net( 169 | labels_df.ix[train_idx], 170 | labels_df.ix[val_idx], 171 | unsupervised_dataframe.ix[val_idx_unsupervised], 172 | generate_model(), 173 | name 174 | ) 175 | -------------------------------------------------------------------------------- /nn_semisupervised_resnet_34.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | from itertools import chain 5 | 6 | import numpy as np 7 | import pandas as pd 8 | 9 | import torchvision.models 10 | import torch.nn.functional as F 11 | import torch.optim as optim 12 | from torch import nn 13 | from torch.utils.data import DataLoader 14 | 15 | from torchsample.callbacks import CSVLogger, LearningRateScheduler 16 | from callbacks import ModelCheckpoint, SemiSupervisedUpdater 17 | 18 | import sklearn.model_selection 19 | 20 | import paths 21 | import labels 22 | import transforms 23 | from datasets import KaggleAmazonUnsupervisedDataset, KaggleAmazonSemiSupervisedDataset, KaggleAmazonJPGDataset, mlb 24 | from ModuleTrainer import ModuleTrainer 25 | 26 | name = os.path.basename(sys.argv[0])[:-3] 27 | 28 | 29 | def generate_model(): 30 | class MyModel(nn.Module): 31 | def __init__(self, pretrained_model): 32 | super(MyModel, self).__init__() 33 | self.pretrained_model = pretrained_model 34 | self.layer1 = pretrained_model.layer1 35 | self.layer2 = pretrained_model.layer2 36 | self.layer3 = pretrained_model.layer3 37 | self.layer4 = pretrained_model.layer4 38 | 39 | pretrained_model.avgpool = nn.AvgPool2d(8) 40 | classifier = [ 41 | nn.Linear(pretrained_model.fc.in_features, 17), 42 | ] 43 | 44 | self.classifier = nn.Sequential(*classifier) 45 | pretrained_model.fc = self.classifier 46 | 47 | def forward(self, x): 48 | return F.sigmoid(self.pretrained_model(x)) 49 | 50 | return MyModel(torchvision.models.resnet34(pretrained=True)) 51 | 52 | 53 | random_state = 1 54 | labels_df = labels.get_labels_df() 55 | unsupervised_dataframe = pd.read_csv(paths.submissions + 'SOTA') 56 | 57 | kf = sklearn.model_selection.KFold(n_splits=5, shuffle=True, random_state=random_state) 58 | kf_unsupervised = sklearn.model_selection.KFold(n_splits=5, shuffle=True, random_state=654) 59 | 60 | split = supervised_split = kf.split(labels_df) 61 | unsupervised_split = kf_unsupervised.split(unsupervised_dataframe) 62 | 63 | 64 | def train_net(train, val, unsupervised, model, name): 65 | unsupervised_initialization = mlb.transform(unsupervised['tags'].str.split()).astype(np.float32) 66 | unsupervised_samples = unsupervised['image_name'].as_matrix() 67 | 68 | unsupervised_initialization = unsupervised_initialization[:len(unsupervised_initialization)//2*3] 69 | unsupervised_samples = unsupervised_samples[:len(unsupervised_samples)//2*3] 70 | 71 | transformations_train = transforms.apply_chain([ 72 | transforms.random_fliplr(), 73 | transforms.random_flipud(), 74 | transforms.augment(), 75 | torchvision.transforms.ToTensor() 76 | ]) 77 | 78 | transformations_val = transforms.apply_chain([ 79 | torchvision.transforms.ToTensor() 80 | ]) 81 | 82 | dset_train_unsupervised = KaggleAmazonUnsupervisedDataset( 83 | unsupervised_samples, 84 | paths.test_jpg, 85 | '.jpg', 86 | transformations_train, 87 | transformations_val, 88 | unsupervised_initialization 89 | ) 90 | 91 | dset_train_supervised = KaggleAmazonJPGDataset(train, paths.train_jpg, transformations_train, divide=False) 92 | dset_train = KaggleAmazonSemiSupervisedDataset(dset_train_supervised, dset_train_unsupervised, None, indices=False) 93 | 94 | train_loader = DataLoader(dset_train, 95 | batch_size=64, 96 | shuffle=True, 97 | num_workers=10, 98 | pin_memory=True) 99 | 100 | dset_val = KaggleAmazonJPGDataset(val, paths.train_jpg, transformations_val, divide=False) 101 | val_loader = DataLoader(dset_val, 102 | batch_size=64, 103 | num_workers=10, 104 | pin_memory=True) 105 | 106 | ignored_params = list(map(id, chain( 107 | model.classifier.parameters(), 108 | model.layer1.parameters(), 109 | model.layer2.parameters(), 110 | model.layer3.parameters(), 111 | model.layer4.parameters() 112 | ))) 113 | base_params = filter(lambda p: id(p) not in ignored_params, 114 | model.parameters()) 115 | 116 | optimizer = optim.Adam([ 117 | {'params': base_params}, 118 | {'params': model.layer1.parameters()}, 119 | {'params': model.layer2.parameters()}, 120 | {'params': model.layer3.parameters()}, 121 | {'params': model.layer4.parameters()}, 122 | {'params': model.classifier.parameters()} 123 | ], lr=0, weight_decay=0.0001) 124 | 125 | trainer = ModuleTrainer(model) 126 | 127 | def schedule(current_epoch, current_lrs, **logs): 128 | lrs = [1e-3, 1e-4, 0.5e-4, 1e-5, 0.5e-5] 129 | epochs = [0, 1, 6, 8, 12] 130 | 131 | for lr, epoch in zip(lrs, epochs): 132 | if current_epoch >= epoch: 133 | current_lrs[5] = lr 134 | if current_epoch >= 2: 135 | current_lrs[4] = lr * 1 136 | current_lrs[3] = lr * 1 137 | current_lrs[2] = lr * 1 138 | current_lrs[1] = lr * 1 139 | current_lrs[0] = lr * 0.1 140 | 141 | return current_lrs 142 | 143 | trainer.set_callbacks([ 144 | ModelCheckpoint( 145 | paths.models, 146 | name, 147 | save_best_only=False, 148 | saving_strategy=lambda epoch: True 149 | ), 150 | CSVLogger(paths.logs + name), 151 | LearningRateScheduler(schedule), 152 | SemiSupervisedUpdater(trainer, dset_train_unsupervised, start_epoch=6, momentum=0.25) 153 | ]) 154 | 155 | trainer.compile(loss=nn.BCELoss(), 156 | optimizer=optimizer) 157 | 158 | trainer.fit_loader(train_loader, 159 | val_loader, 160 | nb_epoch=16, 161 | verbose=1, 162 | cuda_device=0) 163 | 164 | if __name__ == "__main__": 165 | for i, ((train_idx, val_idx), (train_idx_unsupervised, val_idx_unsupervised)) in enumerate(zip(supervised_split, unsupervised_split)): 166 | name = os.path.basename(sys.argv[0])[:-3] + '-split_' + str(i) 167 | train_net( 168 | labels_df.ix[train_idx], 169 | labels_df.ix[val_idx], 170 | unsupervised_dataframe.ix[train_idx_unsupervised][::4], 171 | generate_model(), 172 | name 173 | ) 174 | -------------------------------------------------------------------------------- /ModuleTrainer.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from __future__ import absolute_import 3 | 4 | from torch.autograd import Variable 5 | 6 | from torchsample.modules import ModuleTrainer 7 | 8 | import math 9 | 10 | # local imports 11 | from torchsample.modules._utils import (_get_current_time, _nb_function_args) 12 | from torchsample.callbacks import CallbackModule, History, TQDM 13 | from torchsample.constraints import ConstraintModule 14 | from torchsample.initializers import InitializerModule 15 | from torchsample.metrics import MetricsModule 16 | from torchsample.regularizers import RegularizerModule 17 | 18 | 19 | class SemisupervisedModuleTrainer(ModuleTrainer): 20 | def fit_loader(self, 21 | loader, 22 | val_loader=None, 23 | nb_epoch=100, 24 | cuda_device=-1, 25 | metrics=None, 26 | verbose=1, 27 | custom_loss=None 28 | ): 29 | 30 | # store whether validation data was given 31 | if val_loader is None: 32 | has_validation_data = False 33 | else: 34 | has_validation_data = True 35 | 36 | # create regularizers 37 | if hasattr(self.model, 'regularizers'): 38 | for reg in self.model.regularizers: 39 | self.add_regularizer(reg) 40 | if self._has_regularizers: 41 | regularizers = RegularizerModule(self._regularizers) 42 | 43 | # create constraints 44 | if hasattr(self.model, 'constraints'): 45 | for constraint in self.model.constraints: 46 | self.add_constraint(constraint) 47 | if self._has_constraints: 48 | constraints = ConstraintModule(self._constraints) 49 | constraints.set_model(self.model) 50 | 51 | # create metrics 52 | if hasattr(self.model, 'metrics'): 53 | for metric in self.model.metrics: 54 | self.add_metric(metric) 55 | if self._has_metrics: 56 | metrics = MetricsModule(self._metrics) 57 | 58 | # create initializers 59 | if hasattr(self.model, 'initializers'): 60 | for initializer in self.model.initializers: 61 | self.add_initializer(initializer) 62 | if self._has_initializers: 63 | initializers = InitializerModule(self._initializers) 64 | initializers(self.model) 65 | 66 | if cuda_device > -1: 67 | self.model.cuda(cuda_device) 68 | 69 | # enter context-manager for progress bar 70 | with TQDM() as pbar: 71 | # create callbacks 72 | progressbar = [] 73 | # add progress bar if necessary 74 | if verbose > 0: 75 | progressbar = [pbar] 76 | callbacks = CallbackModule(self._callbacks + progressbar) 77 | callbacks.set_model(self) 78 | 79 | train_begin_logs = { 80 | 'start_time': _get_current_time(), 81 | 'has_validation_data': has_validation_data 82 | } 83 | callbacks.on_train_begin(logs=train_begin_logs) 84 | 85 | # calculate total number of batches 86 | nb_batches = int(math.ceil(len(loader.dataset) / loader.batch_size)) 87 | 88 | # loop through each epoch 89 | for epoch_idx in range(nb_epoch): 90 | epoch_logs = { 91 | 'nb_batches': nb_batches, 92 | 'nb_epoch': nb_epoch, 93 | 'has_validation_data': has_validation_data 94 | } 95 | callbacks.on_epoch_begin(epoch_idx, epoch_logs) 96 | 97 | for batch_idx, data, in enumerate(loader): 98 | batch_logs = {'batch_idx': batch_idx} 99 | callbacks.on_batch_begin(batch_idx, batch_logs) 100 | data = [Variable(ins) for ins in data] 101 | if cuda_device > -1: 102 | data = tuple([ins.cuda(cuda_device) for ins in data]) 103 | 104 | batch_logs['batch_samples'] = len(data[0]) 105 | 106 | x, y, i = data 107 | # zero grads and forward pass 108 | self._optimizer.zero_grad() 109 | outputs = self.model(x) 110 | loss = custom_loss(outputs, y, i, epoch_idx) 111 | 112 | batch_logs['loss'] = loss.data[0] 113 | 114 | # calculate custom/special batch metrics if necessary 115 | if self._has_metrics: 116 | metric_logs = metrics(data) 117 | batch_logs.update(metric_logs) 118 | 119 | # backward pass and optimizer step 120 | loss.backward() 121 | self._optimizer.step() 122 | 123 | callbacks.on_batch_end(batch_idx, batch_logs) 124 | 125 | # apply explicit constraints if necessary 126 | if self._has_constraints: 127 | constraints.on_batch_end(batch_idx) 128 | 129 | if has_validation_data: 130 | val_loss = self.evaluate_loader(val_loader, 131 | cuda_device=cuda_device) 132 | if self._has_metrics: 133 | val_loss, val_metric_logs = val_loss 134 | epoch_logs.update(val_metric_logs) 135 | epoch_logs['val_loss'] = val_loss 136 | self.history.batch_metrics['val_loss'] = val_loss 137 | 138 | # END OF EPOCH 139 | epoch_logs.update(self.history.batch_metrics) 140 | if self._has_metrics: 141 | epoch_logs.update(metric_logs) 142 | 143 | callbacks.on_epoch_end(epoch_idx, epoch_logs) 144 | 145 | # apply Epoch-level constraints if necessary 146 | if self._has_constraints: 147 | constraints.on_epoch_end(epoch_idx) 148 | # reset all metric counters 149 | if self._has_metrics: 150 | metrics.reset() 151 | # exit the training loop if necessary (e.g. EarlyStopping) 152 | if self._stop_training: 153 | break 154 | 155 | train_logs = { 156 | 'final_loss': self.history.losses[-1], 157 | 'best_loss': min(self.history.losses), 158 | 'end_time': _get_current_time() 159 | } 160 | if has_validation_data: 161 | train_logs['final_val_loss'] = self.history.val_losses[-1] 162 | train_logs['best_val_loss'] = min(self.history.val_losses) 163 | 164 | callbacks.on_train_end(logs=train_logs) 165 | -------------------------------------------------------------------------------- /nn_semisupervised_densenet_121.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | from itertools import chain 5 | 6 | import numpy as np 7 | import pandas as pd 8 | 9 | import torchvision.models 10 | import torch.nn.functional as F 11 | import torch.optim as optim 12 | from torch import nn 13 | from torch.utils.data import DataLoader 14 | 15 | from torchsample.callbacks import CSVLogger, LearningRateScheduler 16 | from callbacks import ModelCheckpoint, SemiSupervisedUpdater 17 | 18 | import sklearn.model_selection 19 | 20 | import paths 21 | import labels 22 | import transforms 23 | from datasets import KaggleAmazonUnsupervisedDataset, KaggleAmazonSemiSupervisedDataset, KaggleAmazonJPGDataset, mlb 24 | from ModuleTrainer import ModuleTrainer 25 | 26 | name = os.path.basename(sys.argv[0])[:-3] 27 | 28 | 29 | def generate_model(): 30 | class DenseModel(nn.Module): 31 | def __init__(self, pretrained_model): 32 | super(DenseModel, self).__init__() 33 | self.classifier = nn.Linear(pretrained_model.classifier.in_features, 17) 34 | 35 | for m in self.modules(): 36 | if isinstance(m, nn.Conv2d): 37 | nn.init.kaiming_normal(m.weight) 38 | elif isinstance(m, nn.BatchNorm2d): 39 | m.weight.data.fill_(1) 40 | m.bias.data.zero_() 41 | elif isinstance(m, nn.Linear): 42 | m.bias.data.zero_() 43 | 44 | self.features = pretrained_model.features 45 | self.layer1 = pretrained_model.features._modules['denseblock1'] 46 | self.layer2 = pretrained_model.features._modules['denseblock2'] 47 | self.layer3 = pretrained_model.features._modules['denseblock3'] 48 | self.layer4 = pretrained_model.features._modules['denseblock4'] 49 | 50 | def forward(self, x): 51 | features = self.features(x) 52 | out = F.relu(features, inplace=True) 53 | out = F.avg_pool2d(out, kernel_size=8).view(features.size(0), -1) 54 | out = F.sigmoid(self.classifier(out)) 55 | return out 56 | 57 | return DenseModel(torchvision.models.densenet121(pretrained=True)) 58 | 59 | 60 | random_state = 1 61 | labels_df = labels.get_labels_df() 62 | unsupervised_dataframe = pd.read_csv(paths.submissions + 'SOTA') 63 | 64 | kf = sklearn.model_selection.KFold(n_splits=5, shuffle=True, random_state=random_state) 65 | kf_unsupervised = sklearn.model_selection.KFold(n_splits=5, shuffle=True, random_state=2345) 66 | split = supervised_split = kf.split(labels_df) 67 | unsupervised_split = kf_unsupervised.split(unsupervised_dataframe) 68 | 69 | 70 | def train_net(train, val, unsupervised, model, name): 71 | unsupervised_initialization = mlb.transform(unsupervised['tags'].str.split()).astype(np.float32) 72 | unsupervised_samples = unsupervised['image_name'].as_matrix() 73 | 74 | unsupervised_initialization = unsupervised_initialization[:len(unsupervised_initialization)//2*3] 75 | unsupervised_samples = unsupervised_samples[:len(unsupervised_samples)//2*3] 76 | 77 | transformations_train = transforms.apply_chain([ 78 | transforms.random_fliplr(), 79 | transforms.random_flipud(), 80 | transforms.augment(), 81 | torchvision.transforms.ToTensor() 82 | ]) 83 | 84 | transformations_val = transforms.apply_chain([ 85 | torchvision.transforms.ToTensor() 86 | ]) 87 | 88 | dset_train_unsupervised = KaggleAmazonUnsupervisedDataset( 89 | unsupervised_samples, 90 | paths.test_jpg, 91 | '.jpg', 92 | transformations_train, 93 | transformations_val, 94 | unsupervised_initialization 95 | ) 96 | 97 | dset_train_supervised = KaggleAmazonJPGDataset(train, paths.train_jpg, transformations_train, divide=False) 98 | dset_train = KaggleAmazonSemiSupervisedDataset(dset_train_supervised, dset_train_unsupervised, None, indices=False) 99 | 100 | train_loader = DataLoader(dset_train, 101 | batch_size=32, 102 | shuffle=True, 103 | num_workers=10, 104 | pin_memory=True) 105 | 106 | dset_val = KaggleAmazonJPGDataset(val, paths.train_jpg, transformations_val, divide=False) 107 | val_loader = DataLoader(dset_val, 108 | batch_size=32, 109 | num_workers=10, 110 | pin_memory=True) 111 | 112 | ignored_params = list(map(id, chain( 113 | model.classifier.parameters(), 114 | model.layer1.parameters(), 115 | model.layer2.parameters(), 116 | model.layer3.parameters(), 117 | model.layer4.parameters() 118 | ))) 119 | base_params = filter(lambda p: id(p) not in ignored_params, 120 | model.parameters()) 121 | 122 | optimizer = optim.Adam([ 123 | {'params': base_params}, 124 | {'params': model.layer1.parameters()}, 125 | {'params': model.layer2.parameters()}, 126 | {'params': model.layer3.parameters()}, 127 | {'params': model.layer4.parameters()}, 128 | {'params': model.classifier.parameters()} 129 | ], lr=0, weight_decay=0.0001) 130 | 131 | trainer = ModuleTrainer(model) 132 | 133 | def schedule(current_epoch, current_lrs, **logs): 134 | lrs = [1e-3, 1e-4, 0.5e-4, 1e-5, 0.5e-5] 135 | epochs = [0, 1, 6, 8, 12] 136 | 137 | for lr, epoch in zip(lrs, epochs): 138 | if current_epoch >= epoch: 139 | current_lrs[5] = lr 140 | if current_epoch >= 2: 141 | current_lrs[4] = lr * 1 142 | current_lrs[3] = lr * 1 143 | current_lrs[2] = lr * 1 144 | current_lrs[1] = lr * 1 145 | current_lrs[0] = lr * 0.1 146 | 147 | return current_lrs 148 | 149 | trainer.set_callbacks([ 150 | ModelCheckpoint( 151 | paths.models, 152 | name, 153 | save_best_only=False, 154 | saving_strategy=lambda epoch: True 155 | ), 156 | CSVLogger(paths.logs + name), 157 | LearningRateScheduler(schedule), 158 | SemiSupervisedUpdater(trainer, dset_train_unsupervised, start_epoch=6, momentum=0.25) 159 | ]) 160 | 161 | trainer.compile(loss=nn.BCELoss(), 162 | optimizer=optimizer) 163 | 164 | trainer.fit_loader(train_loader, 165 | val_loader, 166 | nb_epoch=16, 167 | verbose=1, 168 | cuda_device=0) 169 | 170 | if __name__ == "__main__": 171 | for i, ((train_idx, val_idx), (train_idx_unsupervised, val_idx_unsupervised)) in enumerate(zip(supervised_split, unsupervised_split)): 172 | name = os.path.basename(sys.argv[0])[:-3] + '-split_' + str(i) 173 | train_net( 174 | labels_df.ix[train_idx], 175 | labels_df.ix[val_idx], 176 | unsupervised_dataframe.ix[train_idx_unsupervised][::3], 177 | generate_model(), 178 | name 179 | ) 180 | -------------------------------------------------------------------------------- /nn_semisupervised_densenet_169.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | from itertools import chain 5 | 6 | import numpy as np 7 | import pandas as pd 8 | 9 | import torchvision.models 10 | import torch.nn.functional as F 11 | import torch.optim as optim 12 | from torch import nn 13 | from torch.utils.data import DataLoader 14 | 15 | from torchsample.callbacks import CSVLogger, LearningRateScheduler 16 | from callbacks import ModelCheckpoint, SemiSupervisedUpdater 17 | 18 | import sklearn.model_selection 19 | 20 | import paths 21 | import labels 22 | import transforms 23 | from datasets import KaggleAmazonUnsupervisedDataset, KaggleAmazonSemiSupervisedDataset, KaggleAmazonJPGDataset, mlb 24 | from ModuleTrainer import ModuleTrainer 25 | 26 | name = os.path.basename(sys.argv[0])[:-3] 27 | 28 | 29 | def generate_model(): 30 | class DenseModel(nn.Module): 31 | def __init__(self, pretrained_model): 32 | super(DenseModel, self).__init__() 33 | self.classifier = nn.Linear(pretrained_model.classifier.in_features, 17) 34 | 35 | for m in self.modules(): 36 | if isinstance(m, nn.Conv2d): 37 | nn.init.kaiming_normal(m.weight) 38 | elif isinstance(m, nn.BatchNorm2d): 39 | m.weight.data.fill_(1) 40 | m.bias.data.zero_() 41 | elif isinstance(m, nn.Linear): 42 | m.bias.data.zero_() 43 | 44 | self.features = pretrained_model.features 45 | self.layer1 = pretrained_model.features._modules['denseblock1'] 46 | self.layer2 = pretrained_model.features._modules['denseblock2'] 47 | self.layer3 = pretrained_model.features._modules['denseblock3'] 48 | self.layer4 = pretrained_model.features._modules['denseblock4'] 49 | 50 | def forward(self, x): 51 | features = self.features(x) 52 | out = F.relu(features, inplace=True) 53 | out = F.avg_pool2d(out, kernel_size=8).view(features.size(0), -1) 54 | out = F.sigmoid(self.classifier(out)) 55 | return out 56 | 57 | return DenseModel(torchvision.models.densenet169(pretrained=True)) 58 | 59 | 60 | random_state = 1 61 | labels_df = labels.get_labels_df() 62 | unsupervised_dataframe = pd.read_csv(paths.submissions + 'SOTA') 63 | 64 | kf = sklearn.model_selection.KFold(n_splits=5, shuffle=True, random_state=random_state) 65 | kf_unsupervised = sklearn.model_selection.KFold(n_splits=5, shuffle=True, random_state=879) 66 | split = supervised_split = kf.split(labels_df) 67 | unsupervised_split = kf_unsupervised.split(unsupervised_dataframe) 68 | 69 | 70 | def train_net(train, val, unsupervised, model, name): 71 | unsupervised_initialization = mlb.transform(unsupervised['tags'].str.split()).astype(np.float32) 72 | unsupervised_samples = unsupervised['image_name'].as_matrix() 73 | 74 | unsupervised_initialization = unsupervised_initialization[:len(unsupervised_initialization)//2*3] 75 | unsupervised_samples = unsupervised_samples[:len(unsupervised_samples)//2*3] 76 | 77 | transformations_train = transforms.apply_chain([ 78 | transforms.random_fliplr(), 79 | transforms.random_flipud(), 80 | transforms.augment(), 81 | torchvision.transforms.ToTensor() 82 | ]) 83 | 84 | transformations_val = transforms.apply_chain([ 85 | torchvision.transforms.ToTensor() 86 | ]) 87 | 88 | dset_train_unsupervised = KaggleAmazonUnsupervisedDataset( 89 | unsupervised_samples, 90 | paths.test_jpg, 91 | '.jpg', 92 | transformations_train, 93 | transformations_val, 94 | unsupervised_initialization 95 | ) 96 | 97 | dset_train_supervised = KaggleAmazonJPGDataset(train, paths.train_jpg, transformations_train, divide=False) 98 | dset_train = KaggleAmazonSemiSupervisedDataset(dset_train_supervised, dset_train_unsupervised, None, indices=False) 99 | 100 | train_loader = DataLoader(dset_train, 101 | batch_size=32, 102 | shuffle=True, 103 | num_workers=10, 104 | pin_memory=True) 105 | 106 | dset_val = KaggleAmazonJPGDataset(val, paths.train_jpg, transformations_val, divide=False) 107 | val_loader = DataLoader(dset_val, 108 | batch_size=32, 109 | num_workers=10, 110 | pin_memory=True) 111 | 112 | ignored_params = list(map(id, chain( 113 | model.classifier.parameters(), 114 | model.layer1.parameters(), 115 | model.layer2.parameters(), 116 | model.layer3.parameters(), 117 | model.layer4.parameters() 118 | ))) 119 | base_params = filter(lambda p: id(p) not in ignored_params, 120 | model.parameters()) 121 | 122 | optimizer = optim.Adam([ 123 | {'params': base_params}, 124 | {'params': model.layer1.parameters()}, 125 | {'params': model.layer2.parameters()}, 126 | {'params': model.layer3.parameters()}, 127 | {'params': model.layer4.parameters()}, 128 | {'params': model.classifier.parameters()} 129 | ], lr=0, weight_decay=0.0001) 130 | 131 | trainer = ModuleTrainer(model) 132 | 133 | def schedule(current_epoch, current_lrs, **logs): 134 | lrs = [1e-3, 1e-4, 0.5e-4, 1e-5, 0.5e-5] 135 | epochs = [0, 1, 6, 8, 12] 136 | 137 | for lr, epoch in zip(lrs, epochs): 138 | if current_epoch >= epoch: 139 | current_lrs[5] = lr 140 | if current_epoch >= 2: 141 | current_lrs[4] = lr * 1 142 | current_lrs[3] = lr * 1 143 | current_lrs[2] = lr * 1 144 | current_lrs[1] = lr * 1 145 | current_lrs[0] = lr * 0.1 146 | 147 | return current_lrs 148 | 149 | trainer.set_callbacks([ 150 | ModelCheckpoint( 151 | paths.models, 152 | name, 153 | save_best_only=False, 154 | saving_strategy=lambda epoch: True 155 | ), 156 | CSVLogger(paths.logs + name), 157 | LearningRateScheduler(schedule), 158 | SemiSupervisedUpdater(trainer, dset_train_unsupervised, start_epoch=6, momentum=0.25) 159 | ]) 160 | 161 | trainer.compile(loss=nn.BCELoss(), 162 | optimizer=optimizer) 163 | 164 | trainer.fit_loader(train_loader, 165 | val_loader, 166 | nb_epoch=16, 167 | verbose=1, 168 | cuda_device=0) 169 | 170 | if __name__ == "__main__": 171 | for i, ((train_idx, val_idx), (train_idx_unsupervised, val_idx_unsupervised)) in enumerate(zip(supervised_split, unsupervised_split)): 172 | name = os.path.basename(sys.argv[0])[:-3] + '-split_' + str(i) 173 | train_net( 174 | labels_df.ix[train_idx], 175 | labels_df.ix[val_idx], 176 | unsupervised_dataframe.ix[train_idx_unsupervised][::4], 177 | generate_model(), 178 | name 179 | ) 180 | -------------------------------------------------------------------------------- /transforms.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from skimage.transform import AffineTransform, SimilarityTransform, PolynomialTransform, warp 3 | from skimage.filters import gaussian 4 | from skimage import exposure, img_as_float 5 | from scipy.misc import imresize 6 | import Augmentor 7 | 8 | center_shift = 256 / 2 9 | tf_center = SimilarityTransform(translation=-center_shift) 10 | tf_uncenter = SimilarityTransform(translation=center_shift) 11 | 12 | 13 | def apply_chain(chain): 14 | def call(x): 15 | 16 | for fn in chain: 17 | x = fn(x) 18 | 19 | return x 20 | 21 | return call 22 | 23 | 24 | def resize(output_size): 25 | def call(x): 26 | x = imresize(x, output_size) # x[::256//output_size, ::256//output_size, :] 27 | return x 28 | 29 | return call 30 | 31 | 32 | def random_crop(size): 33 | def call(x): 34 | height, width, d = x.shape 35 | 36 | r1 = np.random.random() 37 | r2 = np.random.random() 38 | 39 | left = round(r1 * (width - size)) 40 | right = round((1 - r1) * (width - size)) 41 | 42 | top = round(r2 * (height - size)) 43 | bottom = round((1 - r2) * (height - size)) 44 | 45 | crop = x[top:height-bottom, left:width-right, :] 46 | 47 | return crop 48 | 49 | return call 50 | 51 | 52 | def center_crop(size): 53 | def call(x): 54 | height, width, d = x.shape 55 | a = int(0.5 * (height - size)) 56 | b = int(0.5 * (width - size)) 57 | 58 | top, bottom = a, (height - size) - a 59 | left, right = b, (height - size) - b 60 | 61 | crop = x[top:height-bottom, left:width-right, :] 62 | 63 | return crop 64 | 65 | return call 66 | 67 | 68 | def crop_top_left(size): 69 | def call(x): 70 | crop = x[:size, :size, :] 71 | return crop 72 | 73 | return call 74 | 75 | 76 | def crop_top_right(size): 77 | def call(x): 78 | crop = x[:size, x.shape[1] - size:, :] 79 | return crop 80 | 81 | return call 82 | 83 | 84 | def crop_bottom_left(size): 85 | def call(x): 86 | crop = x[x.shape[0] - size:, :size, :] 87 | return crop 88 | 89 | return call 90 | 91 | 92 | def crop_bottom_right(size): 93 | def call(x): 94 | crop = x[x.shape[0] - size:, x.shape[1] - size:, :] 95 | return crop 96 | 97 | return call 98 | 99 | 100 | def image_to_array(x): 101 | return np.array(x) 102 | 103 | 104 | def to_float(x): 105 | return img_as_float(x) 106 | 107 | 108 | def blur(sigma=0.1): 109 | def call(x): 110 | x = gaussian(x, sigma=sigma, preserve_range=True, multichannel=True) 111 | return x 112 | return call 113 | 114 | 115 | def random_blur(sigma=lambda: np.random.random_sample()*1): 116 | def call(x): 117 | x = gaussian(x, sigma=sigma(), preserve_range=True, multichannel=True) 118 | return x 119 | return call 120 | 121 | 122 | def random_gamma(gamma=lambda: np.random.rand() * 0.4 + 0.8): 123 | def call(x): 124 | return exposure.adjust_gamma(x, gamma()) 125 | 126 | return call 127 | 128 | 129 | def random_contrast(weight=lambda: np.random.rand() * 0.3 + 0.7): 130 | def call(x): 131 | w = weight() 132 | return x * w + (1 - w) * exposure.rescale_intensity(x) 133 | 134 | return call 135 | 136 | 137 | def augment_color(weight=0.1): 138 | def call(x): 139 | height, width, channels = x.shape 140 | 141 | img_rgb_col = x.reshape(height*width, channels) 142 | cov = np.cov(img_rgb_col.T) 143 | eigvals, eigvects = np.linalg.eigh(cov) 144 | random_eigvals = np.sqrt(eigvals) * np.random.randn(channels) * weight 145 | scaled_eigvects = np.dot(eigvects, random_eigvals) 146 | x = np.clip(x + scaled_eigvects, 0, 1) 147 | 148 | return x 149 | 150 | return call 151 | 152 | 153 | def augment_color_deterministic(weight=0.1): 154 | def call(x): 155 | height, width, channels = x.shape 156 | 157 | img_rgb_col = x.reshape(height*width, channels) 158 | cov = np.cov(img_rgb_col.T) 159 | eigvals, eigvects = np.linalg.eigh(cov) 160 | random_eigvals = np.sqrt(eigvals) * np.array([1, 1, 1]) * weight 161 | scaled_eigvects = np.dot(eigvects, random_eigvals) 162 | x = np.clip(x + scaled_eigvects, 0, 1) 163 | 164 | return x 165 | 166 | return call 167 | 168 | 169 | def distort(): 170 | p = Augmentor.Pipeline() 171 | p.random_distortion(probability=1, grid_width=5, grid_height=5, magnitude=8) 172 | 173 | def call(x): 174 | x = p.sample_with_array(x.astype('uint8'), False) 175 | return x 176 | 177 | return call 178 | 179 | 180 | def random_zoom_range(zoom_range=[1/1.2, 1.2]): 181 | def call(): 182 | #https://github.com/benanne/kaggle-galaxies/blob/master/realtime_augmentation.py#L147 183 | #log_zoom_range = [np.log(z) for z in zoom_range] 184 | #zoom = np.exp(np.random.uniform(*log_zoom_range)) 185 | 186 | zoom = np.random.uniform(zoom_range[0], zoom_range[1]) 187 | 188 | return zoom, zoom 189 | 190 | return call 191 | 192 | 193 | def augment( 194 | rotation_fn=lambda: np.random.random_integers(0, 360), 195 | translation_fn=lambda: (np.random.random_integers(-20, 20), np.random.random_integers(-20, 20)), 196 | scale_factor_fn=random_zoom_range(), 197 | shear_fn=lambda: np.random.random_integers(-10, 10) 198 | ): 199 | def call(x): 200 | rotation = rotation_fn() 201 | translation = translation_fn() 202 | scale = scale_factor_fn() 203 | shear = shear_fn() 204 | 205 | tf_augment = AffineTransform(scale=scale, rotation=np.deg2rad(rotation), translation=translation, shear=np.deg2rad(shear)) 206 | tf = tf_center + tf_augment + tf_uncenter 207 | 208 | x = warp(x, tf, order=1, preserve_range=True, mode='symmetric') 209 | 210 | return x 211 | 212 | return call 213 | 214 | 215 | def rotate_90(k=1): 216 | def call(x): 217 | x = np.rot90(x, k).copy() 218 | return x 219 | 220 | return call 221 | 222 | 223 | def fliplr(): 224 | def call(x): 225 | x = np.fliplr(x).copy() 226 | return x 227 | 228 | return call 229 | 230 | 231 | def flipud(): 232 | def call(x): 233 | x = np.flipud(x).copy() 234 | return x 235 | 236 | return call 237 | 238 | 239 | def random_fliplr(): 240 | def call(x): 241 | if np.random.randint(2) > 0: 242 | x = np.fliplr(x).copy() 243 | return x 244 | 245 | return call 246 | 247 | 248 | def random_flipud(): 249 | def call(x): 250 | if np.random.randint(2) > 0: 251 | x = np.flipud(x).copy() 252 | return x 253 | 254 | return call 255 | 256 | 257 | def augment_deterministic( 258 | rotation=0, 259 | translation=0, 260 | scale_factor=1, 261 | shear=0 262 | ): 263 | def call(x): 264 | scale = scale_factor, scale_factor 265 | rotation_tmp = rotation 266 | 267 | tf_augment = AffineTransform( 268 | scale=scale, 269 | rotation=np.deg2rad(rotation_tmp), 270 | translation=translation, 271 | shear=np.deg2rad(shear) 272 | ) 273 | tf = tf_center + tf_augment + tf_uncenter 274 | 275 | x = warp(x, tf, order=1, preserve_range=True, mode='symmetric') 276 | 277 | return x 278 | 279 | return call -------------------------------------------------------------------------------- /model_tta_hyperopt.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | import numpy as np 5 | 6 | from sklearn.model_selection import KFold 7 | from sklearn.metrics import accuracy_score 8 | from xgboost import XGBClassifier 9 | from hyperopt import hp, fmin, tpe, STATUS_OK, Trials 10 | 11 | import paths 12 | import labels 13 | from datasets import mlb 14 | from find_best_threshold import fbeta, optimise_f2_thresholds_fast, optimise_f2_thresholds 15 | 16 | models = [ 17 | 'nn_semisupervised_densenet_121', 18 | ] 19 | 20 | for model_name in models: 21 | try: 22 | random_state = 1 23 | labels_df = labels.get_labels_df() 24 | kf = KFold(n_splits=5, shuffle=True, random_state=random_state) 25 | split = list(kf.split(labels_df)) 26 | 27 | 28 | x_train_all = [] 29 | x_val_all = [] 30 | x_test_all = [] 31 | labels_train_all = [] 32 | labels_val_all = [] 33 | thresholds_all = [] 34 | 35 | for i, (labels_train, labels_val) in enumerate(split): 36 | net = np.load(paths.predictions + '{}-split_{}.npz'.format(model_name, i)) 37 | thresholds = np.load(paths.thresholds + '{}-split_{}.npy'.format(model_name, i)) 38 | 39 | thresholds_all.append(np.average(thresholds, axis=1)) 40 | train = net['train'] 41 | val = net['val'] 42 | test = net['test'] 43 | 44 | x_train_all.append(train) 45 | x_val_all.append(val) 46 | x_test_all.append(test) 47 | 48 | labels_train = mlb.transform(labels_df.ix[labels_train]['tags'].str.split()).astype(np.float32) 49 | labels_val = mlb.transform(labels_df.ix[labels_val]['tags'].str.split()).astype(np.float32) 50 | 51 | labels_train_all.append(labels_train) 52 | labels_val_all.append(labels_val) 53 | 54 | train = x_train_all = np.concatenate(x_train_all, axis=1) 55 | val = x_val_all = np.concatenate(x_val_all, axis=1) 56 | # Test gets stacked over folds instead of concat 57 | test = x_test_all = np.stack(x_test_all, axis=0) 58 | 59 | # Thresholds get averaged as comparison 60 | thresholds_average = np.average(np.stack(thresholds_all, axis=0), axis=0) 61 | 62 | # Labels get concatenated for right order 63 | labels_train = labels_train_all = np.concatenate(labels_train_all, axis=0) 64 | labels_val = labels_val_all = np.concatenate(labels_val_all, axis=0) 65 | 66 | 67 | p_reference_average = [] 68 | p_val = [] 69 | p_val_hard = [] 70 | p_test = [] 71 | for i in range(17): 72 | x_train = train[:, :, i] 73 | x_train = np.rollaxis(x_train, 1) 74 | y_train = labels_train[:, i] 75 | 76 | x_val = val[:, :, i] 77 | x_val = np.rollaxis(x_val, 1) 78 | y_val = labels_val[:, i] 79 | 80 | x_test = test[:, :, :, i] 81 | x_test = np.rollaxis(x_test, 2, 1) 82 | 83 | x_val_avg = np.average(np.copy(x_val), axis=1) 84 | x_val_avg[x_val_avg >= thresholds_average[i]] = 1 85 | x_val_avg[x_val_avg < thresholds_average[i]] = 0 86 | 87 | n_estimators = 1000 88 | max_evals = 15 89 | 90 | def objective(space): 91 | estimator = XGBClassifier( 92 | n_estimators=n_estimators, 93 | max_depth=int(space['max_depth']), 94 | min_child_weight=int(space['min_child_weight']), 95 | gamma=space['gamma'], 96 | subsample=space['subsample'], 97 | colsample_bytree=space['colsample_bytree'] 98 | ) 99 | 100 | estimator.fit( 101 | x_train, 102 | y_train, 103 | eval_set=[(x_train, y_train), (x_val, y_val)], 104 | early_stopping_rounds=30, 105 | verbose=False, 106 | eval_metric='error' 107 | ) 108 | 109 | score = accuracy_score(y_val, estimator.predict(x_val)) 110 | 111 | return {'loss': 1 - score, 'status': STATUS_OK} 112 | 113 | space = { 114 | 'max_depth': hp.quniform("max_depth", 3, 20, 1), 115 | 'min_child_weight': hp.quniform('min_child_weight', 1, 10, 1), 116 | 'subsample': hp.uniform('subsample', 0.7, 0.9), 117 | 'colsample_bytree': hp.uniform('colsample_bytree', 0.7, 0.9), 118 | 'gamma': hp.choice('gamma', [0, 0.01, 0.1, 1]) 119 | } 120 | 121 | 122 | trials = Trials() 123 | best = fmin( 124 | fn=objective, 125 | space=space, 126 | algo=tpe.suggest, 127 | max_evals=max_evals, 128 | trials=trials 129 | ) 130 | 131 | # Fit best estimator 132 | estimator = XGBClassifier( 133 | n_estimators=n_estimators*5, 134 | max_depth=int(best['max_depth']), 135 | min_child_weight=int(best['min_child_weight']), 136 | gamma=best['gamma'], 137 | subsample=best['subsample'], 138 | colsample_bytree=best['colsample_bytree'], 139 | ) 140 | 141 | estimator.fit( 142 | x_train, 143 | y_train, 144 | eval_set=[(x_train, y_train), (x_val, y_val)], 145 | early_stopping_rounds=50, 146 | verbose=False, 147 | eval_metric='error' 148 | ) 149 | 150 | print('Feature', i) 151 | print('XGBoost', accuracy_score(y_val, estimator.predict(x_val))) 152 | print('Average', accuracy_score(y_val, x_val_avg)) 153 | p_val.append(estimator.predict_proba(x_val)[:, 1]) 154 | p_val_hard.append(estimator.predict(x_val)) 155 | p_reference_average.append(x_val_avg) 156 | 157 | fold_test = [] 158 | for f in range(5): 159 | fold_test.append(estimator.predict_proba(x_test[f, :, :])[:, 1]) 160 | fold_test = np.average(np.stack(fold_test, axis=0), axis=0) 161 | p_test.append(fold_test) 162 | 163 | 164 | # Stack predictions for each of the 17 targets 165 | # Predictions by XGBoost 166 | p_val = np.stack(p_val, axis=1) 167 | p_test = np.stack(p_test, axis=1) 168 | # Predictions by XGBoost which will not be optimised for f2 169 | p_val_hard = np.stack(p_val_hard, axis=1) 170 | # Predictions from averaging 171 | p_reference_average = np.stack(p_reference_average, axis=1) 172 | 173 | print(p_val.shape) 174 | print(p_val_hard.shape) 175 | print(p_reference_average.shape) 176 | 177 | 178 | print('=======================') 179 | print('fbeta(labels_val, p_reference_average)', fbeta(labels_val, p_reference_average)) 180 | print('fbeta(labels_val, p_val_hard)', fbeta(labels_val, p_val_hard)) 181 | 182 | threshold = optimise_f2_thresholds(labels_val, p_val) 183 | p_val[p_val >= threshold] = 1 184 | p_val[p_val < threshold] = 0 185 | 186 | print('fbeta(labels_val, p_val)', fbeta(labels_val, p_val)) 187 | 188 | p_test[p_test >= threshold] = 1 189 | p_test[p_test < threshold] = 0 190 | 191 | 192 | # WRITE PREDICTIONS 193 | test_images = list(map(lambda path: path[:-len('.jpg')], os.listdir(paths.test_jpg))) 194 | predictions = mlb.inverse_transform(p_test) 195 | test_results = zip(predictions, test_images) 196 | 197 | with open(paths.submissions + '{}_XGB_HYPEROPT'.format(model_name), 'w') as submission: 198 | submission.write('image_name,tags\n') 199 | for tags, target in test_results: 200 | output = target + ',' + ' '.join(tags) 201 | submission.write("%s\n" % output) 202 | 203 | print('Submission ready!') 204 | 205 | except: 206 | print("Unexpected error:", sys.exc_info()[0]) --------------------------------------------------------------------------------