├── README.md ├── view_pred_poly.py ├── simplify_sub.py ├── predict_utils.py ├── nn_predict.py ├── raw_to_mask.py ├── multi_predict.py ├── blend_raw.py ├── b3_gen_prediction.py ├── utils.py ├── LICENSE ├── b3_train.py └── b3_data_iter.py /README.md: -------------------------------------------------------------------------------- 1 | # [kaggle-dstl-satellite-imagery-feature-detection](https://www.kaggle.com/c/dstl-satellite-imagery-feature-detection) 2 | Scripts for the my 6th place solution with [MXNet](https://github.com/dmlc/mxnet). 3 | Key features: 4 | - Unet-like network architecture with multiple input branches(A, M, P channels) 5 | - Adaptive crop sampler, based on the performance of the network on different classes, updated each epoch. 6 | 7 | Brief description: 8 | [b3_data_iter](b3_data_iter.py) - data generation, augmentation 9 | [utils](utils.py) - wkt polygon rasterization, raster polygonization utils 10 | [b3_train](b3_train.py) - network symbol definition, training loop 11 | -------------------------------------------------------------------------------- /view_pred_poly.py: -------------------------------------------------------------------------------- 1 | import mxnet as mx 2 | import numpy as np 3 | import sys, os 4 | import cv2 5 | import time 6 | import pandas as pd 7 | from utils import get_rgb_image, get_scale_factor, get_raster, colorize_raster, rasterize_polgygon 8 | from shapely import wkt, affinity 9 | import multiprocessing 10 | from collections import defaultdict 11 | import csv 12 | 13 | size = 700 14 | version = 'v84' 15 | epoch = 1300 16 | csv.field_size_limit(sys.maxsize) 17 | 18 | 19 | df = pd.read_csv('input/sample_submission.csv') 20 | test_list = df.ImageId.unique() 21 | 22 | d = defaultdict(list) 23 | with open('{}-{}-cv.csv'.format(version, epoch)) as in_file: 24 | reader = csv.reader(in_file) 25 | header = next(reader) 26 | for qq in range(429): 27 | rows = [next(reader) for _ in range(10)] 28 | image_id = rows[0][0] 29 | for i in range(10): 30 | d[image_id].append(rows[i][2]) 31 | print(len(d)) 32 | 33 | 34 | def f(image_id): 35 | # if os.path.exists('test_poly_{}_{}/{}.png'.format(version, epoch, image_id)): 36 | # print(image_id) 37 | # return 38 | print('begin: {}'.format(image_id)) 39 | 40 | p = d[image_id] 41 | p = [wkt.loads(x) for x in p] 42 | y_sf, x_sf = get_scale_factor(image_id, size, size) 43 | p = [affinity.scale(x, xfact=x_sf, yfact=y_sf, origin=(0, 0, 0)) for x in p] 44 | rst = rasterize_polgygon(p, size, size) 45 | color_rst = colorize_raster(rst) 46 | im = get_rgb_image(image_id, size, size) 47 | 48 | rr = np.hstack([color_rst, im]) 49 | cv2.imwrite('test_poly_{}_{}-cv/{}.png'.format(version, epoch, image_id), rr) 50 | print('end: {}'.format(image_id)) 51 | 52 | 53 | try: 54 | os.mkdir('test_poly_{}_{}-cv'.format(version, epoch)) 55 | except: 56 | pass 57 | 58 | 59 | pool = multiprocessing.Pool(8) 60 | pool.imap(f, test_list, chunksize=1) 61 | pool.close() 62 | pool.join() 63 | -------------------------------------------------------------------------------- /simplify_sub.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import print_function 3 | import pandas as pd 4 | from shapely import wkt, affinity, ops 5 | import sys 6 | import csv 7 | import multiprocessing 8 | from shapely.geometry import MultiPolygon, Polygon 9 | import numpy as np 10 | 11 | 12 | def clip_poly(polys): 13 | new_polys = [] 14 | for poly in polys: 15 | x, y = poly.exterior.coords.xy 16 | x = np.clip(x, 0.00001, 0.9999) 17 | y = np.clip(y, -0.9999, 0.09001) 18 | new_polys.append(Polygon(shell=zip(x, y))) 19 | return MultiPolygon(new_polys) 20 | 21 | csv.field_size_limit(sys.maxsize) 22 | 23 | version = 'v61' 24 | epoch = 2620 25 | 26 | shs = [] 27 | id_class = [] 28 | with open('v65-3950-cv-zero12.csv'.format(version, epoch)) as in_file: 29 | reader = csv.reader(in_file) 30 | header = next(reader) 31 | for i, row in enumerate(reader): 32 | shs.append((row[0], row[1], row[2])) 33 | # id_class.append((row[0], row[1])) 34 | if i % 100 == 99: 35 | print(i) 36 | # break 37 | print('csv reading completed') 38 | 39 | 40 | def func(dat): 41 | image_id, class_id, sh = dat 42 | sh = wkt.loads(sh) 43 | # print(len(sh.wkt)) 44 | # print(len(sh.wkt)) 45 | # if not sh.is_valid: 46 | # if not isinstance(sh, MultiPolygon): 47 | # sh = MultiPolygon([sh]) 48 | # sh = clip_poly(sh) 49 | # if not isinstance(sh, MultiPolygon): 50 | # sh = MultiPolygon([sh]) 51 | # sh = clip_poly(sh) 52 | 53 | sh = sh.buffer(0.0000000000001) 54 | sh = sh.simplify(0.00000001, preserve_topology=True) 55 | # sh = MultiPolygon([x.buffer(0) for x in sh]) 56 | 57 | # sh = ops.cascaded_union(sh) 58 | if not sh.is_valid: 59 | print(image_id, class_id) 60 | # qwe 61 | pol = sh.wkt 62 | # pol = wkt.dumps(sh, rounding_precision=8) 63 | return image_id, class_id, pol 64 | 65 | pool = multiprocessing.Pool(40) 66 | shs = pool.map(func, shs) 67 | pool.close() 68 | pool.join() 69 | print('simplified') 70 | 71 | fo = open('v65-3950-cv-zero12-simp.csv'.format(version, epoch), 'w') 72 | print('ImageId,ClassType,MultipolygonWKT', file=fo) 73 | for image_id, class_id, sh in shs: 74 | print('{},{},"{}"'.format(image_id, class_id, sh), file=fo) 75 | -------------------------------------------------------------------------------- /predict_utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import mxnet as mx 3 | import numpy as np 4 | from utils import unsoft 5 | from utils import jaccard_raster 6 | 7 | 8 | def load_model(version, epoch, patch_size, batch_size=8, ctx=mx.gpu()): 9 | sym, arg, aux = mx.model.load_checkpoint('models/' + version, epoch) 10 | mod = mx.module.Module(sym, context=ctx) 11 | mod.bind(data_shapes=[('data', (batch_size, 20, patch_size, patch_size))], 12 | for_training=False) 13 | mod.set_params(arg, aux) 14 | return mod 15 | 16 | 17 | def predict(mod, image_data, patch_size, map_size): 18 | n = map_size / patch_size 19 | 20 | patch_data = [] 21 | for i in range(n): 22 | for j in range(n): 23 | patch_data.append(image_data[patch_size*i: patch_size*(i+1), 24 | patch_size*j: patch_size*(j+1)]) 25 | patch_data = np.array(patch_data) 26 | patch_data = patch_data.transpose((0, 3, 1, 2)) 27 | 28 | data_iter = mx.io.NDArrayIter(data=patch_data, batch_size=8) 29 | preds = mod.predict(data_iter).asnumpy() 30 | 31 | gg = np.zeros((10, map_size, map_size)) 32 | 33 | for i in range(n): # TODO via reshape 34 | for j in range(n): 35 | gg[:, patch_size*i: patch_size*(i+1), 36 | patch_size*j: patch_size*(j+1)] = preds[i*n+j] 37 | preds = gg 38 | return preds 39 | 40 | 41 | def calc_jaccard(mod, X_data, y_mask, patch_size, map_size): 42 | tp, fp, fn = [], [], [] 43 | assert len(X_data) == len(y_mask) 44 | for i in range(len(X_data)): 45 | preds = predict(mod, X_data[i], patch_size, map_size) 46 | # preds = np.argmax(preds, axis=0) 47 | # preds = unsoft(preds) 48 | 49 | thresholds = np.array([0.4, 0.4, 0.4, 0.4, 0.4, 50 | 0.4, 0.4, 0.4, 0.4, 0.4]).reshape((10, 1)) 51 | # for i in range(len(thresholds)): 52 | # preds[i] = (preds[i] > thresholds[i]) 53 | preds = (preds.reshape((10, -1)) > thresholds).reshape((10, map_size, map_size)) 54 | preds = preds.astype(np.uint8) 55 | 56 | pred_raster = preds.transpose((1, 2, 0)) 57 | 58 | score = jaccard_raster(y_mask[i], pred_raster) 59 | tp.append(score[:, 0]) 60 | fp.append(score[:, 1]) 61 | fn.append(score[:, 2]) 62 | 63 | tp = np.stack(tp) 64 | fp = np.stack(fp) 65 | fn = np.stack(fn) 66 | tp = np.mean(tp, axis=0) 67 | fp = np.mean(fp, axis=0) 68 | fn = np.mean(fn, axis=0) 69 | jac = tp / (tp + fp + fn) 70 | jac[np.isnan(jac)] = 0 71 | return jac 72 | -------------------------------------------------------------------------------- /nn_predict.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import pandas as pd 3 | import numpy as np 4 | import mxnet as mx 5 | import cv2 6 | from utils import root_path 7 | from utils import unsoft 8 | from utils import colorize_raster 9 | from data_iter import get_image_data 10 | from utils import polygonize 11 | from shapely import affinity 12 | from utils import get_scale_factor 13 | 14 | df = pd.read_csv(root_path + '/sample_submission.csv') 15 | test_list = df.ImageId.unique() 16 | 17 | #test_list = test_list[:1] 18 | batch_size = 2 19 | g_size = 720 20 | l_size = 180 21 | assert g_size % l_size == 0 22 | h = w = l_size 23 | batch_size = 2 24 | 25 | fo = open('v6-3.csv', 'w') 26 | print('ImageId,ClassType,MultipolygonWKT', file=fo) 27 | 28 | 29 | def predict(mod, image_id, g_size, l_size): 30 | n = g_size / l_size 31 | 32 | image_data = get_image_data(image_id, g_size, g_size) 33 | 34 | patch_data = [] 35 | for i in range(n): 36 | for j in range(n): 37 | patch_data.append(image_data[l_size*i: l_size*(i+1), 38 | l_size*j: l_size*(j+1)]) 39 | patch_data = np.array(patch_data) 40 | patch_data = patch_data.transpose((0, 3, 1, 2)) 41 | 42 | data_iter = mx.io.NDArrayIter(data=patch_data, batch_size=batch_size) 43 | preds = mod.predict(data_iter).asnumpy() 44 | gg = np.zeros((11, g_size, g_size)) 45 | for i in range(n): # TODO via reshape 46 | for j in range(n): 47 | gg[:, l_size*i: l_size*(i+1), 48 | l_size*j: l_size*(j+1)] = preds[i*n+j] 49 | preds = gg 50 | 51 | preds = np.argmax(preds, axis=0) 52 | preds = unsoft(preds) 53 | # pred_raster = preds.transpose((1, 2, 0)) 54 | 55 | assert preds.shape[0] == 10 56 | y_sf, x_sf = get_scale_factor(image_id, g_size, g_size) 57 | y_sf = 1. / y_sf 58 | x_sf = 1. / x_sf 59 | for i in range(10): 60 | sh = polygonize(preds[i]) 61 | sh = affinity.scale(sh, xfact=x_sf, yfact=y_sf, origin=(0, 0, 0)) 62 | print('{},{},"{}"'.format(image_id, i + 1, sh.wkt), file=fo) 63 | 64 | 65 | 66 | # y_pred = colorize_raster(pred_raster) 67 | # cv2.imshow('full pred', y_pred) 68 | # cv2.waitKey() 69 | 70 | sym, arg, aux = mx.model.load_checkpoint('models/v6', 324) 71 | mod = mx.module.Module(sym, context=mx.gpu(0)) 72 | mod.bind(data_shapes=[('data', (batch_size, 20, l_size, l_size))], 73 | for_training=False) 74 | mod.set_params(arg, aux) 75 | 76 | print('model loaded!') 77 | 78 | for i, image_id in enumerate(test_list): 79 | predict(mod, image_id, g_size, l_size) 80 | print(i) 81 | -------------------------------------------------------------------------------- /raw_to_mask.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import print_function 3 | import pandas as pd 4 | import numpy as np 5 | import mxnet as mx 6 | import cv2 7 | import os 8 | import shutil 9 | from sklearn.externals import joblib 10 | from shapely import affinity 11 | import multiprocessing 12 | from shapely.geometry import MultiPolygon, Polygon 13 | from collections import defaultdict 14 | 15 | from utils import polygonize, polygonize_cv, polygonize_sk, get_rgb_image 16 | from utils import get_scale_factor, root_path, colorize_raster, unsoft 17 | 18 | from b3_data_iter import get_data, crop_maps 19 | from b3_data_iter import MultiInputSegDataIter 20 | 21 | n_out = 11 22 | 23 | df = pd.read_csv('input/sample_submission.csv') 24 | test_list = df.ImageId.unique() 25 | 26 | 27 | def mask_to_poly(image_id): 28 | preds = joblib.load('raw_preds/raw_blend5/{}.pkl'.format(image_id)) 29 | size = preds.shape[1] 30 | if n_out == 10: 31 | # preds = (preds > 0.3).astype(np.uint8) 32 | 33 | thresholds = np.array([0.4, 0.4, 0.4, 0.4, 0.8, 34 | 0.4, 0.4, 0.4, 0.1, 0.1]).reshape((10, 1)) 35 | preds = (preds.reshape((10, -1)) > thresholds).reshape((10, size, size)) 36 | preds = preds.astype(np.uint8) 37 | else: 38 | preds = np.argmax(preds, axis=0) 39 | preds = unsoft(preds) 40 | 41 | rg = colorize_raster(preds.transpose((1, 2, 0))) 42 | # cv2.imwrite('1.png', rg) 43 | size = 900 44 | rg = cv2.resize(rg, (size, size)) 45 | # cv2.imshow('mask', rg) 46 | # cv2.waitKey() 47 | im = get_rgb_image(image_id, size, size) 48 | rg = np.hstack([rg, im]) 49 | cv2.imwrite('raw_temp5_1/{}.png'.format(image_id), rg) 50 | 51 | shs = [] 52 | for i in range(10): 53 | mask = preds[i] 54 | 55 | y_sf, x_sf = get_scale_factor(image_id, mask.shape[0], mask.shape[1]) 56 | y_sf = 1. / y_sf 57 | x_sf = 1. / x_sf 58 | 59 | sh = polygonize_cv(mask) 60 | # sh = polygonize_sk((mask>0)*255, 0) 61 | # sh = (sh1.buffer(0).intersection(sh2.buffer(0))).buffer(0) 62 | 63 | # if not sh.is_valid: 64 | # sh = sh.buffer(0) 65 | sh = affinity.scale(sh, xfact=x_sf, yfact=y_sf, origin=(0, 0, 0)) 66 | shs.append(sh) 67 | return shs 68 | 69 | try: 70 | shutil.rmtree('raw_temp5_1/') 71 | except: 72 | pass 73 | os.mkdir('raw_temp5_1') 74 | 75 | #test_list = test_list[:12] 76 | pool = multiprocessing.Pool(20) 77 | res = pool.imap(mask_to_poly, test_list, chunksize=1) 78 | pool.close() 79 | 80 | fo = open('raw_blend5_1.csv', 'w') 81 | print('ImageId,ClassType,MultipolygonWKT', file=fo) 82 | 83 | for i, shs in enumerate(res): 84 | image_id = test_list[i] 85 | print('mip: {}'.format(i)) 86 | for j, sh in enumerate(shs): 87 | print('{},{},"{}"'.format(image_id, j + 1, sh.wkt), file=fo) 88 | print('mip ended') 89 | -------------------------------------------------------------------------------- /multi_predict.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import pandas as pd 3 | import numpy as np 4 | import mxnet as mx 5 | import cv2 6 | from utils import root_path 7 | from utils import unsoft 8 | from utils import colorize_raster 9 | from data_iter import get_image_data 10 | from utils import polygonize 11 | from shapely import affinity 12 | from utils import get_scale_factor 13 | 14 | df = pd.read_csv(root_path + '/sample_submission.csv') 15 | test_list = df.ImageId.unique() 16 | 17 | g_size = 640 18 | l_size = 160 19 | assert g_size % l_size == 0 20 | h = w = l_size 21 | batch_size = 2 22 | 23 | version = 'v23' 24 | epoch = 224 25 | ctx = mx.gpu(0) 26 | 27 | fo = open('{}-{}.csv'.format(version, epoch), 'w') 28 | print('ImageId,ClassType,MultipolygonWKT', file=fo) 29 | 30 | 31 | def predict(mod, image_id, g_size, l_size): 32 | sfs = [4] 33 | ms_preds = [] 34 | for sf in sfs: 35 | image_data = get_image_data(image_id, l_size*sf, l_size*sf) 36 | 37 | patch_data = [] 38 | for i in range(sf): 39 | for j in range(sf): 40 | patch_data.append(image_data[l_size*i: l_size*(i+1), 41 | l_size*j: l_size*(j+1)]) 42 | patch_data = np.array(patch_data) 43 | patch_data = patch_data.transpose((0, 3, 1, 2)) 44 | 45 | data_iter = mx.io.NDArrayIter(data=patch_data, batch_size=batch_size) 46 | preds = mod.predict(data_iter).asnumpy() 47 | gg = np.zeros((11, l_size*sf, l_size*sf)) 48 | for i in range(sf): # TODO via reshape 49 | for j in range(sf): 50 | gg[:, l_size*i: l_size*(i+1), 51 | l_size*j: l_size*(j+1)] = preds[i*sf+j] 52 | preds = gg 53 | preds = cv2.resize(preds.transpose((1, 2, 0)), (g_size, g_size)) 54 | preds = preds.transpose((2, 0, 1)) 55 | assert preds.shape == (11, g_size, g_size) 56 | ms_preds.append(preds) 57 | 58 | ms_preds = np.stack(ms_preds) 59 | preds = np.mean(ms_preds, axis=0) # multiscale averaging 60 | 61 | preds = np.argmax(preds, axis=0) 62 | preds = unsoft(preds) 63 | 64 | # y_pred = colorize_raster(preds.transpose((1, 2, 0))) 65 | # cv2.imshow('pred', y_pred) 66 | # cv2.waitKey() 67 | 68 | # pred_raster = preds.transpose((1, 2, 0)) 69 | 70 | assert preds.shape[0] == 10 71 | y_sf, x_sf = get_scale_factor(image_id, g_size, g_size) 72 | y_sf = 1. / y_sf 73 | x_sf = 1. / x_sf 74 | for i in range(10): 75 | sh = polygonize(preds[i]) 76 | sh = affinity.scale(sh, xfact=x_sf, yfact=y_sf, origin=(0, 0, 0)) 77 | print('{},{},"{}"'.format(image_id, i + 1, sh.wkt), file=fo) 78 | 79 | # y_pred = colorize_raster(pred_raster) 80 | # cv2.imshow('full pred', y_pred) 81 | # cv2.waitKey() 82 | 83 | sym, arg, aux = mx.model.load_checkpoint('models/' + version, epoch) 84 | mod = mx.module.Module(sym, context=ctx) 85 | mod.bind(data_shapes=[('data', (batch_size, 20, l_size, l_size))], 86 | for_training=False) 87 | mod.set_params(arg, aux) 88 | 89 | print('model loaded!') 90 | 91 | for i, image_id in enumerate(test_list): 92 | predict(mod, image_id, g_size, l_size) 93 | print(i) 94 | -------------------------------------------------------------------------------- /blend_raw.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import print_function 3 | import pandas as pd 4 | import numpy as np 5 | import mxnet as mx 6 | import cv2 7 | import os 8 | import shutil 9 | from sklearn.externals import joblib 10 | from shapely import affinity 11 | import multiprocessing 12 | from shapely.geometry import MultiPolygon, Polygon 13 | from collections import defaultdict 14 | 15 | from utils import polygonize, polygonize_cv, polygonize_sk, get_rgb_image 16 | from utils import get_scale_factor, root_path, colorize_raster, unsoft 17 | 18 | 19 | nets = [ 20 | # ('v57', 1400), 21 | # ('v58', 1290), 22 | # ('v59', 3820), 23 | # ('v60', 3780), 24 | # ('v61', 2620), 25 | # ('v62', 1500), 26 | # ('v63', 2700), 27 | # ('v74', 390), 28 | # ('v75', 850) 29 | 30 | ('v65', 3950), 31 | ('v67', 560), 32 | ('v68', 1810), 33 | ('v84', 1300) 34 | # ('v65', 3950), 35 | # ('v66', 2300) 36 | ] 37 | 38 | n_out = 11 39 | size = 2048 40 | 41 | df = pd.read_csv('input/sample_submission.csv') 42 | test_list = df.ImageId.unique() 43 | 44 | 45 | def mask_to_poly(image_id): 46 | global size 47 | xx = [] 48 | for version, epoch in nets: 49 | x = joblib.load('/data/raw_preds/{}-{}/{}.pkl'.format(version, epoch, image_id)) 50 | x = cv2.resize(x.transpose((1, 2, 0)), (size, size)) 51 | xx.append(x) 52 | 53 | xx = np.stack(xx, axis=3) 54 | preds = np.mean(xx, axis=3) 55 | # preds = joblib.load('raw_preds/{}-{}/{}.pkl'.format('v63', 2700, image_id)) 56 | # preds = cv2.resize(preds.transpose((1, 2, 0)), (size, size)) 57 | preds = preds.transpose((2, 0, 1)).copy() 58 | print(image_id) 59 | 60 | if n_out == 10: 61 | # preds = (preds > 0.3).astype(np.uint8) 62 | 63 | thresholds = np.array([0.4, 0.3, 0.3, 0.3, 0.7, 64 | 0.4, 0.4, 0.4, 0.04, 0.04]).reshape((10, 1)) 65 | preds = (preds.reshape((10, -1)) > thresholds).reshape((10, size, size)) 66 | preds = preds.astype(np.uint8) 67 | else: 68 | preds = np.argmax(preds, axis=0) 69 | preds = unsoft(preds) 70 | 71 | rg = colorize_raster(preds.transpose((1, 2, 0))) 72 | rg_size = 700 73 | rg = cv2.resize(rg, (rg_size, rg_size)) 74 | im = get_rgb_image(image_id, rg_size, rg_size) 75 | rg = np.hstack([rg, im]) 76 | cv2.imwrite('raw_blend_temp5/{}.png'.format(image_id), rg) 77 | joblib.dump(preds.astype(np.float32), 'raw_preds/raw_blend5/{}.pkl'.format(image_id)) 78 | return 79 | 80 | shs = [] 81 | for i in range(10): 82 | mask = preds[i] 83 | 84 | y_sf, x_sf = get_scale_factor(image_id, mask.shape[0], mask.shape[1]) 85 | y_sf = 1. / y_sf 86 | x_sf = 1. / x_sf 87 | 88 | sh = polygonize_cv(mask) 89 | # sh = polygonize_sk((mask>0)*255, 0) 90 | # sh = (sh1.buffer(0).intersection(sh2.buffer(0))).buffer(0) 91 | 92 | # if not sh.is_valid: 93 | # sh = sh.buffer(0) 94 | sh = affinity.scale(sh, xfact=x_sf, yfact=y_sf, origin=(0, 0, 0)) 95 | 96 | try: 97 | sh = MultiPolygon(sh) 98 | except: 99 | print('ERRRROR!!') 100 | sh = MultiPolygon() 101 | 102 | shs.append(sh) 103 | 104 | return shs 105 | 106 | try: 107 | shutil.rmtree('raw_blend_temp5/') 108 | except: 109 | pass 110 | os.mkdir('raw_blend_temp5') 111 | 112 | #test_list = test_list[:4] 113 | pool = multiprocessing.Pool(16) 114 | res = pool.imap(mask_to_poly, test_list, chunksize=1) 115 | pool.close() 116 | pool.join() 117 | qwe 118 | 119 | fo = open('blend_raw5.csv', 'w') 120 | print('ImageId,ClassType,MultipolygonWKT', file=fo) 121 | 122 | for i, shs in enumerate(res): 123 | image_id = test_list[i] 124 | print('mip: {}'.format(i)) 125 | for j, sh in enumerate(shs): 126 | print('{},{},"{}"'.format(image_id, j + 1, sh.wkt), file=fo) 127 | print('mip ended') -------------------------------------------------------------------------------- /b3_gen_prediction.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import pandas as pd 3 | import numpy as np 4 | import mxnet as mx 5 | import cv2 6 | import os 7 | from sklearn.externals import joblib 8 | from shapely import affinity 9 | import multiprocessing 10 | from shapely.geometry import MultiPolygon, Polygon 11 | from collections import defaultdict 12 | 13 | from utils import polygonize, polygonize_cv, polygonize_sk 14 | from utils import get_scale_factor, root_path, colorize_raster, unsoft 15 | 16 | from b3_data_iter import get_data, crop_maps 17 | from b3_data_iter import MultiInputSegDataIter 18 | 19 | df = pd.read_csv(root_path + '/sample_submission.csv') 20 | test_list = df.ImageId.unique() 21 | batch_size = 2 22 | 23 | version = 'v84' 24 | epoch = 1300 25 | ctx = mx.gpu(0) 26 | 27 | n_out = 11 28 | 29 | sf = 16 30 | a_size = 16 31 | m_size = 64 32 | p_size = 128 33 | l_size = 128 34 | 35 | d = {'a_data': (batch_size, 8, a_size, a_size), 36 | 'm_data': (batch_size, 8, m_size, m_size), 37 | 'p_data': (batch_size, 4, p_size, p_size) 38 | } 39 | 40 | 41 | def load_model(version, epoch, batch_size=8, ctx=mx.gpu()): 42 | sym, arg, aux = mx.model.load_checkpoint('models/' + version, epoch) 43 | mod = mx.module.Module(sym, context=ctx, data_names=list(d)) 44 | mod.bind(data_shapes=list(d.iteritems()), for_training=False) 45 | mod.set_params(arg, aux) 46 | return mod 47 | 48 | 49 | def predict(d): 50 | '''get predicted raster 51 | ''' 52 | image_id, i = d 53 | mod = load_model(version, epoch, batch_size, mx.gpu(0)) 54 | 55 | image_data = get_data(image_id, a_size, m_size, p_size, sf) 56 | 57 | P = [] 58 | M = [] 59 | A = [] 60 | for i in range(sf): 61 | for j in range(sf): 62 | rel_size = 1. / sf 63 | a, m, p = crop_maps(image_data, i*rel_size, j*rel_size, rel_size) 64 | P.append(p) 65 | M.append(m) 66 | A.append(a) 67 | 68 | A = np.array(A).transpose((0, 3, 1, 2)) 69 | M = np.array(M).transpose((0, 3, 1, 2)) 70 | P = np.array(P).transpose((0, 3, 1, 2)) 71 | data_iter = mx.io.NDArrayIter(data={'a_data': A, 72 | 'm_data': M, 73 | 'p_data': P}, batch_size=batch_size) 74 | preds = mod.predict(data_iter).asnumpy() 75 | 76 | gg = np.zeros((n_out, l_size*sf, l_size*sf)) 77 | for i in range(sf): # TODO via reshape 78 | for j in range(sf): 79 | gg[:, l_size*i: l_size*(i+1), 80 | l_size*j: l_size*(j+1)] = preds[i*sf+j] 81 | preds = gg 82 | # preds = preds.transpose((1, 2, 0)) 83 | assert preds.shape[0] == n_out 84 | return preds.astype(np.float32) 85 | 86 | 87 | def mask_to_poly(dd): 88 | preds, image_id = dd 89 | if n_out == 10: 90 | # preds = (preds > 0.3).astype(np.uint8) 91 | thresholds = np.array([0.3, 0.3, 0.3, 0.3, 0.4, 92 | 0.4, 0.3, 0.3, 0.2, 0.2]).reshape((10, 1)) 93 | preds = (preds.reshape((10, -1)) > thresholds).reshape((10, l_size*sf, l_size*sf)) 94 | preds = preds.astype(np.uint8) 95 | else: 96 | preds = np.argmax(preds, axis=0) 97 | preds = unsoft(preds) 98 | 99 | # rg = colorize_raster(preds.transpose((1, 2, 0))) 100 | 101 | # cv2.imwrite('1.png', rg) 102 | # cv2.imshow('mask', rg) 103 | # cv2.waitKey() 104 | 105 | shs = [] 106 | for i in range(10): 107 | mask = preds[i] 108 | 109 | y_sf, x_sf = get_scale_factor(image_id, mask.shape[0], mask.shape[1]) 110 | y_sf = 1. / y_sf 111 | x_sf = 1. / x_sf 112 | 113 | sh = polygonize_cv(mask) 114 | # sh = polygonize_sk((mask>0)*255, 0) 115 | # sh = (sh1.buffer(0).intersection(sh2.buffer(0))).buffer(0) 116 | 117 | # if not sh.is_valid: 118 | # sh = sh.buffer(0) 119 | 120 | sh = affinity.scale(sh, xfact=x_sf, yfact=y_sf, origin=(0, 0, 0)) 121 | shs.append(sh) 122 | 123 | return shs 124 | 125 | 126 | try: 127 | os.mkdir('raw_preds/{}-{}'.format(version, epoch)) 128 | except: 129 | pass 130 | 131 | #test_list = test_list[220:221] 132 | 133 | sip = multiprocessing.Pool(4) 134 | sip_res = sip.imap(predict, zip(test_list, range(len(test_list))), chunksize=1) 135 | 136 | mip = multiprocessing.Pool(40) 137 | mip_results = [] 138 | for i, mask in enumerate(sip_res): 139 | joblib.dump(mask, 'raw_preds/{}-{}/{}.pkl'.format(version, epoch, test_list[i])) 140 | print('sip: {}'.format(i)) 141 | mip_r = mip.map_async(mask_to_poly, [(mask, test_list[i])]) 142 | mip_results.append(mip_r) 143 | print('sip ended') 144 | mip.close() 145 | 146 | fo = open('{}-{}-{}.csv'.format(version, epoch, 'cv'), 'w') 147 | print('ImageId,ClassType,MultipolygonWKT', file=fo) 148 | 149 | for i, shs in enumerate(mip_results): 150 | image_id = test_list[i] 151 | print(image_id) 152 | shs = shs.get()[0] 153 | print('mip: {}'.format(i)) 154 | for j, sh in enumerate(shs): 155 | print('{},{},"{}"'.format(image_id, j + 1, sh.wkt), file=fo) 156 | print('mip ended') 157 | -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import tifffile as tiff 3 | import cv2 4 | import numpy as np 5 | import pandas as pd 6 | from shapely import wkt 7 | from shapely import affinity 8 | from rasterio.features import rasterize 9 | from rasterio import features 10 | from shapely import geometry 11 | from collections import defaultdict 12 | from shapely.geometry import MultiPolygon, Polygon 13 | from skimage import measure 14 | 15 | root_path = 'input/' 16 | three_band_path = root_path + '/three_band/' 17 | sixteen_band_path = root_path + '/sixteen_band/' 18 | grid_sizes = pd.read_csv(root_path + '/grid_sizes.csv') 19 | polygons_raw = pd.read_csv(root_path + '/train_wkt_v4.csv') 20 | 21 | 22 | def get_spectral_data(img_id, h, w, bands=['A', 'M', 'P']): 23 | res = [] 24 | for waveband in bands: 25 | image_path = '{}/{}_{}.tif'.format(sixteen_band_path, img_id, waveband) 26 | image = tiff.imread(image_path) 27 | if len(image.shape) == 2: # for panchromatic band 28 | image.shape = (1,) + image.shape 29 | image = image.transpose((1, 2, 0)) 30 | image = cv2.resize(image, (w, h), interpolation=cv2.INTER_LANCZOS4) 31 | if len(image.shape) == 2: # for panchromatic band 32 | image.shape += (1,) 33 | res.append(image) 34 | image = np.concatenate(res, axis=2) 35 | image = image.astype(np.float32) 36 | return image 37 | 38 | 39 | def get_rgb_data(img_id): 40 | image_path = three_band_path + '/' + img_id + '.tif' 41 | image = tiff.imread(image_path) 42 | image = image.transpose((1, 2, 0)) 43 | image = image.astype(np.float32) 44 | return image 45 | 46 | 47 | def get_rgb_image(img_id, h=None, w=None): 48 | image = get_rgb_data(img_id) 49 | image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) 50 | for c in range(3): 51 | min_val, max_val = np.percentile(image[:, :, c], [2, 98]) 52 | image[:, :, c] = 255*(image[:, :, c] - min_val) / (max_val - min_val) 53 | image[:, :, c] = np.clip(image[:, :, c], 0, 255) 54 | image = (image).astype(np.uint8) 55 | if h and w: 56 | image = cv2.resize(image, (w, h), interpolation=cv2.INTER_LANCZOS4) 57 | return image 58 | 59 | 60 | def get_polygons(img_id, h, w): 61 | y_sf, x_sf = get_scale_factor(img_id, w, h) 62 | polygons = [] 63 | image = polygons_raw[polygons_raw.ImageId == img_id] 64 | for cType in image.ClassType.unique(): 65 | wkt_str = image[image.ClassType == cType].MultipolygonWKT.values[0] 66 | sh = wkt.loads(wkt_str) 67 | sh = affinity.scale(sh, xfact=x_sf, yfact=y_sf, origin=(0, 0, 0)) 68 | polygons.append(sh) 69 | return polygons 70 | 71 | 72 | def get_polygon_train_image(img_id, h, w): 73 | im = cv2.imread('train_poly/{}.png'.format(img_id)) 74 | im = cv2.resize(im, (w, h), interpolation=cv2.INTER_LANCZOS4) 75 | return im 76 | 77 | 78 | def get_grid_size(img_id): 79 | i_grid_size = grid_sizes[grid_sizes.iloc[:, 0] == img_id] 80 | x_max = i_grid_size.Xmax.values[0] 81 | y_min = i_grid_size.Ymin.values[0] 82 | return y_min, x_max 83 | 84 | 85 | def get_scale_factor(img_id, h, w): 86 | w = float(w) 87 | h = float(h) 88 | y_min, x_max = get_grid_size(img_id) 89 | w_ = w * (w/(w+1)) 90 | h_ = h * (h/(h+1)) 91 | x_sf = w_ / x_max 92 | y_sf = h_ / y_min 93 | return y_sf, x_sf 94 | 95 | 96 | def get_raster(img_id, h, w): 97 | polygons = get_polygons(img_id, h, w) 98 | return rasterize_polgygon(polygons, h, w) 99 | 100 | 101 | def rasterize_polgygon(polygons, h, w): 102 | r = [] 103 | for pol in polygons: 104 | result = rasterize([pol], out_shape=(h, w)) 105 | r.append(result) 106 | r = np.stack(r) 107 | r = r.transpose((1, 2, 0)) 108 | return r 109 | 110 | 111 | def colorize_raster(masks): 112 | ''' (H, W, 10) -> (H, W, 3) 113 | ''' 114 | assert masks.shape[2] == 10 115 | palette = np.array([(180, 180, 180), (100, 100, 100), # Buildings, Misc. 116 | (6, 88, 179), (125, 194, 223), # Road, Track 117 | (55, 120, 27), (160, 219, 166), # Trees, Crops 118 | (209, 173, 116), (180, 117, 69), # Waterway, Standing 119 | (67, 109, 244), (39, 48, 215)], dtype=np.uint8) # Car 120 | 121 | r = [] 122 | for obj_type in range(10): 123 | c = palette[obj_type] 124 | result = np.stack([masks[:, :, obj_type]] * 3, axis=2) 125 | r.append(result * c) 126 | r = np.stack(r) 127 | r = np.max(r, axis=0) 128 | return r 129 | 130 | 131 | def polygonize(im): 132 | assert len(im.shape) == 2 133 | shapes = features.shapes(im) 134 | polygons = [] 135 | for i, shape in enumerate(shapes): 136 | # if i % 10000 == 0: 137 | # print(i) 138 | if shape[1] == 0: 139 | continue 140 | polygons.append(geometry.shape(shape[0])) 141 | # polygons = [geometry.shape(shape[0]) for shape in shapes if shape[1] > 0] 142 | mp = geometry.MultiPolygon(polygons) 143 | return mp 144 | 145 | 146 | def jaccard_raster(true_raster, pred_raster): 147 | assert true_raster.shape[2] == pred_raster.shape[2] == 10 148 | score = [] 149 | for i in range(10): 150 | true = true_raster[:, :, i] != 0 151 | pred = pred_raster[:, :, i] != 0 152 | tp = np.sum(true * pred) 153 | fp = np.sum(pred) - tp 154 | fn = np.sum(true) - tp 155 | if tp == 0: 156 | jac = 0. 157 | else: 158 | jac = tp / float(fp + fn + tp) 159 | score.append((tp, fp, fn, jac)) 160 | score = np.array(score) 161 | assert score.shape == (10, 4) 162 | return score 163 | 164 | 165 | def unsoft(max_idx, num_class=11): 166 | max_idx = max_idx.squeeze() 167 | assert len(max_idx.shape) == 2 168 | preds = np.zeros((num_class - 1, max_idx.shape[0], max_idx.shape[1])) 169 | for i in range(1, num_class): 170 | preds[i - 1] = (max_idx == i) 171 | preds = preds.astype(np.uint8) 172 | return preds 173 | 174 | 175 | def polygonize_cv(mask, epsilon=1., min_area=10.): 176 | contours, hierarchy = cv2.findContours(mask, cv2.RETR_CCOMP, cv2.CHAIN_APPROX_TC89_KCOS) 177 | # create approximate contours to have reasonable submission size 178 | approx_contours = [cv2.approxPolyDP(cnt, epsilon, True) 179 | for cnt in contours] 180 | approx_contours = contours 181 | if not contours: 182 | return MultiPolygon() 183 | # now messy stuff to associate parent and child contours 184 | cnt_children = defaultdict(list) 185 | child_contours = set() 186 | assert hierarchy.shape[0] == 1 187 | # http://docs.opencv.org/3.1.0/d9/d8b/tutorial_py_contours_hierarchy.html 188 | for idx, (_, _, _, parent_idx) in enumerate(hierarchy[0]): 189 | if parent_idx != -1: 190 | child_contours.add(idx) 191 | cnt_children[parent_idx].append(approx_contours[idx]) 192 | # create actual polygons filtering by area (removes artifacts) 193 | all_polygons = [] 194 | for idx, cnt in enumerate(approx_contours): 195 | if idx not in child_contours and cv2.contourArea(cnt) >= min_area: 196 | assert cnt.shape[1] == 1 197 | poly = Polygon( 198 | shell=cnt[:, 0, :], 199 | holes=[c[:, 0, :] for c in cnt_children.get(idx, []) 200 | if cv2.contourArea(c) >= min_area]) 201 | all_polygons.append(poly) 202 | # approximating polygons might have created invalid ones, fix them 203 | all_polygons = MultiPolygon(all_polygons) 204 | if not all_polygons.is_valid: 205 | all_polygons = all_polygons.buffer(0) 206 | # Sometimes buffer() converts a simple Multipolygon to just a Polygon, 207 | # need to keep it a Multi throughout 208 | if all_polygons.type == 'Polygon': 209 | all_polygons = MultiPolygon([all_polygons]) 210 | return all_polygons 211 | 212 | 213 | def polygonize_sk(mask, level): 214 | contours = measure.find_contours(mask, level) 215 | polys = [] 216 | for contour in contours: 217 | if contour.shape[0] < 4: 218 | continue 219 | poly = Polygon(shell=contour[:, [1, 0]]) 220 | polys.append(poly) 221 | polys = MultiPolygon(polys) 222 | return polys 223 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /b3_train.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import mxnet as mx 3 | import cv2 4 | import pandas as pd 5 | import time 6 | import multiprocessing 7 | import sys 8 | import logging 9 | import numpy as np 10 | import datetime 11 | sys.path.append('../') 12 | 13 | from utils import colorize_raster 14 | from utils import get_raster 15 | from utils import jaccard_raster 16 | 17 | from b3_data_iter import unsoft 18 | from b3_data_iter import get_data 19 | from b3_data_iter import a_size, m_size, p_size 20 | from b3_data_iter import MultiInputSegDataIter 21 | 22 | 23 | np.set_printoptions(2, suppress=True) 24 | logger = logging.getLogger() 25 | logger.setLevel(logging.INFO) 26 | 27 | batch_size = 8 28 | ctx = [mx.gpu(0)] 29 | version = 'v96' 30 | print(version) 31 | 32 | d = { 33 | 'a_data': (batch_size, 8, a_size, a_size), 34 | 'm_data': (batch_size, 8, m_size, m_size), 35 | 'p_data': (batch_size, 4, p_size, p_size) 36 | } 37 | 38 | def print_inferred_shape(net): 39 | ar, ou, au = net.infer_shape(**d) 40 | print(net.name, ou) 41 | 42 | def up_module(net, num_filter, name=''): 43 | net = mx.sym.Deconvolution(net, kernel=(4, 4), pad=(1, 1), stride=(2, 2), 44 | num_filter=num_filter, name=name+'_deconv', no_bias=True) 45 | # net = mx.sym.UpSampling(net, scale=2, sample_type='nearest') 46 | # net = mx.symbol.Convolution(data=net, kernel=(3, 3), stride=(1, 1), pad=(1, 1), num_filter=num_filter) 47 | 48 | net = mx.sym.BatchNorm(net, name=name+'_bn') 49 | # net = mx.sym.InstanceNorm(net) 50 | 51 | net = mx.sym.LeakyReLU(net, act_type='leaky', name=name+'_act') 52 | # net = mx.sym.Activation(net, act_type='relu') 53 | return net 54 | 55 | 56 | def down_module(net, kernel_size, pad_size, num_filter, stride=(1, 1), dilate_size=(1, 1), name='', down=False, dilate=False): 57 | # if down: 58 | # stride=(2, 2) 59 | 60 | if dilate: 61 | dilate_size = (3, 3) 62 | pad_size = (3, 3) 63 | 64 | net = mx.sym.Convolution(data=net, kernel=kernel_size, stride=stride, dilate=dilate_size, 65 | pad=pad_size, num_filter=num_filter, name='{}_conv'.format(name), no_bias=True) 66 | 67 | 68 | net = mx.sym.BatchNorm(net, name=name+'_bn') 69 | # net = mx.sym.InstanceNorm(net) 70 | 71 | # net = mx.sym.Activation(net, act_type='relu') 72 | net = mx.sym.LeakyReLU(net, act_type='leaky', name=name+'_act') 73 | 74 | if down: 75 | net = mx.sym.Pooling(net, pool_type="max", kernel=(2, 2), stride=(2, 2), name=name+'_pool') 76 | 77 | return net 78 | 79 | 80 | def residual_unit(data, num_filter, stride, dim_match, name, bottle_neck=True, num_group=32, bn_mom=0.9, workspace=256, memonger=False): 81 | if bottle_neck: 82 | conv1 = mx.sym.Convolution(data=data, num_filter=int(num_filter*0.5), kernel=(1,1), stride=(1,1), pad=(0,0), 83 | no_bias=True, workspace=workspace, name=name + '_conv1') 84 | # bn1 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn1') 85 | bn1 = mx.sym.InstanceNorm(conv1) 86 | 87 | act1 = mx.sym.Activation(data=bn1, act_type='relu', name=name + '_relu1') 88 | 89 | conv2 = mx.sym.Convolution(data=act1, num_filter=int(num_filter*0.5), num_group=num_group, kernel=(3,3), stride=stride, pad=(1,1), 90 | no_bias=True, workspace=workspace, name=name + '_conv2') 91 | # bn2 = mx.sym.BatchNorm(data=conv2, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn2') 92 | bn2 = mx.sym.InstanceNorm(conv2) 93 | act2 = mx.sym.Activation(data=bn2, act_type='relu', name=name + '_relu2') 94 | 95 | 96 | conv3 = mx.sym.Convolution(data=act2, num_filter=num_filter, kernel=(1,1), stride=(1,1), pad=(0,0), no_bias=True, 97 | workspace=workspace, name=name + '_conv3') 98 | # bn3 = mx.sym.BatchNorm(data=conv3, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn3') 99 | bn3 = mx.sym.InstanceNorm(conv3) 100 | 101 | if dim_match: 102 | shortcut = data 103 | else: 104 | shortcut_conv = mx.sym.Convolution(data=data, num_filter=num_filter, kernel=(1,1), stride=stride, no_bias=True, 105 | workspace=workspace, name=name+'_sc') 106 | # shortcut = mx.sym.BatchNorm(data=shortcut_conv, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_sc_bn') 107 | shortcut = mx.sym.InstanceNorm(shortcut_conv) 108 | 109 | if memonger: 110 | shortcut._set_attr(mirror_stage='True') 111 | eltwise = bn3 + shortcut 112 | return mx.sym.Activation(data=eltwise, act_type='relu', name=name + '_relu') 113 | 114 | 115 | def conv_3d(net, num_filter, name='3d'): 116 | net = mx.sym.Reshape(net, shape=(0, -4, 1, -1, -2)) 117 | net = mx.sym.Convolution(data=net, kernel=(64, 3, 3), stride=(1, 1, 1), pad=(1,1,1), num_filter=10, name='{}_conv'.format(name)) 118 | net = mx.sym.Reshape(net, shape=(0, -1, 128, 128)) 119 | 120 | net = mx.sym.Convolution(data=net, kernel=(3, 3), num_filter=10, name='{}_conv'.format(name)) 121 | 122 | net = mx.sym.InstanceNorm(net) 123 | net = mx.sym.LeakyReLU(net, act_type='leaky', name=name+'_act') 124 | return net 125 | 126 | 127 | def get_symbol(): 128 | kernel_size = (3, 3) 129 | pad_size = (1, 1) 130 | num_filter = 64 131 | 132 | a_data = mx.sym.Variable("a_data") 133 | a_data = mx.sym.BatchNorm(a_data) 134 | # a_data = mx.sym.InstanceNorm(a_data) 135 | 136 | m_data = mx.sym.Variable("m_data") 137 | m_data = mx.sym.BatchNorm(m_data) 138 | # m_data = mx.sym.InstanceNorm(m_data) 139 | 140 | p_data = mx.sym.Variable("p_data") 141 | p_data = mx.sym.BatchNorm(p_data) 142 | # p_data = mx.sym.InstanceNorm(p_data) 143 | 144 | 145 | ######## p down 146 | # p_net = p_data 147 | p_net = down1 = down_module(p_data, kernel_size, pad_size, num_filter=num_filter, name='p_down1', down=True) 148 | p_net = down2 = down_module(p_net, kernel_size, pad_size, num_filter=num_filter*2, name='p_down2', down=True) 149 | # p_net = down3 = down_module(p_net, kernel_size, pad_size, num_filter=num_filter*3, name='p_down3', down=True) 150 | 151 | ######## m down 152 | m_net = m_data 153 | m_net = down4 = down_module(m_data, kernel_size, pad_size, num_filter=num_filter, name='m_down1', down=True) 154 | 155 | ######## a down 156 | a_net = a_data 157 | a_net = down_module(a_data, kernel_size, pad_size, num_filter=num_filter, name='a_down1', down=True) 158 | 159 | ######## p, m concat 160 | pm_net = mx.sym.Concat(*[p_net, m_net]) 161 | pm_net = pm_net_down1 = down_module(pm_net, kernel_size, pad_size, num_filter=num_filter*3, name='pm_down1', down=True) 162 | pm_net = down_module(pm_net, kernel_size, pad_size, num_filter=num_filter*4, name='pm_down2', down=True) 163 | 164 | ######## pm, a concat 165 | pma_net = mx.sym.Concat(*[pm_net, a_net]) 166 | pma_net = down_module(pma_net, kernel_size, pad_size, num_filter=num_filter*4, name='pma_down1', down=False) 167 | # pma_net = down_module(pma_net, kernel_size, pad_size, num_filter=num_filter*3, name='pma_down2', down=True) 168 | # pma_net = down_module(pma_net, kernel_size, pad_size, num_filter=num_filter*4, name='pma_down3', down=True) 169 | 170 | net = pma_net 171 | 172 | # net = down_module(net, kernel_size, pad_size, num_filter=num_filter*4, name='pma_down1', down=True, dilate=False) 173 | # net = down_module(net, kernel_size, pad_size, num_filter=num_filter*2, name='convup2', down=False, dilate=False) 174 | 175 | net = up_module(net, num_filter=num_filter*4, name='up1') 176 | # print_inferred_shape(net) 177 | # qwe 178 | 179 | # net = mx.sym.Concat(*[net, pm_net_down1]) 180 | # net = up_module(net, num_filter=num_filter*3, name='up2') 181 | 182 | net = down_module(net, kernel_size, pad_size, num_filter=num_filter*3, name='convup2', down=False, dilate=False) 183 | net = up_module(net, num_filter=num_filter*3, name='up2') 184 | 185 | # net = mx.sym.Concat(*[net, m_net, down2]) 186 | net = down_module(net, kernel_size, pad_size, num_filter=num_filter*2, name='convup3', down=False, dilate=False) 187 | net = up_module(net, num_filter=num_filter*2, name='up3') 188 | 189 | net = mx.sym.Concat(*[net, down1]) 190 | net = down_module(net, kernel_size, pad_size, num_filter=num_filter, name='convup4', down=False, dilate=False) 191 | net = up_module(net, num_filter=num_filter, name='up4') 192 | 193 | 194 | # p_ref = down_module(p_data, kernel_size, pad_size, num_filter=num_filter, name='ref', down=False, dilate=False) 195 | # net = mx.sym.Concat(*[net, p_ref]) 196 | # net = down_module(net, kernel_size, pad_size, num_filter=num_filter, name='convup5', down=False, dilate=False) 197 | 198 | # net = up_module(net, num_filter=num_filter, name='up5') 199 | 200 | # net = residual_unit(net, num_filter=num_filter*2, stride=(1, 1), dim_match=True, name='res', num_group=1) 201 | 202 | 203 | # net = mx.sym.Concat(*[net, down1]) 204 | # net = down_module(net, kernel_size, pad_size, num_filter=num_filter*2, name='convup4', down=False) 205 | 206 | # net = up_module(net, num_filter=num_filter*2, name='up4') 207 | 208 | # net = up_module(net, num_filter=num_filter, name='up5') 209 | 210 | # net = down_module(net, kernel_size, pad_size, num_filter=num_filter*2, name='convup5', down=False) 211 | # net = up_module(net, num_filter=num_filter, name='up5') 212 | 213 | # x = down_module(p_data, kernel_size, pad_size, num_filter=num_filter, name='convup_p', down=False) 214 | 215 | # net = down_module(net, kernel_size, pad_size, num_filter=num_filter, name='convup6', down=False) 216 | # net = up_module(net, num_filter=num_filter, name='up5') 217 | 218 | net = mx.sym.Convolution(data=net, kernel=(5,5), stride=(1,1), 219 | pad=(2,2), num_filter=10, name='final', no_bias=True) 220 | 221 | 222 | # print_inferred_shape(net) 223 | # net = conv_3d(net, 10) 224 | # print_inferred_shape(net) 225 | # qwe 226 | 227 | # net = mx.symbol.SoftmaxOutput(data=net, multi_output=True, use_ignore=False, ignore_label=0, name="softmax") 228 | # net = mx.symbol.LinearRegressionOutput(net, name='softmax') 229 | net = mx.symbol.LogisticRegressionOutput(net, name='softmax') 230 | # print_inferred_shape(net) 231 | return net 232 | 233 | 234 | sym = get_symbol() 235 | sym.list_arguments() 236 | 237 | all_list = pd.read_csv('input/train_wkt_v4.csv').ImageId.unique().tolist() 238 | 239 | test_list = ['6040_1_0', '6060_2_3', '6070_2_3', '6120_2_2', '6170_2_4'] # v3 240 | #all_list = ['6070_2_3', '6100_1_3', '6110_3_1', '6010_4_2'] 241 | #all_list = ['6100_2_2'] 242 | #test_list = pd.read_csv('blend.csv').ImageId.unique().tolist() 243 | #test_list = [x for x in test_list if not x.startswith('6080')] 244 | #train_list += test_list 245 | #train_list = list(set(all_list) - set(test_list)) 246 | #all_list = train_list + test_list 247 | train_list = all_list 248 | 249 | print('Number image in train: {}'.format(len(train_list))) 250 | print('Number image in test: {}'.format(len(test_list))) 251 | 252 | train_iter = MultiInputSegDataIter(train_list, batch_size, 500, test_list=[]) 253 | #train_iter = mx.io.PrefetchingIter(train_iter) 254 | 255 | print('Data shapes: ', train_iter.provide_data, train_iter.provide_label) 256 | 257 | a = mx.viz.plot_network(sym, shape=d) 258 | a.render('{}.pdf'.format(version)) 259 | 260 | #args, out, aux = sym.infer_shape(data=train_iter.provide_data[0][1]) 261 | #for n, s in zip(sym.list_arguments(), args): 262 | # print(n, s) 263 | #print('-'*30) 264 | #for n, s in zip(sym.list_outputs(), out): 265 | # print(n, s) 266 | #print('-'*30) 267 | #internals = sym.get_internals() 268 | #_, out, _ = internals.infer_shape(data=train_iter.provide_data[0][1]) 269 | #for n, s in zip(internals.list_outputs(), out): 270 | # print(n, s) 271 | #print('-'*30) 272 | 273 | mod = mx.module.Module(sym, context=ctx, data_names=['a_data', 'm_data', 'p_data']) 274 | mod.bind(data_shapes=train_iter.provide_data, 275 | label_shapes=train_iter.provide_label) 276 | mod.init_params(initializer=mx.initializer.Xavier()) 277 | mod.init_optimizer(optimizer=mx.optimizer.Adam()) 278 | 279 | 280 | class SegMetric(mx.metric.EvalMetric): 281 | def __init__(self): 282 | super(SegMetric, self).__init__('seg-metric') 283 | self.cnt = 0 284 | self.tp = [] 285 | self.fp = [] 286 | self.fn = [] 287 | 288 | def update(self, labels, preds): 289 | preds = preds[0][0].asnumpy() 290 | labels = labels[0][0].asnumpy() 291 | 292 | if preds.shape[0] == 11: 293 | preds = np.argmax(preds, axis=0) 294 | preds = unsoft(preds) 295 | labels = unsoft(labels) 296 | else: 297 | preds = preds > 0.5 298 | 299 | labels = labels.transpose((1, 2, 0)) 300 | preds = preds.transpose((1, 2, 0)) 301 | score = jaccard_raster(labels, preds) 302 | self.tp.append(score[:, 0]) 303 | self.fp.append(score[:, 1]) 304 | self.fn.append(score[:, 2]) 305 | return 306 | 307 | def aggregate_jaccard_score(self): 308 | tp = np.stack(self.tp) 309 | fp = np.stack(self.fp) 310 | fn = np.stack(self.fn) 311 | tp = np.mean(tp, axis=0) 312 | fp = np.mean(fp, axis=0) 313 | fn = np.mean(fn, axis=0) 314 | jac = tp / (tp + fp + fn) 315 | jac[np.isnan(jac)] = 0 316 | for i in range(10): 317 | print('Patch-based jaccard {}: {:.2f}'.format(i, jac[i])) 318 | print('Patch-based jaccard: {}'.format(np.mean(jac))) 319 | return jac 320 | 321 | n_epoch = 5000 322 | for epoch in range(1, n_epoch): 323 | ts = time.time() 324 | acc = mx.metric.Accuracy() 325 | # mse = mx.metric.RMSE() 326 | sgm = SegMetric() 327 | 328 | for i, data_batch in enumerate(train_iter): 329 | mod.forward_backward(data_batch) 330 | mod.update() 331 | # mod.update_metric(eval_metric=mse, labels=data_batch.label) 332 | mod.update_metric(eval_metric=sgm, labels=data_batch.label) 333 | 334 | # print('train mse: ', epoch, mse.sum_metric / mse.num_inst) 335 | jac_acc = sgm.aggregate_jaccard_score() 336 | train_iter.update_sampler(jac_acc) 337 | 338 | if epoch % 10 == 0: 339 | mx.model.save_checkpoint('models/' + version, epoch, 340 | mod.symbol, *mod.get_params()) 341 | print('epoch: {} time: {}'.format(epoch, time.time() - ts)) 342 | print(datetime.datetime.now()) 343 | train_iter.reset() 344 | -------------------------------------------------------------------------------- /b3_data_iter.py: -------------------------------------------------------------------------------- 1 | """ Data iterator""" 2 | import mxnet as mx 3 | import numpy as np 4 | import sys, os 5 | import cv2 6 | import time 7 | import multiprocessing 8 | import itertools 9 | 10 | from scipy import ndimage 11 | from sklearn import neighbors 12 | sys.path.append('../') 13 | 14 | from utils import get_rgb_data 15 | from utils import get_spectral_data 16 | 17 | from utils import get_polygons 18 | from utils import rasterize_polgygon 19 | from utils import get_raster 20 | from utils import colorize_raster 21 | from utils import get_rgb_image 22 | from utils import unsoft, get_scale_factor, rasterize_polgygon 23 | 24 | import tifffile as tiff 25 | import cv2 26 | import numpy as np 27 | import pandas as pd 28 | from shapely import wkt 29 | from shapely import affinity 30 | from rasterio.features import rasterize 31 | from rasterio import features 32 | from shapely import geometry 33 | from collections import defaultdict 34 | from shapely.geometry import MultiPolygon, Polygon 35 | from skimage import measure, exposure 36 | 37 | A_data = [] 38 | M_data = [] 39 | P_data = [] 40 | y_mask = [] 41 | 42 | sf = 24 43 | a_size = 16 44 | m_size = 64 45 | p_size = 128 46 | l_size = 128 47 | n_out = 10 48 | 49 | print('sf: {}'.format(sf)) 50 | 51 | 52 | class CropSampler(object): 53 | ''' Draw a class_i from the class probability distribution; 54 | Draw a random ImageId with given class_i, from the prev step; 55 | Sample a crop position from ImageId based on the kde of labels 56 | ''' 57 | def __init__(self, masks): 58 | n_class = 10 59 | self.maps_with_class = [[], [], [], [], [], [], [], [], [], []] 60 | self.kde_samplers = [] 61 | self.class_probs = np.ones(n_class) / n_class 62 | # self.class_probs = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0.5, 0.5]) 63 | self.mask_size = None 64 | ts = time.time() 65 | for mask_i, mask in enumerate(masks): 66 | assert mask.shape[2] == n_class 67 | if not self.mask_size: 68 | self.mask_size = mask.shape[1] 69 | samplers = [] 70 | for class_i in range(n_class): 71 | X = np.nonzero(mask[:, :, class_i]) 72 | X = np.stack(X, axis=1) 73 | 74 | # np.random.shuffle(X) 75 | # X = X[:50000] 76 | 77 | if not X.size: 78 | samplers.append(None) 79 | else: 80 | self.maps_with_class[class_i].append(mask_i) 81 | sampler = neighbors.KernelDensity(self.mask_size * 0.02).fit(X) 82 | samplers.append(sampler) 83 | 84 | assert len(samplers) == n_class 85 | self.kde_samplers.append(samplers) 86 | print('sampler init time: {}'.format(time.time() - ts)) 87 | 88 | def update(self, probs): 89 | assert self.class_probs.size == probs.size 90 | self.class_probs = np.copy(probs) 91 | 92 | def sample_crop(self, n): 93 | kx = np.array([len(x) for x in self.maps_with_class]) 94 | class_hist = np.random.multinomial(n, self.class_probs * (kx != 0)) 95 | class_ids = np.repeat(np.arange(class_hist.shape[0]), class_hist) 96 | X = [] 97 | for class_id in class_ids: 98 | for i in range(20): 99 | random_image_idx = np.random.choice(self.maps_with_class[class_id]) 100 | if random_image_idx < 25: 101 | break 102 | x = self.kde_samplers[random_image_idx][class_id].sample()[0] 103 | x /= self.mask_size 104 | x = np.clip(x, 0., 1.) 105 | return x, class_id, random_image_idx 106 | X.append(x) 107 | return X 108 | 109 | sampler = None 110 | 111 | 112 | def flip_mat(mat): 113 | n_mat = np.zeros(mat.shape, dtype=np.float32) 114 | for i in range(mat.shape[2]): 115 | n_mat[:, :, i] = np.fliplr(mat[:, :, i]) 116 | return n_mat 117 | 118 | 119 | def rot90_mat(mat, k): 120 | n_mat = np.zeros(mat.shape, dtype=np.float32) 121 | for i in range(mat.shape[2]): 122 | n_mat[:, :, i] = np.rot90(mat[:, :, i], k) 123 | return n_mat 124 | 125 | 126 | def get_data(image_id, a_size, m_size, p_size, sf): 127 | rgb_data = get_rgb_data(image_id) 128 | rgb_data = cv2.resize(rgb_data, (p_size*sf, p_size*sf), 129 | interpolation=cv2.INTER_LANCZOS4) 130 | 131 | # rgb_data = rgb_data.astype(np.float) / 2500. 132 | # print(np.max(rgb_data), np.mean(rgb_data)) 133 | 134 | # rgb_data[:, :, 0] = exposure.equalize_adapthist(rgb_data[:, :, 0], clip_limit=0.04) 135 | # rgb_data[:, :, 1] = exposure.equalize_adapthist(rgb_data[:, :, 1], clip_limit=0.04) 136 | # rgb_data[:, :, 2] = exposure.equalize_adapthist(rgb_data[:, :, 2], clip_limit=0.04) 137 | 138 | A_data = get_spectral_data(image_id, a_size*sf, a_size*sf, bands=['A']) 139 | M_data = get_spectral_data(image_id, m_size*sf, m_size*sf, bands=['M']) 140 | P_data = get_spectral_data(image_id, p_size*sf, p_size*sf, bands=['P']) 141 | 142 | # lab_data = cv2.cvtColor(rgb_data, cv2.COLOR_BGR2LAB) 143 | P_data = np.concatenate([rgb_data, P_data], axis=2) 144 | 145 | return A_data, M_data, P_data 146 | 147 | 148 | def crop_maps(maps, rel_x, rel_y, rel_size): 149 | ''' Crop with relative coords 150 | ''' 151 | # assert all([0. <= rel_x, rel_y, rel_size <= 1.]) 152 | assert rel_x + rel_size <= 1 153 | res = [] 154 | for m in maps: 155 | abs_x = int(rel_x * m.shape[1]) 156 | abs_y = int(rel_y * m.shape[1]) 157 | abs_size = int(rel_size * m.shape[1]) 158 | res.append(m[abs_x: abs_x + abs_size, abs_y: abs_y + abs_size]) 159 | return res 160 | 161 | 162 | def get_crop_position(rel_cx, rel_cy, crop_size, map_size): 163 | abs_cx = rel_cx * map_size - crop_size / 2. 164 | abs_cy = rel_cy * map_size - crop_size / 2. 165 | abs_cx = int(min(max(abs_cx, 0), map_size - crop_size)) # out of border 166 | abs_cy = int(min(max(abs_cy, 0), map_size - crop_size)) 167 | return abs_cx, abs_cy 168 | 169 | 170 | def rel_crop(im, rel_cx, rel_cy, crop_size): 171 | 172 | map_size = im.shape[1] 173 | r = crop_size / 2 174 | abs_cx = rel_cx * map_size 175 | abs_cy = rel_cy * map_size 176 | na = np.floor([abs_cy-r, abs_cy+r, abs_cx-r, abs_cx+r]).astype(np.int32) 177 | a = np.clip(na, 0, map_size) 178 | px0 = a[2] - na[2] 179 | px1 = na[3] - a[3] 180 | py0 = a[0] - na[0] 181 | py1 = na[1] - a[1] 182 | crop = im[a[0]:a[1], a[2]:a[3]] 183 | crop = np.pad(crop, ((py0, py1), (px0, px1), (0, 0)), 184 | mode='reflect') 185 | 186 | assert crop.shape == (crop_size, crop_size, im.shape[2]) 187 | return crop 188 | 189 | 190 | def get_random_data(): 191 | (y, x), class_id, im_idx = sampler.sample_crop(1) 192 | 193 | a_data_glob = A_data[im_idx] 194 | m_data_glob = M_data[im_idx] 195 | p_data_glob = P_data[im_idx] 196 | label_glob = y_mask[im_idx] 197 | 198 | a_x, a_y = get_crop_position(x, y, a_size, a_data_glob.shape[1]) 199 | m_x, m_y = get_crop_position(x, y, m_size, m_data_glob.shape[1]) 200 | p_x, p_y = get_crop_position(x, y, p_size, p_data_glob.shape[1]) 201 | l_x, l_y = get_crop_position(x, y, l_size, label_glob.shape[1]) 202 | a_data = a_data_glob[a_y: a_y + a_size, a_x: a_x + a_size] 203 | m_data = m_data_glob[m_y: m_y + m_size, m_x: m_x + m_size] 204 | p_data = p_data_glob[p_y: p_y + p_size, p_x: p_x + p_size] 205 | label = label_glob[l_y: l_y + l_size, l_x: l_x + l_size] 206 | 207 | # a_data = rel_crop(a_data_glob, x, y, a_size) 208 | # m_data = rel_crop(m_data_glob, x, y, m_size) 209 | # p_data = rel_crop(p_data_glob, x, y, p_size) 210 | # label = rel_crop(label_glob, x, y, l_size) 211 | 212 | # rgb = colorize_raster(label) 213 | # cv2.circle(rgb, (int(x * label_glob.shape[1]), int(y * label_glob.shape[1])), 30, (0, 0, 255)) 214 | # cv2.imshow('label', rgb) 215 | # 216 | # def get_rgb_image1(image, h=None, w=None): 217 | # image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) 218 | # for c in range(3): 219 | # min_val, max_val = np.percentile(image[:, :, c], [2, 98]) 220 | # image[:, :, c] = 255*(image[:, :, c] - min_val) / (max_val - min_val) 221 | # image[:, :, c] = np.clip(image[:, :, c], 0, 255) 222 | # image = (image).astype(np.uint8) 223 | # return image 224 | # 225 | # rgb_data = get_rgb_image1(p_data[:, :, 0:3]) 226 | # cv2.imshow('rgb_data', rgb_data) 227 | # cv2.waitKey() 228 | 229 | if np.random.randint(0, 2): 230 | a_data = flip_mat(a_data) 231 | m_data = flip_mat(m_data) 232 | p_data = flip_mat(p_data) 233 | label = flip_mat(label) 234 | 235 | if np.random.randint(0, 2): 236 | k = np.random.randint(0, 4) 237 | a_data = rot90_mat(a_data, k) 238 | m_data = rot90_mat(m_data, k) 239 | p_data = rot90_mat(p_data, k) 240 | label = rot90_mat(label, k) 241 | 242 | # if np.random.randint(0, 2): 243 | # angle = np.random.randint(0, 180) 244 | # data = ndimage.interpolation.rotate(data, angle, reshape=False) 245 | # label = ndimage.interpolation.rotate(label, angle, reshape=False) 246 | 247 | # assert label.shape[:2] == p_data.shape[:2] 248 | 249 | a_data = np.transpose(a_data, (2, 0, 1)) 250 | m_data = np.transpose(m_data, (2, 0, 1)) 251 | p_data = np.transpose(p_data, (2, 0, 1)) 252 | label = np.transpose(label, (2, 0, 1)) 253 | 254 | if n_out == 11: 255 | label = np.argmax(label, axis=0) + (np.max(label, axis=0) != 0) # 0 256 | label.shape = (1,) + label.shape 257 | 258 | return a_data, m_data, p_data, label 259 | 260 | 261 | class Batch(mx.io.DataBatch): 262 | def __init__(self, data_names, data, label_names, label): 263 | self.data = data 264 | self.label = label 265 | self.data_names = data_names 266 | self.label_names = label_names 267 | self.pad = 0 268 | self.index = 0 269 | 270 | @property 271 | def provide_data(self): 272 | return [(n, x.shape) for n, x in zip(self.data_names, self.data)] 273 | 274 | @property 275 | def provide_label(self): 276 | return [(n, x.shape) for n, x in zip(self.label_names, self.label)] 277 | 278 | #polygons_test = pd.read_csv('blend.csv') 279 | #def get_test_polygons(img_id, h, w): 280 | # y_sf, x_sf = get_scale_factor(img_id, w, h) 281 | # polygons = [] 282 | # image = polygons_test[polygons_test.ImageId == img_id] 283 | # for cType in image.ClassType.unique(): 284 | # wkt_str = image[image.ClassType == cType].MultipolygonWKT.values[0] 285 | # sh = wkt.loads(wkt_str) 286 | # sh = affinity.scale(sh, xfact=x_sf, yfact=y_sf, origin=(0, 0, 0)) 287 | # polygons.append(sh) 288 | # return polygons 289 | 290 | 291 | class MultiInputSegDataIter(mx.io.DataIter): 292 | def __init__(self, image_list, batch_size, epoch_size, 293 | data_name="data", label_name="softmax_label", start_aug=True, test_list=[]): 294 | super(MultiInputSegDataIter, self).__init__() 295 | print('Data iterator initialization..') 296 | self.data_name = data_name 297 | self.label_name = label_name 298 | global y_mask, A_data, M_data, P_data, a_size, p_size, m_size, l_size 299 | self.batch_size = batch_size 300 | self.epoch_size = epoch_size 301 | 302 | self.cursor = -1 303 | self.image_data = [] 304 | self.true_raster = [] 305 | 306 | for image_id in image_list: 307 | a, m, p = get_data(image_id, a_size, m_size, p_size, sf) 308 | A_data.append(a) 309 | M_data.append(m) 310 | P_data.append(p) 311 | mask = get_raster(image_id, l_size*sf, l_size*sf) 312 | y_mask.append(mask) 313 | 314 | # test to train 315 | # for image_id in test_list: 316 | # a, m, p = get_data(image_id, a_size, m_size, p_size, sf) 317 | # A_data.append(a) 318 | # M_data.append(m) 319 | # P_data.append(p) 320 | # polygons = get_test_polygons(image_id, p_size*sf, p_size*sf) 321 | # y_mask.append(rasterize_polgygon(polygons, p_size*sf, p_size*sf)) 322 | 323 | print('number of maps(train + test): {}'.format(len(y_mask))) 324 | global sampler 325 | sampler = CropSampler(y_mask) 326 | print('Sampler is ready.') 327 | 328 | self.a_data_depth = A_data[0].shape[2] 329 | self.m_data_depth = M_data[0].shape[2] 330 | self.p_data_depth = P_data[0].shape[2] 331 | self.label_depth = y_mask[0].shape[2] 332 | 333 | self.thread_number = 4 334 | self.prefetch_threads = [] 335 | 336 | if not start_aug: 337 | return 338 | print('Data loaded.') 339 | 340 | self.manager = multiprocessing.Manager() 341 | self.q = self.manager.Queue(1024) 342 | for i in range(self.thread_number): 343 | pt = multiprocessing.Process(target=self.gen, args=[self.q]) 344 | pt.daemon = True 345 | pt.start() 346 | self.prefetch_threads.append(pt) 347 | print('Daemon prefetcher threads started.') 348 | 349 | def gen(self, q): 350 | while True: 351 | a, m, p, label = zip(*[get_random_data() 352 | for _ in range(self.batch_size)]) 353 | q.put((a, m, p, label)) 354 | 355 | def update_sampler(self, class_weights): 356 | # print(class_weights) 357 | class_weights = 1. / (0.02 + class_weights) 358 | # class_weights /= np.sum(class_weights) 359 | # class_weights = np.clip(class_weights, 0.1, 0.9) 360 | class_weights /= np.sum(class_weights) 361 | sampler.update(class_weights) 362 | # print(class_weights) 363 | 364 | @property 365 | def provide_data(self): 366 | return [('a_data', (self.batch_size, self.a_data_depth, 367 | a_size, a_size)), 368 | ('m_data', (self.batch_size, self.m_data_depth, 369 | m_size, m_size)), 370 | ('p_data', (self.batch_size, self.p_data_depth, 371 | p_size, p_size))] 372 | 373 | @property 374 | def provide_label(self): 375 | return [('softmax_label', (self.batch_size, 1 if n_out == 11 else 10, 376 | l_size, l_size))] 377 | 378 | def get_batch_size(self): 379 | return self.batch_size 380 | 381 | def reset(self): 382 | self.cursor = -1 383 | 384 | def iter_next(self): 385 | self.cursor += 1 386 | if(self.cursor < self.epoch_size): 387 | return True 388 | else: 389 | return False 390 | 391 | def next(self): 392 | if self.iter_next(): 393 | a, m, p, label = self.q.get(True) 394 | 395 | data_all = [mx.nd.array(a), mx.nd.array(m), mx.nd.array(p)] 396 | label_all = [mx.nd.array(label)] 397 | data_names = ['a_data', 'm_data', 'p_data'] 398 | label_names = ['softmax_label'] 399 | 400 | return Batch(data_names, data_all, label_names, label_all) 401 | else: 402 | raise StopIteration 403 | 404 | def close(self): 405 | for t in self.prefetch_threads: 406 | t.terminate() 407 | self.manager.shutdown() 408 | 409 | #train_iter = SegDataIter(['6040_2_2'], 8, 128) 410 | #train_iter = mx.io.PrefetchingIter(train_iter) 411 | # 412 | #n_epoch = 100 413 | #ts = time.time() 414 | #for epoch in range(n_epoch): 415 | # for i, batch in enumerate(train_iter): 416 | # data = batch.data 417 | # print('epoch time: {}'.format(time.time() - ts)) 418 | # train_iter.reset() 419 | # ts = time.time() 420 | #train_iter.close() 421 | --------------------------------------------------------------------------------