├── .gitignore ├── __init__.py ├── bow.py ├── compare_results.py ├── hierarchical_segmentation.py ├── kraehenbuehl_potentials.py ├── load_eval.py ├── mnist_svm_experiment.py ├── msrc ├── __init__.py ├── example_void_crf.py ├── hierarchical_crf.py ├── hierarchical_helpers.py ├── ignore_void_crf.py ├── msrc_crf.py ├── msrc_helpers.py ├── msrc_svm.py └── parts.py ├── nyu ├── __init__.py ├── nyu_baselines.py ├── nyu_crf.py ├── nyu_helpers.py └── nyu_hierarchical.py ├── pascal ├── __init__.py ├── hierarchical_crf.py ├── pascal_baselines.py ├── pascal_bow.py ├── pascal_colors.txt ├── pascal_crf.py ├── pascal_helpers.py ├── tests_helpers.py └── visualize_segment_sps.py ├── plotting.py ├── tests ├── __init__.py └── test_ignore_void_crf.py ├── toy_experiments ├── directional_bars.py ├── directional_bars_joint.py ├── harder_crosses.py ├── simple_crosses.py └── square_with_hole.py ├── utils.py ├── visualize_edge_features.py └── visualize_new_gt.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.png 2 | *.pyc 3 | cache 4 | *.pickle 5 | *.bmp 6 | *.png 7 | *.data 8 | *.swp 9 | -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amueller/segmentation/f6d252135d91985002fdaccc59214a7199dabed9/__init__.py -------------------------------------------------------------------------------- /bow.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from scipy import sparse 3 | 4 | #from quickshift import quickshift 5 | from vlfeat import vl_dsift 6 | from skimage.color import rgb2gray 7 | from sklearn.utils import shuffle 8 | 9 | from sklearn.cluster import MiniBatchKMeans 10 | from sklearn.preprocessing import Normalizer 11 | #from sklearn.cluster import KMeans 12 | #import os 13 | 14 | from joblib import Memory 15 | #from joblib import Parallel, delayed 16 | 17 | from utils import DataBunch, gt_in_sp 18 | 19 | memory = Memory(cachedir="/tmp/cache", verbose=0) 20 | 21 | from IPython.core.debugger import Tracer 22 | tracer = Tracer() 23 | 24 | 25 | from sklearn.metrics.pairwise import chi2_kernel 26 | 27 | 28 | class Chi2Kernel(object): 29 | def __init__(self, gamma=1): 30 | self.gamma = gamma 31 | 32 | def __call__(self, x, y): 33 | return chi2_kernel(x, y, gamma=self.gamma) 34 | 35 | def __repr__(self): 36 | return "Chi2Kernel(gamma=%f)" % self.gamma 37 | 38 | 39 | @memory.cache 40 | def color_descriptors(images, spixel, dataset, vq): 41 | n_words = 300 42 | some_colors = [] 43 | for f in images: 44 | image = dataset.get_image(f) 45 | some_colors.append(image.reshape(-1, 3)[::10, :]) 46 | if vq is None: 47 | vq = MiniBatchKMeans(n_clusters=n_words, verbose=1, init='random', 48 | batch_size=2 * n_words, random_state=1) 49 | vq.fit(shuffle(np.vstack(some_colors))) 50 | 51 | bows = [] 52 | for f, sp in zip(images, spixel): 53 | image = dataset.get_image(f) 54 | words = vq.predict(image.reshape(-1, 3).astype(np.float)) 55 | bins = [np.arange(np.max(sp) + 2), np.arange(n_words + 1)] 56 | bow = np.histogram2d(sp.ravel(), words, bins=bins)[0] 57 | bows.append(bow) 58 | return vq, bows 59 | 60 | 61 | def rgsift(image): 62 | from skimage import img_as_float 63 | shaped_image = img_as_float(image) 64 | gray = rgb2gray(image) 65 | s = shaped_image.sum(axis=2) 66 | red = shaped_image[:, :, 0] / (s + 1e-5) 67 | green = shaped_image[:, :, 1] / (s + 1e-5) 68 | descs = [] 69 | for channel in [gray, red, green]: 70 | loc, desc = vl_dsift(channel, step=4, size=6) 71 | descs.append(desc.T.copy()) 72 | return loc, np.hstack(descs) 73 | 74 | 75 | @memory.cache 76 | def extract_spatial_pyramid(images, dataset, vq=None, n_words=1000): 77 | descriptors, locations = sift_descriptors(images, dataset) 78 | if vq is None: 79 | vq = MiniBatchKMeans(n_clusters=n_words, verbose=1, init='random', 80 | batch_size=2 * n_words, compute_labels=False, 81 | reassignment_ratio=0.0, random_state=1, n_init=3) 82 | #vq = KMeans(n_clusters=n_words, verbose=10, init='random') 83 | vq.fit(shuffle(np.vstack(descriptors))) 84 | else: 85 | n_words = vq.n_clusters 86 | 87 | pyramids = [] 88 | for descr, locs in zip(descriptors, locations): 89 | words = vq.predict(descr) 90 | global_ = np.bincount(words, minlength=n_words).astype(np.float) 91 | global_ /= max(global_.sum(), 1) 92 | third_of_image = locs[1].max() // 3 + 1 93 | stripe_indicator = locs[1] // third_of_image 94 | inds = np.vstack([stripe_indicator, words]) 95 | stripe_hists = sparse.coo_matrix((np.ones(len(words)), inds), 96 | shape=(3, n_words)).toarray() 97 | 98 | stripe_hists = [x / max(x.sum(), 1) for x in stripe_hists] 99 | pyramids.append(np.hstack([np.hstack(stripe_hists), global_])) 100 | 101 | return vq, np.vstack(pyramids) 102 | 103 | 104 | @memory.cache 105 | def sift_descriptors(images, dataset): 106 | descs = [] 107 | coordinates = [] 108 | print("computing sift descriptors") 109 | for f in images: 110 | print("processing image %s" % f) 111 | image = dataset.get_image(f) 112 | #coords, sift = rgsift(image) 113 | #tracer() 114 | gray_image = rgb2gray(image) 115 | coords, sift = vl_dsift(gray_image, step=3, size=4) 116 | #coords2, sift2 = vl_dsift(gray_image, step=3, size=8) 117 | #coords3, sift3 = vl_dsift(gray_image, step=3, size=16) 118 | #tracer() 119 | #sift = np.hstack([sift, sift2, sift3]) 120 | #coords = np.hstack([coords, coords2, coords3]) 121 | descs.append(sift.T) 122 | coordinates.append(coords) 123 | return descs, coordinates 124 | 125 | 126 | @memory.cache 127 | def color_sift_descriptors(images, dataset): 128 | descs = [] 129 | coordinates = [] 130 | print("computing color sift descriptors") 131 | for f in images: 132 | print("processing image %s" % f) 133 | image = dataset.get_image(f) 134 | coords, sift = rgsift(image) 135 | descs.append(sift) 136 | coordinates.append(coords) 137 | return descs, coordinates 138 | 139 | 140 | @memory.cache 141 | def bag_of_words(descs, spixel, coordinates, vq=None, n_words=1000): 142 | """Compute bag of words from sift descriptors and superpixels. 143 | 144 | Parameters 145 | ---------- 146 | descs : list of ndarray 147 | For each image, array of sift descriptors. 148 | 149 | spixel : list of ndarray 150 | For each image, superpixel index for each pixel. 151 | 152 | coordinates : list of ndarray 153 | For each image, coordinate positions of sift descriptors. 154 | 155 | vq : Clustering Object or None. 156 | Fitted clustering object or None if clustering should be performed. 157 | 158 | n_words : int 159 | Number of words, i.e. clusters to find. Default=1000. 160 | """ 161 | 162 | if vq is None: 163 | vq = MiniBatchKMeans(n_clusters=n_words, verbose=1, init='random', 164 | batch_size=2 * n_words, compute_labels=False, 165 | reassignment_ratio=0.0, random_state=1, n_init=3) 166 | #vq = KMeans(n_clusters=n_words, verbose=10, init='random') 167 | descs_stacked = shuffle(np.vstack(descs)) 168 | if len(descs_stacked) > 1e6: 169 | descs_stacked = descs_stacked[::10] 170 | vq.fit(descs_stacked) 171 | else: 172 | n_words = vq.n_clusters 173 | 174 | bows = [] 175 | for desc, sp, coords in zip(descs, spixel, coordinates): 176 | coords = coords.astype(np.int) 177 | desc_in_sp = sp[coords[1], coords[0]] 178 | bins = [np.arange(np.max(sp) + 2), np.arange(n_words + 1)] 179 | bow = np.histogram2d(desc_in_sp, vq.predict(desc), bins=bins)[0] 180 | bows.append(bow) 181 | return vq, bows 182 | 183 | 184 | class SiftBOW(object): 185 | def __init__(self, dataset, n_words=300, add_global_desc=True, 186 | color_sift=False): 187 | self.dataset = dataset 188 | self.n_words = n_words 189 | self.add_global_desc = add_global_desc 190 | self.normalizer = Normalizer(norm='l1') 191 | self.color_sift = color_sift 192 | if self.color_sift: 193 | self.feature_extractor = color_sift_descriptors 194 | else: 195 | self.feature_extractor = sift_descriptors 196 | 197 | def fit_transform(self, image_names, superpixels): 198 | descriptors, coordinates = self.feature_extractor(image_names, 199 | self.dataset) 200 | print("end sift descriptors") 201 | vq, X = bag_of_words(descriptors, superpixels, coordinates) 202 | X = [self.normalizer.transform(x) for x in X] 203 | 204 | self.vq_ = vq 205 | Y = [gt_in_sp(self.dataset, f, sp) for f, sp in zip(image_names, 206 | superpixels)] 207 | return DataBunch(X, Y, image_names, superpixels) 208 | 209 | def fit(self, image_names, spixel): 210 | self.fit_predict(image_names, spixel) 211 | return self 212 | 213 | def transform(self, image_names, superpixels): 214 | descriptors, coordinates = self.feature_extractor(image_names, 215 | self.dataset) 216 | _, X = bag_of_words(descriptors, superpixels, coordinates, vq=self.vq_) 217 | Y = [gt_in_sp(self.dataset, f, sp) for f, sp in zip(image_names, 218 | superpixels)] 219 | X = [self.normalizer.transform(x) for x in X] 220 | return DataBunch(X, Y, image_names, superpixels) 221 | -------------------------------------------------------------------------------- /compare_results.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import sys 3 | import os 4 | import numpy as np 5 | import matplotlib.pyplot as plt 6 | 7 | from pystruct.utils import SaveLogger 8 | from datasets.nyu import NYUSegmentation 9 | 10 | from nyu.nyu_helpers import load_nyu 11 | from utils import add_edges, add_edge_features, eval_on_pixels 12 | 13 | 14 | def main(): 15 | argv = sys.argv 16 | print("loading %s ..." % argv[1]) 17 | ssvm1 = SaveLogger(file_name=argv[1]).load() 18 | ssvm2 = SaveLogger(file_name=argv[2]).load() 19 | 20 | data_str = 'val' 21 | if len(argv) <= 3: 22 | raise ValueError("Need a folder name for plotting.") 23 | print("loading data...") 24 | data = load_nyu(data_str, n_sp=500) 25 | dataset = NYUSegmentation() 26 | print("done") 27 | data1 = add_edges(data, kind="pairwise") 28 | data2 = add_edges(data, kind="pairwise") 29 | data1 = add_edge_features(dataset, data1) 30 | data2 = add_edge_features(dataset, data2, depth_diff=True) 31 | Y_pred1 = ssvm1.predict(data1.X) 32 | Y_pred2 = ssvm2.predict(data2.X) 33 | folder = argv[3] 34 | 35 | if not os.path.exists(folder): 36 | os.mkdir(folder) 37 | 38 | np.random.seed(0) 39 | for image_name, superpixels, y_pred1, y_pred2 in zip(data.file_names, 40 | data.superpixels, 41 | Y_pred1, Y_pred2): 42 | if np.all(y_pred1 == y_pred2): 43 | continue 44 | gt = dataset.get_ground_truth(image_name) 45 | perf1 = eval_on_pixels(dataset, [gt], [y_pred1[superpixels]], 46 | print_results=False)[0] 47 | perf1 = np.mean(perf1[np.isfinite(perf1)]) 48 | 49 | perf2 = eval_on_pixels(dataset, [gt], [y_pred2[superpixels]], 50 | print_results=False)[0] 51 | perf2 = np.mean(perf2[np.isfinite(perf2)]) 52 | if np.abs(perf1 - perf2) < 2: 53 | continue 54 | 55 | image = dataset.get_image(image_name) 56 | fig, axes = plt.subplots(2, 3, figsize=(12, 6)) 57 | axes[0, 0].imshow(image) 58 | axes[0, 0].imshow((y_pred1 != y_pred2)[superpixels], vmin=0, vmax=1, 59 | alpha=.7) 60 | 61 | axes[0, 1].set_title("ground truth") 62 | axes[0, 1].imshow(image) 63 | axes[0, 1].imshow(gt, alpha=.7, cmap=dataset.cmap, vmin=0, 64 | vmax=dataset.cmap.N) 65 | axes[1, 0].set_title("%.2f" % perf1) 66 | axes[1, 0].imshow(image) 67 | axes[1, 0].imshow(y_pred1[superpixels], vmin=0, vmax=dataset.cmap.N, 68 | alpha=.7, cmap=dataset.cmap) 69 | 70 | axes[1, 1].set_title("%.2f" % perf2) 71 | axes[1, 1].imshow(image) 72 | axes[1, 1].imshow(y_pred2[superpixels], alpha=.7, cmap=dataset.cmap, 73 | vmin=0, vmax=dataset.cmap.N) 74 | present_y = np.unique(np.hstack([y_pred1, y_pred2, np.unique(gt)])) 75 | present_y = np.array([y_ for y_ in present_y if y_ != 76 | dataset.void_label]) 77 | axes[0, 2].imshow(present_y[:, np.newaxis], interpolation='nearest', 78 | cmap=dataset.cmap, vmin=0, vmax=dataset.cmap.N) 79 | for i, c in enumerate(present_y): 80 | axes[0, 2].text(1, i, dataset.classes[c]) 81 | for ax in axes.ravel(): 82 | ax.set_xticks(()) 83 | ax.set_yticks(()) 84 | axes[1, 2].set_visible(False) 85 | fig.savefig(folder + "/%s.png" % image_name, bbox_inches="tight") 86 | plt.close(fig) 87 | 88 | 89 | if __name__ == "__main__": 90 | main() 91 | -------------------------------------------------------------------------------- /hierarchical_segmentation.py: -------------------------------------------------------------------------------- 1 | from collections import namedtuple 2 | import numpy as np 3 | from scipy import sparse 4 | 5 | from sklearn.cluster import Ward 6 | #from sklearn.cluster import KMeans 7 | from sklearn.externals.joblib import Memory 8 | 9 | 10 | memory = Memory(cachedir="/tmp/cache", verbose=0) 11 | 12 | HierarchicalDataBunch = namedtuple('HierarchicalDataBunch', 'X, Y, file_names,' 13 | 'superpixels, segments') 14 | 15 | 16 | def make_hierarchy_edges(segments, superpixels): 17 | all_edges = [] 18 | for seg, sps in zip(segments, superpixels): 19 | seg = seg[sps] 20 | edges = np.vstack([seg.ravel() + sps.max() + 1, sps.ravel()]) 21 | edge_matrix = sparse.coo_matrix((np.ones(edges.shape[1]), edges)) 22 | # make edges unique 23 | edges = np.vstack(edge_matrix.tocsr().nonzero()).T 24 | all_edges.append(np.sort(edges, axis=1)) 25 | return all_edges 26 | 27 | 28 | def get_colors(img, sps): 29 | reds = np.bincount(sps.ravel(), weights=img[:, :, 0].ravel()) 30 | greens = np.bincount(sps.ravel(), weights=img[:, :, 1].ravel()) 31 | blues = np.bincount(sps.ravel(), weights=img[:, :, 2].ravel()) 32 | counts = np.bincount(sps.ravel()) 33 | reds /= counts 34 | greens /= counts 35 | blues /= counts 36 | return np.vstack([reds, greens, blues]).T 37 | 38 | 39 | def get_centers(sps): 40 | gridx, gridy = np.mgrid[:sps.shape[0], :sps.shape[1]] 41 | n_vertices = len(np.unique(sps)) 42 | centers = np.zeros((n_vertices, 2)) 43 | for v in xrange(n_vertices): 44 | centers[v] = [gridy[sps == v].mean(), gridx[sps == v].mean()] 45 | return centers 46 | 47 | 48 | def get_km_segments(x, image, sps, n_segments=25): 49 | if len(x) == 2: 50 | feats, edges = x 51 | else: 52 | feats, edges, _ = x 53 | colors_ = get_colors(image, sps) 54 | centers = get_centers(sps) 55 | n_spixel = len(feats) 56 | graph = sparse.coo_matrix((np.ones(edges.shape[0]), edges.T), 57 | shape=(n_spixel, n_spixel)) 58 | ward = Ward(n_clusters=n_segments, connectivity=graph + graph.T) 59 | #km = KMeans(n_clusters=n_segments) 60 | color_feats = np.hstack([colors_, centers * .5]) 61 | #return km.fit_predict(color_feats) 62 | return ward.fit_predict(color_feats) 63 | 64 | 65 | @memory.cache 66 | def get_segment_features(x, y, image, sps): 67 | segments = get_km_segments(x, image, sps) 68 | if len(x) == 2: 69 | feats, edges = x 70 | else: 71 | feats, edges, _ = x 72 | segment_edges = segments[edges] 73 | # make direction of edges unique 74 | segment_edges = np.sort(segment_edges, axis=1) 75 | # to get rid of duplicate edges, self edges, become sparse matrix 76 | graph = sparse.coo_matrix((np.ones(segment_edges.shape[0]), 77 | segment_edges.T)) 78 | # conversion removes duplicates 79 | graph = graph.tocsr() 80 | # remove self edges at diag 81 | graph.setdiag(np.zeros(graph.shape[0])) 82 | segment_edges = np.vstack(graph.nonzero()).T 83 | 84 | features = [np.mean(feats[segments == i], axis=0) for i in 85 | np.unique(segments)] 86 | labels = [np.argmax(np.bincount(y[segments == i])) for i in 87 | np.unique(segments)] 88 | return segments, features, np.array(labels), edges 89 | 90 | 91 | @memory.cache 92 | def make_hierarchical_data(dataset, data, lateral=False, latent=False, 93 | latent_lateral=False, add_edge_features=False): 94 | images = [dataset.get_image(f) for f in data.file_names] 95 | segment_features = [get_segment_features(*stuff) 96 | for stuff in zip(data.X, data.Y, 97 | images, data.superpixels)] 98 | all_segments, all_features, all_labels, segment_edges =\ 99 | zip(*segment_features) 100 | 101 | all_edges = make_hierarchy_edges(all_segments, data.superpixels) 102 | 103 | X_stacked, Y_stacked = [], [] 104 | for x, y, feat, edges, labels in zip(data.X, data.Y, all_features, 105 | all_edges, all_labels): 106 | edges_stacked = np.vstack([x[1], edges] if lateral else edges) 107 | 108 | if latent: 109 | y_stacked = y 110 | n_nodes = len(x[0]) 111 | if latent_lateral: 112 | hierarchy = sparse.csr_matrix( 113 | (np.ones(len(edges)), edges.T), shape=(n_nodes + len(feat), 114 | n_nodes)) 115 | visible_lateral = sparse.csr_matrix( 116 | (np.ones(len(x[1])), x[1].T), shape=(n_nodes, n_nodes)) 117 | graph_latent_lateral = (hierarchy * visible_lateral * 118 | hierarchy.T) 119 | # make symmetric 120 | graph_latent_lateral = (graph_latent_lateral + 121 | graph_latent_lateral.T) 122 | edges_latent_lateral = np.c_[graph_latent_lateral.nonzero()] 123 | # remove self-edges and make sorted 124 | edges_latent_lateral = \ 125 | edges_latent_lateral[edges_latent_lateral[:, 0] < 126 | edges_latent_lateral[:, 1]] 127 | edges_stacked = np.vstack([edges_stacked, 128 | edges_latent_lateral]) 129 | if add_edge_features: 130 | edge_features = x[2] 131 | # we assume that thie fist edge feature is symmetric, I guess.. 132 | n_edge_features = x[2].shape[1] 133 | edge_features_new = np.zeros(n_edge_features) 134 | edge_features_new[0] = 1 135 | edge_features_new = np.repeat(edge_features_new[np.newaxis, :], 136 | len(edges), axis=0) 137 | edge_features_stacked = np.vstack([edge_features, 138 | edge_features_new]) 139 | x_stacked = (x[0], edges_stacked, edge_features_stacked, 140 | len(feat)) 141 | else: 142 | x_stacked = (x[0], edges_stacked, len(feat)) 143 | else: 144 | if latent_lateral: 145 | raise ValueError("wut?") 146 | feat = np.vstack(x[0], feat) 147 | y_stacked = np.hstack([y, labels]) 148 | x_stacked = (feat, edges_stacked) 149 | 150 | X_stacked.append(x_stacked) 151 | Y_stacked.append(y_stacked) 152 | 153 | return HierarchicalDataBunch(X_stacked, Y_stacked, data.file_names, 154 | data.superpixels, all_segments) 155 | 156 | 157 | def main(): 158 | from pascal.pascal_helpers import load_pascal 159 | from datasets.pascal import PascalSegmentation 160 | from utils import add_edges 161 | from scipy.misc import imsave 162 | from skimage.segmentation import mark_boundaries 163 | 164 | ds = PascalSegmentation() 165 | data = load_pascal("train1") 166 | 167 | data = add_edges(data, independent=False) 168 | #X, Y, image_names, images, all_superpixels = load_data( 169 | #"train", independent=False) 170 | for x, name, sps in zip(data.X, data.file_names, data.superpixels): 171 | segments = get_km_segments(x, ds.get_image(name), sps, n_segments=25) 172 | boundary_image = mark_boundaries(mark_boundaries(ds.get_image(name), 173 | sps), segments[sps], 174 | color=[1, 0, 0]) 175 | imsave("hierarchy_sp_own_25/%s.png" % name, boundary_image) 176 | 177 | 178 | def plot_results_hierarchy(dataset, data, Y_pred, folder="figures"): 179 | import os 180 | import matplotlib.pyplot as plt 181 | from skimage.segmentation import mark_boundaries 182 | if not os.path.exists(folder): 183 | os.mkdir(folder) 184 | import matplotlib.colors as cl 185 | np.random.seed(0) 186 | random_colormap = cl.ListedColormap(np.random.uniform(size=(100, 3))) 187 | for stuff in zip(data.file_names, data.superpixels, 188 | data.segments, data.Y, Y_pred): 189 | image_name, superpixels, segments, y, y_pred = stuff 190 | image = dataset.get_image(image_name) 191 | h = y_pred[len(y):] 192 | y_pred = y_pred[:len(y)] 193 | 194 | fig, axes = plt.subplots(2, 3, figsize=(12, 6)) 195 | 196 | axes[0, 0].imshow(image) 197 | axes[0, 1].set_title("ground truth") 198 | axes[0, 1].imshow(image) 199 | gt = dataset.get_ground_truth(image_name) 200 | axes[0, 1].imshow(gt, alpha=.7, cmap=dataset.cmap) 201 | axes[1, 0].set_title("sp ground truth") 202 | axes[1, 0].imshow(image) 203 | axes[1, 0].imshow(y[superpixels], vmin=0, vmax=23, alpha=.7, 204 | cmap=dataset.cmap) 205 | 206 | axes[1, 1].set_title("prediction") 207 | axes[1, 1].imshow(image) 208 | axes[1, 1].imshow(y_pred[superpixels], vmin=0, vmax=23, 209 | alpha=.7, cmap=dataset.cmap) 210 | present_y = np.unique(np.hstack([y, y_pred])) 211 | 212 | vmax = np.max(np.hstack(Y_pred)) 213 | vmin = np.min(np.hstack(Y_pred)) 214 | axes[1, 2].imshow(mark_boundaries(image, segments[superpixels])) 215 | axes[1, 2].imshow(h[segments[superpixels]], vmin=vmin, vmax=vmax, 216 | alpha=.7, cmap=random_colormap) 217 | 218 | axes[0, 2].imshow(present_y[np.newaxis, :], interpolation='nearest', 219 | alpha=.7, cmap=dataset.cmap) 220 | for i, c in enumerate(present_y): 221 | axes[0, 2].text(1, i, dataset.classes[c]) 222 | for ax in axes.ravel(): 223 | ax.set_xticks(()) 224 | ax.set_yticks(()) 225 | fig.savefig(folder + "/%s.png" % image_name, bbox_inches="tight") 226 | plt.close(fig) 227 | 228 | 229 | if __name__ == "__main__": 230 | main() 231 | -------------------------------------------------------------------------------- /kraehenbuehl_potentials.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | 4 | from datasets.msrc import MSRCDataset 5 | from msrc_helpers import (load_kraehenbuehl, load_data, eval_on_pixels, 6 | get_kraehenbuehl_pot_sp, add_kraehenbuehl_features) 7 | 8 | 9 | def pixelwise(): 10 | msrc = MSRCDataset() 11 | train = msrc.get_split('train') 12 | predictions = [] 13 | for filename in train: 14 | probs = load_kraehenbuehl(filename) 15 | prediction = np.argmax(probs, axis=-1) 16 | predictions.append(prediction) 17 | 18 | msrc.eval_pixel_performance(train, predictions) 19 | #plt.matshow(results['confusion']) 20 | #plt.show() 21 | 22 | 23 | def on_slic_superpixels(): 24 | data = load_data('train', independent=True) 25 | probs = get_kraehenbuehl_pot_sp(data) 26 | results = eval_on_pixels(data, [np.argmax(prob, axis=-1) for prob in 27 | probs]) 28 | plt.matshow(results['confusion']) 29 | plt.show() 30 | 31 | 32 | def with_aureliens_potentials_svm(test=False): 33 | data = load_data('train', independent=True) 34 | data = add_kraehenbuehl_features(data) 35 | features = [x[0] for x in data.X] 36 | y = np.hstack(data.Y) 37 | 38 | if test: 39 | data_ = load_data('val', independent=True) 40 | data_ = add_kraehenbuehl_features(data_) 41 | features.extend([x[0] for x in data.X]) 42 | y = np.hstack([y, np.hstack(data_.Y)]) 43 | 44 | new_features_flat = np.vstack(features) 45 | from sklearn.svm import LinearSVC 46 | print("training svm") 47 | svm = LinearSVC(C=.001, dual=False, class_weight='auto') 48 | svm.fit(new_features_flat[y != 21], y[y != 21]) 49 | print(svm.score(new_features_flat[y != 21], y[y != 21])) 50 | print("evaluating") 51 | eval_on_pixels(data, [svm.predict(x) for x in features]) 52 | 53 | if test: 54 | print("test data") 55 | data_val = load_data('test', independent=True) 56 | else: 57 | data_val = load_data('val', independent=True) 58 | 59 | data_val = add_kraehenbuehl_features(data_val) 60 | features_val = [x[0] for x in data_val.X] 61 | eval_on_pixels(data_val, [svm.predict(x) for x in features_val]) 62 | #msrc = MSRCDataset() 63 | 64 | 65 | if __name__ == "__main__": 66 | #on_slic_superpixels() 67 | #with_aureliens_potentials_svm(test=True) 68 | pixelwise() 69 | -------------------------------------------------------------------------------- /load_eval.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import sys 3 | 4 | import matplotlib.pyplot as plt 5 | import numpy as np 6 | 7 | 8 | #from sklearn.metrics import confusion_matrix 9 | 10 | from pystruct.utils import SaveLogger 11 | from pystruct.models import LatentNodeCRF 12 | 13 | from msrc import msrc_helpers 14 | from pascal import pascal_helpers 15 | from nyu import nyu_helpers 16 | from latent_crf_experiments.hierarchical_segmentation import \ 17 | make_hierarchical_data 18 | 19 | from datasets.msrc import MSRC21Dataset 20 | from datasets.pascal import PascalSegmentation 21 | from datasets.nyu import NYUSegmentation 22 | 23 | from utils import add_edges, eval_on_sp, add_edge_features 24 | from plotting import plot_results 25 | 26 | 27 | 28 | def main(): 29 | argv = sys.argv 30 | print("loading %s ..." % argv[1]) 31 | ssvm = SaveLogger(file_name=argv[1]).load() 32 | if hasattr(ssvm, 'problem'): 33 | ssvm.model = ssvm.problem 34 | print(ssvm) 35 | if hasattr(ssvm, 'base_ssvm'): 36 | ssvm = ssvm.base_ssvm 37 | print("Iterations: %d" % len(ssvm.objective_curve_)) 38 | print("Objective: %f" % ssvm.objective_curve_[-1]) 39 | inference_run = None 40 | if hasattr(ssvm, 'cached_constraint_'): 41 | inference_run = ~np.array(ssvm.cached_constraint_) 42 | print("Gap: %f" % 43 | (np.array(ssvm.primal_objective_curve_)[inference_run][-1] - 44 | ssvm.objective_curve_[-1])) 45 | 46 | if len(argv) <= 2: 47 | argv.append("acc") 48 | 49 | if len(argv) <= 3: 50 | dataset = 'nyu' 51 | else: 52 | dataset = argv[3] 53 | 54 | if argv[2] == 'acc': 55 | 56 | ssvm.n_jobs = 1 57 | 58 | for data_str, title in zip(["train", "val"], 59 | ["TRAINING SET", "VALIDATION SET"]): 60 | print(title) 61 | edge_type = "pairwise" 62 | 63 | if dataset == 'msrc': 64 | ds = MSRC21Dataset() 65 | data = msrc_helpers.load_data(data_str, which="piecewise_new") 66 | #data = add_kraehenbuehl_features(data, which="train_30px") 67 | data = msrc_helpers.add_kraehenbuehl_features(data, which="train") 68 | elif dataset == 'pascal': 69 | ds = PascalSegmentation() 70 | data = pascal_helpers.load_pascal(data_str, sp_type="cpmc") 71 | #data = pascal_helpers.load_pascal(data_str) 72 | elif dataset == 'nyu': 73 | ds = NYUSegmentation() 74 | data = nyu_helpers.load_nyu(data_str, n_sp=500, sp='rgbd') 75 | else: 76 | raise ValueError("Excepted dataset to be 'nyu', 'pascal' or 'msrc'," 77 | " got %s." % dataset) 78 | 79 | if type(ssvm.model).__name__ == "LatentNodeCRF": 80 | print("making data hierarchical") 81 | data = pascal_helpers.make_cpmc_hierarchy(ds, data) 82 | #data = make_hierarchical_data( 83 | #ds, data, lateral=True, latent=True, latent_lateral=False, 84 | #add_edge_features=False) 85 | else: 86 | data = add_edges(data, edge_type) 87 | 88 | if type(ssvm.model).__name__ == 'EdgeFeatureGraphCRF': 89 | data = add_edge_features(ds, data, depth_diff=True, normal_angles=True) 90 | 91 | if type(ssvm.model).__name__ == "EdgeFeatureLatentNodeCRF": 92 | data = add_edge_features(ds, data) 93 | data = make_hierarchical_data( 94 | ds, data, lateral=True, latent=True, latent_lateral=False, 95 | add_edge_features=True) 96 | #ssvm.model.inference_method = "qpbo" 97 | Y_pred = ssvm.predict(data.X) 98 | 99 | if isinstance(ssvm.model, LatentNodeCRF): 100 | Y_pred = [ssvm.model.label_from_latent(h) for h in Y_pred] 101 | Y_flat = np.hstack(data.Y) 102 | 103 | print("superpixel accuracy: %.2f" 104 | % (np.mean((np.hstack(Y_pred) == Y_flat)[Y_flat != ds.void_label]) * 100)) 105 | 106 | if dataset == 'msrc': 107 | res = msrc_helpers.eval_on_pixels(data, Y_pred, 108 | print_results=True) 109 | print("global: %.2f, average: %.2f" % (res['global'] * 100, 110 | res['average'] * 100)) 111 | #msrc_helpers.plot_confusion_matrix(res['confusion']) 112 | else: 113 | hamming, jaccard = eval_on_sp(ds, data, Y_pred, 114 | print_results=True) 115 | print("Jaccard: %.2f, Hamming: %.2f" % (jaccard.mean(), 116 | hamming.mean())) 117 | 118 | plt.show() 119 | 120 | elif argv[2] == 'plot': 121 | data_str = 'val' 122 | if len(argv) <= 4: 123 | raise ValueError("Need a folder name for plotting.") 124 | if dataset == "msrc": 125 | ds = MSRC21Dataset() 126 | data = msrc_helpers.load_data(data_str, which="piecewise") 127 | data = add_edges(data, independent=False) 128 | data = msrc_helpers.add_kraehenbuehl_features( 129 | data, which="train_30px") 130 | data = msrc_helpers.add_kraehenbuehl_features( 131 | data, which="train") 132 | 133 | elif dataset == "pascal": 134 | ds = PascalSegmentation() 135 | data = pascal_helpers.load_pascal("val") 136 | data = add_edges(data) 137 | 138 | elif dataset == "nyu": 139 | ds = NYUSegmentation() 140 | data = nyu_helpers.load_nyu("test") 141 | data = add_edges(data) 142 | 143 | if type(ssvm.model).__name__ == 'EdgeFeatureGraphCRF': 144 | data = add_edge_features(ds, data, depth_diff=True, normal_angles=True) 145 | Y_pred = ssvm.predict(data.X) 146 | 147 | plot_results(ds, data, Y_pred, argv[4]) 148 | 149 | 150 | if __name__ == "__main__": 151 | main() 152 | -------------------------------------------------------------------------------- /mnist_svm_experiment.py: -------------------------------------------------------------------------------- 1 | from sklearn.datasets import fetch_mldata 2 | from sklearn.utils import shuffle 3 | 4 | from pystruct.problems import CrammerSingerSVMProblem 5 | #from pystruct.learners import SubgradientStructuredSVM 6 | #from pystruct.learners import StructuredSVM 7 | from pystruct.learners import OneSlackSSVM 8 | 9 | mnist = fetch_mldata("MNIST original") 10 | 11 | X, y = mnist.data, mnist.target 12 | X = X / 255. 13 | 14 | X_train, y_train = X[:60000], y[:60000] 15 | X_test, y_test = X[60000:], y[60000:] 16 | 17 | X_train, y_train = shuffle(X_train, y_train) 18 | 19 | pblm = CrammerSingerSVMProblem(n_classes=10, n_features=28 ** 2) 20 | #svm = SubgradientStructuredSVM(pblm, verbose=10, n_jobs=1, plot=True, 21 | #max_iter=10, batch=False, learning_rate=0.0001, 22 | #momentum=0) 23 | #svm = SubgradientStructuredSVM(pblm, verbose=10, n_jobs=1, plot=True, 24 | #max_iter=2, batch=False, momentum=.9, 25 | #learning_rate=0.001, show_loss='true', C=1000) 26 | svm = OneSlackSSVM(pblm, verbose=2, n_jobs=1, plot=True, max_iter=2, C=1000) 27 | #svm = StructuredSVM(pblm, verbose=50, n_jobs=1, plot=True, max_iter=10, 28 | #C=1000) 29 | svm.fit(X_train, y_train) 30 | print(svm.score(X_train, y_train)) 31 | print(svm.score(X_test, y_test)) 32 | -------------------------------------------------------------------------------- /msrc/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amueller/segmentation/f6d252135d91985002fdaccc59214a7199dabed9/msrc/__init__.py -------------------------------------------------------------------------------- /msrc/example_void_crf.py: -------------------------------------------------------------------------------- 1 | # a CRF with one node is the same as a multiclass SVM 2 | # evaluation on iris dataset (really easy) 3 | 4 | from time import time 5 | import numpy as np 6 | import matplotlib.pyplot as plt 7 | 8 | from sklearn.decomposition import PCA 9 | from sklearn.datasets import load_iris 10 | from sklearn.cross_validation import train_test_split 11 | from sklearn.metrics import confusion_matrix 12 | 13 | #from pystruct.problems import GraphCRF 14 | from ignore_void_crf import IgnoreVoidCRF 15 | #from pystruct.learners import OneSlackSSVM 16 | from pystruct.learners import SubgradientStructuredSVM 17 | #from pystruct.learners import StructuredSVM 18 | 19 | #from IPython.core.debugger import Tracer 20 | #tracer = Tracer() 21 | 22 | iris = load_iris() 23 | X, y = iris.data, iris.target 24 | 25 | # make each example into a tuple of a single feature vector and an empty edge 26 | # list 27 | #X_ = [(np.atleast_2d(x), np.empty((0, 2), dtype=np.int)) for x in X] 28 | 29 | X_train_org, X_test_org, y_train, y_test = \ 30 | train_test_split(X, y) 31 | X_train = [(X_train_org, np.empty((0, 2), dtype=np.int))] 32 | y_train = y_train[np.newaxis, :] 33 | 34 | X_test = [(X_test_org, np.empty((0, 2), dtype=np.int))] 35 | y_test = y_test[np.newaxis, :] 36 | 37 | inds = np.arange(y_train.size) 38 | np.random.shuffle(inds) 39 | y_train[0, inds[:20]] = 3 40 | print(y_train.ravel()) 41 | 42 | pbl = IgnoreVoidCRF(n_features=4, n_states=4, inference_method='lp', 43 | void_label=3) 44 | #svm = StructuredSVM(pbl, verbose=1, check_constraints=True, C=100, n_jobs=1, 45 | #max_iter=1000, tol=-10) 46 | svm = SubgradientStructuredSVM(pbl, verbose=1, C=100, n_jobs=1, max_iter=1000, 47 | learning_rate=.001) 48 | 49 | 50 | start = time() 51 | svm.fit(X_train, y_train) 52 | time_svm = time() - start 53 | 54 | print(svm.w) 55 | 56 | y_pred = np.vstack(svm.predict(X_test)) 57 | print("Score with pystruct crf svm: %f (took %f seconds)" 58 | % (np.mean(y_pred == y_test), time_svm)) 59 | print(confusion_matrix(y_train.ravel(), np.hstack(svm.predict(X_train)))) 60 | print(confusion_matrix(y_test.ravel(), np.hstack(svm.predict(X_test)))) 61 | fig, axes = plt.subplots(2) 62 | 63 | X_train_pca = PCA(n_components=2).fit_transform(X_train_org) 64 | #plt.prism() 65 | axes[0].scatter(X_train_pca[:, 0], X_train_pca[:, 1], c=y_train, vmin=0, 66 | vmax=3) 67 | axes[1].scatter(X_train_pca[:, 0], X_train_pca[:, 1], 68 | c=np.hstack(svm.predict(X_train)), vmin=0, vmax=3) 69 | plt.show() 70 | -------------------------------------------------------------------------------- /msrc/hierarchical_crf.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from sklearn.utils import shuffle 4 | #from sklearn.grid_search import GridSearchCV 5 | 6 | from pystruct.models import EdgeFeatureGraphCRF, LatentNodeCRF 7 | from pystruct import learners 8 | from pystruct.utils import SaveLogger 9 | from pystruct.models.latent_node_crf import kmeans_init 10 | 11 | from hierarchical_segmentation import plot_results_hierarchy 12 | from hierarchical_helpers import make_hierarchical_data 13 | from hierarchical_helpers import load_data_global_probs 14 | from msrc_helpers import (load_data, 15 | add_kraehenbuehl_features) # , add_edge_features) 16 | 17 | from latent_crf_experiments.utils import discard_void, add_edges 18 | from IPython.core.debugger import Tracer 19 | tracer = Tracer() 20 | 21 | 22 | def svm_on_segments(C=.1, learning_rate=.001, subgradient=True): 23 | # load and prepare data 24 | lateral = True 25 | latent = True 26 | test = False 27 | #data_train = load_data(which="piecewise") 28 | #data_train = add_edges(data_train, independent=False) 29 | #data_train = add_kraehenbuehl_features(data_train, which="train_30px") 30 | #data_train = add_kraehenbuehl_features(data_train, which="train") 31 | #if lateral: 32 | #data_train = add_edge_features(data_train) 33 | data_train = load_data_global_probs(latent=latent) 34 | X_org_ = data_train.X 35 | #data_train = make_hierarchical_data(data_train, lateral=lateral, 36 | #latent=latent, latent_lateral=True) 37 | data_train = discard_void(data_train, 21, latent_features=True) 38 | X_, Y_ = data_train.X, data_train.Y 39 | # remove edges 40 | if not lateral: 41 | X_org_ = [(x[0], np.zeros((0, 2), dtype=np.int)) for x in X_org_] 42 | 43 | if test: 44 | data_val = load_data('val', which="piecewise") 45 | data_val = add_edges(data_val, independent=False) 46 | data_val = add_kraehenbuehl_features(data_val) 47 | data_val = make_hierarchical_data(data_val, lateral=lateral, 48 | latent=latent) 49 | data_val = discard_void(data_val, 21) 50 | 51 | X_.extend(data_val.X) 52 | Y_.extend(data_val.Y) 53 | 54 | n_states = 21 55 | class_weights = 1. / np.bincount(np.hstack(Y_)) 56 | class_weights *= 21. / np.sum(class_weights) 57 | experiment_name = ("latent5_features_C%f_top_node" % C) 58 | logger = SaveLogger(experiment_name + ".pickle", save_every=10) 59 | if latent: 60 | model = LatentNodeCRF(n_labels=n_states, 61 | n_features=data_train.X[0][0].shape[1], 62 | n_hidden_states=5, inference_method='qpbo' if 63 | lateral else 'dai', class_weight=class_weights, 64 | latent_node_features=True) 65 | if subgradient: 66 | ssvm = learners.LatentSubgradientSSVM( 67 | model, C=C, verbose=1, show_loss_every=10, logger=logger, 68 | n_jobs=-1, learning_rate=learning_rate, decay_exponent=1, 69 | momentum=0., max_iter=100000) 70 | else: 71 | latent_logger = SaveLogger("lssvm_" + experiment_name + 72 | "_%d.pickle", save_every=1) 73 | base_ssvm = learners.OneSlackSSVM( 74 | model, verbose=2, C=C, max_iter=100000, n_jobs=-1, tol=0.001, 75 | show_loss_every=200, inference_cache=50, logger=logger, 76 | cache_tol='auto', inactive_threshold=1e-5, break_on_bad=False, 77 | switch_to_ad3=True) 78 | ssvm = learners.LatentSSVM(base_ssvm, logger=latent_logger) 79 | warm_start = False 80 | if warm_start: 81 | ssvm = logger.load() 82 | ssvm.logger = SaveLogger(experiment_name + "_retrain.pickle", 83 | save_every=10) 84 | ssvm.max_iter = 100000 85 | ssvm.learning_rate = 0.00001 86 | ssvm.momentum = 0 87 | else: 88 | #model = GraphCRF(n_states=n_states, 89 | #n_features=data_train.X[0][0].shape[1], 90 | #inference_method='qpbo' if lateral else 'dai', 91 | #class_weight=class_weights) 92 | model = EdgeFeatureGraphCRF(n_states=n_states, 93 | n_features=data_train.X[0][0].shape[1], 94 | inference_method='qpbo' if lateral else 95 | 'dai', class_weight=class_weights, 96 | n_edge_features=4, 97 | symmetric_edge_features=[0, 1], 98 | antisymmetric_edge_features=[2]) 99 | ssvm = learners.OneSlackSSVM( 100 | model, verbose=2, C=C, max_iter=100000, n_jobs=-1, 101 | tol=0.0001, show_loss_every=200, inference_cache=50, logger=logger, 102 | cache_tol='auto', inactive_threshold=1e-5, break_on_bad=False) 103 | 104 | #ssvm = logger.load() 105 | 106 | X_, Y_ = shuffle(X_, Y_) 107 | #ssvm.fit(data_train.X, data_train.Y) 108 | #ssvm.fit(X_, Y_, warm_start=warm_start) 109 | ssvm.fit(X_, Y_) 110 | print("fit finished!") 111 | 112 | 113 | def plot_init(): 114 | data = load_data("train", independent=False) 115 | data = make_hierarchical_data(data, lateral=False, latent=True) 116 | #X, Y = discard_void(data.X, data.Y, 21) 117 | #data.X, data.Y = X, Y 118 | H = kmeans_init(data.X, data.Y, n_labels=22, n_hidden_states=22) 119 | plot_results_hierarchy(data, H) 120 | 121 | 122 | def plot_results(): 123 | data = load_data("val", independent=False) 124 | data = make_hierarchical_data(data, lateral=False, latent=True) 125 | logger = SaveLogger("test_latent_2.0001.pickle", save_every=100) 126 | ssvm = logger.load() 127 | plot_results_hierarchy(data, ssvm.predict(data.X), 128 | folder="latent_results_val_50_states_no_lateral") 129 | 130 | 131 | if __name__ == "__main__": 132 | #for C in 10. ** np.arange(-5, 2): 133 | #for lr in 10. ** np.arange(-3, 2)[::-1]: 134 | #svm_on_segments(C=.01, learning_rate=lr) 135 | #svm_on_segments(C=.01, learning_rate=0.1) 136 | svm_on_segments(C=.1, subgradient=False) 137 | #plot_init() 138 | #plot_results() 139 | -------------------------------------------------------------------------------- /msrc/hierarchical_helpers.py: -------------------------------------------------------------------------------- 1 | 2 | import numpy as np 3 | 4 | from msrc_helpers import (DataBunch, load_data, sigm, add_edges, 5 | add_kraehenbuehl_features) 6 | 7 | 8 | def add_top_node(data): 9 | X_stacked, Y_stacked = [], [] 10 | for x, y in zip(data.X, data.Y): 11 | new_node = np.max(x[1]) + 1 12 | n_nodes = len(x[0]) 13 | edges = np.c_[np.arange(n_nodes), np.repeat(new_node, n_nodes)] 14 | edges_stacked = np.vstack([x[1], edges]) 15 | 16 | y_stacked = y 17 | x_stacked = (x[0], edges_stacked, 1) 18 | X_stacked.append(x_stacked) 19 | Y_stacked.append(y_stacked) 20 | return DataBunch(X_stacked, Y_stacked, data.file_names, data.superpixels) 21 | 22 | 23 | def load_data_global_probs(dataset="train", latent=False): 24 | def padded_vstack(blub): 25 | a, b = blub 26 | if a.shape[0] > b.shape[0]: 27 | b = np.hstack([b, np.zeros((b.shape[0], a.shape[1] - b.shape[1]))]) 28 | return np.vstack([a, b]) 29 | 30 | data = load_data(dataset=dataset, which="piecewise") 31 | data = add_kraehenbuehl_features(data, which="train_30px") 32 | data = add_kraehenbuehl_features(data, which="train") 33 | data = add_edges(data) 34 | if latent: 35 | data = add_top_node(data) 36 | descs = np.load("/home/user/amueller/checkout/superpixel_crf/" 37 | "global_probs_%s.npy" % dataset) 38 | X = [] 39 | for x, glob_desc in zip(data.X, descs): 40 | if latent: 41 | x_ = padded_vstack([x[0], 42 | np.repeat(sigm(glob_desc)[np.newaxis, 1:], 43 | x[2], axis=0)]) 44 | else: 45 | x_ = np.hstack([x[0], np.repeat(sigm(glob_desc)[np.newaxis, :], 46 | x[0].shape[0], axis=0)]) 47 | # add features for latent node 48 | if len(x) == 3: 49 | X.append((x_, x[1], x[2])) 50 | else: 51 | X.append((x_, x[1])) 52 | 53 | return DataBunch(X, data.Y, data.file_names, data.superpixels) 54 | -------------------------------------------------------------------------------- /msrc/ignore_void_crf.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from pystruct.problems import GraphCRF 4 | from pystruct.inference import inference_dispatch 5 | 6 | 7 | from IPython.core.debugger import Tracer 8 | 9 | tracer = Tracer() 10 | 11 | 12 | class IgnoreVoidCRF(GraphCRF): 13 | """GraphCRF that ignores nodes with void label in ground truth. 14 | """ 15 | def __init__(self, n_states=2, n_features=None, inference_method='qpbo', 16 | void_label=21): 17 | if void_label >= n_states: 18 | raise ValueError("void_label must be one of the states!") 19 | GraphCRF.__init__(self, n_states, n_features, inference_method) 20 | self.void_label = void_label 21 | 22 | def max_loss(self, y): 23 | return np.sum(y != self.void_label) 24 | 25 | def loss(self, y, y_hat): 26 | # hamming loss: 27 | return np.sum((y != y_hat)[y != self.void_label]) 28 | 29 | def loss_augmented_inference(self, x, y, w, relaxed=False, 30 | return_energy=False): 31 | self.inference_calls += 1 32 | self._check_size_w(w) 33 | unary_potentials = self.get_unary_potentials(x, w) 34 | pairwise_potentials = self.get_pairwise_potentials(x, w) 35 | edges = self.get_edges(x) 36 | # do loss-augmentation 37 | for l in np.arange(self.n_states): 38 | # for each class, decrement features 39 | # for loss-agumention 40 | unary_potentials[(y != l) * (y != self.void_label), l] += 1. 41 | unary_potentials[:, l] += 1. / y.size 42 | return inference_dispatch(unary_potentials, pairwise_potentials, edges, 43 | self.inference_method, relaxed=relaxed, 44 | return_energy=return_energy) 45 | 46 | def continuous_loss(self, y, y_hat): 47 | # continuous version of the loss 48 | # y is the result of linear programming 49 | mask = y != self.void_label 50 | return (GraphCRF.continuous_loss(self, y[mask], y_hat[mask]) 51 | + np.sum(y_hat == self.void_label) / y.size) 52 | -------------------------------------------------------------------------------- /msrc/msrc_crf.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from pystruct import learners 4 | import pystruct.models as crfs 5 | from pystruct.utils import SaveLogger 6 | 7 | from msrc_helpers import (discard_void, add_edge_features, add_edges, 8 | load_data, add_kraehenbuehl_features, 9 | concatenate_datasets) 10 | #from hierarchical_helpers import load_data_global_probs 11 | #from msrc_helpers import SimpleSplitCV, concatenate_datasets 12 | 13 | 14 | from IPython.core.debugger import Tracer 15 | tracer = Tracer() 16 | 17 | 18 | def main(C=1, test=False): 19 | # load training data 20 | #independent = True 21 | independent = False 22 | data_train = load_data(which="piecewise") 23 | data_train = add_edges(data_train, independent=independent, 24 | fully_connected=True) 25 | data_train = add_kraehenbuehl_features(data_train, which="train_30px") 26 | data_train = add_kraehenbuehl_features(data_train, which="train") 27 | 28 | #data_train = load_data_global_probs() 29 | 30 | if not independent: 31 | data_train = add_edge_features(data_train) 32 | 33 | data_train = discard_void(data_train, 21) 34 | 35 | if test: 36 | data_val = load_data("val", which="piecewise_train") 37 | data_val = add_edges(data_val, independent=independent) 38 | data_val = add_kraehenbuehl_features(data_val, which="train_30px") 39 | data_val = add_kraehenbuehl_features(data_val, which="train") 40 | data_val = add_edge_features(data_val) 41 | data_val = discard_void(data_val, 21) 42 | data_train = concatenate_datasets(data_train, data_val) 43 | 44 | #X_.extend(data_val.X) 45 | #Y_.extend(data_val.Y) 46 | 47 | n_states = 21 48 | print("number of samples: %s" % len(data_train.X)) 49 | class_weights = 1. / np.bincount(np.hstack(data_train.Y)) 50 | #class_weights[21] = 0 51 | class_weights *= 21. / np.sum(class_weights) 52 | #class_weights = np.ones(n_states) 53 | print(class_weights) 54 | #model = crfs.GraphCRF(n_states=n_states, 55 | #n_features=data_train.X[0][0].shape[1], 56 | #inference_method='qpbo', class_weight=class_weights) 57 | model = crfs.EdgeFeatureGraphCRF(n_states=n_states, 58 | n_features=data_train.X[0][0].shape[1], 59 | inference_method='qpbo', 60 | class_weight=class_weights, 61 | n_edge_features=3, 62 | symmetric_edge_features=[0, 1], 63 | antisymmetric_edge_features=[2]) 64 | experiment_name = "fully_connected_%f" % C 65 | #warm_start = True 66 | warm_start = False 67 | ssvm = learners.OneSlackSSVM( 68 | model, verbose=2, C=C, max_iter=100000, n_jobs=-1, 69 | tol=0.0001, show_loss_every=50, inference_cache=50, cache_tol='auto', 70 | logger=SaveLogger(experiment_name + ".pickle", save_every=100), 71 | inactive_threshold=1e-5, break_on_bad=False, inactive_window=50, 72 | switch_to_ad3=False) 73 | #ssvm = learners.SubgradientSSVM( 74 | #model, verbose=3, C=C, max_iter=10000, n_jobs=-1, show_loss_every=10, 75 | #logger=SaveLogger(experiment_name + ".pickle", save_every=10), 76 | #momentum=0, learning_rate=0.001, decay_exponent=1) 77 | 78 | if warm_start: 79 | ssvm = SaveLogger(experiment_name + ".pickle").load() 80 | ssvm.logger = SaveLogger( 81 | file_name=experiment_name + "_refit.pickle", 82 | save_every=10) 83 | ssvm.learning_rate = 0.000001 84 | #ssvm.model.inference_method = 'ad3' 85 | #ssvm.n_jobs = 1 86 | 87 | ssvm.fit(data_train.X, data_train.Y, warm_start=warm_start) 88 | print("fit finished!") 89 | return 90 | 91 | 92 | if __name__ == "__main__": 93 | #for C in 10. ** np.arange(-4, 2): 94 | #main(C) 95 | main(.01, test=False) 96 | -------------------------------------------------------------------------------- /msrc/msrc_helpers.py: -------------------------------------------------------------------------------- 1 | from collections import namedtuple 2 | #from glob import glob 3 | 4 | import cPickle 5 | 6 | import numpy as np 7 | #from scipy.misc import imread 8 | from scipy import sparse 9 | import matplotlib.pyplot as plt 10 | 11 | from sklearn.externals.joblib import Memory 12 | from latent_crf_experiments.utils import transform_chi2 13 | #from sklearn.metrics import confusion_matrix 14 | 15 | from datasets.msrc import MSRC21Dataset 16 | 17 | 18 | # stores information that was COMPUTED from the dataset + file names for 19 | # correspondence 20 | DataBunch = namedtuple('DataBunch', 'X, Y, file_names, superpixels') 21 | 22 | base_path = "/home/user/amueller/datasets/aurelien_msrc_features/msrc/" 23 | memory = Memory(cachedir="/tmp/cache") 24 | 25 | 26 | class PixelwiseScorer(object): 27 | def __init__(self, data): 28 | self.data = data 29 | self.greater_is_better = True 30 | 31 | def __call__(self, estimator, X, y): 32 | result = eval_on_pixels(self.data, 33 | [estimator.predict(x) for x in self.data.X]) 34 | return result['average'] 35 | 36 | 37 | class SimpleSplitCV(): 38 | def __init__(self, n_train, n_test): 39 | self.n_train = n_train 40 | self.n_test = n_test 41 | 42 | def __iter__(self): 43 | mask_train = np.zeros(self.n_train + self.n_test, dtype=np.bool) 44 | mask_test = mask_train.copy() 45 | mask_train[:self.n_train] = True 46 | mask_test[self.n_train:] = True 47 | yield mask_train, mask_test 48 | 49 | 50 | def load_data(dataset="train", which="bow"): 51 | if which == "bow": 52 | filename = ("/home/user/amueller/checkout/superpixel_crf/" 53 | "data_%s_1000_color.pickle" % dataset) 54 | elif which == "bow_old": 55 | filename = ("/home/user/amueller/checkout/superpixel_crf/" 56 | "data_%s_1000_color_old.pickle" % dataset) 57 | elif which == "bow_new": 58 | filename = ("/home/user/amueller/checkout/superpixel_crf/" 59 | "data_%s_1000_color_new.pickle" % dataset) 60 | elif which == "bow_5k": 61 | filename = ("/home/user/amueller/checkout/superpixel_crf/" 62 | "data_%s_5000_color.pickle" % dataset) 63 | elif which == "piecewise": 64 | filename = ("/home/user/amueller/checkout/superpixel_crf/" 65 | "data_probs_%s_cw_2.pickle" % dataset) 66 | elif which == "piecewise_trainval": 67 | filename = ("/home/user/amueller/checkout/superpixel_crf/" 68 | "data_probs_%s_cw_trainval.pickle" % dataset) 69 | elif which == "piecewise_new": 70 | filename = ("/home/user/amueller/checkout/superpixel_crf/" 71 | "data_probs_%s_new.pickle" % dataset) 72 | else: 73 | raise ValueError("'which' should be 'bow' or 'piecewise'") 74 | 75 | with open(filename) as f: 76 | data = cPickle.load(f) 77 | if which in ["bow", "bow_old", "bow_new", "bow_5k"]: 78 | data = transform_chi2(data) 79 | if which == "piecewise_new": 80 | X = [sigm(x) for x in data.X] 81 | data = DataBunch(X, data.Y, data.file_names, data.superpixels) 82 | return data 83 | 84 | 85 | #def load_data_aurelien(dataset="train", independent=False): 86 | #mountain_idx = np.where(classes == "mountain")[0] 87 | #horse_idx = np.where(classes == "horse")[0] 88 | #void_idx = np.where(classes == "void")[0] 89 | 90 | #ds_dict = dict(train="Train", val="Validation", test="Test") 91 | #if dataset not in ds_dict.keys(): 92 | #raise ValueError("dataset must be one of 'train', 'val', 'test'," 93 | #" got %s" % dataset) 94 | #ds_path = base_path + ds_dict[dataset] 95 | #file_names, all_superpixels = [], [] 96 | #X, Y = [], [] 97 | #for f in glob(ds_path + "/*.dat"): 98 | #name = os.path.basename(f).split('.')[0] 99 | #img = imread("%s/%s.bmp" % (ds_path, name)) 100 | #labels = np.loadtxt(base_path + "labels/%s.txt" % name, dtype=np.int) 101 | #file_names.append(name) 102 | ## features 103 | #feat = np.hstack([np.loadtxt("%s/%s.local%s" % (ds_path, name, i)) for 104 | #i in xrange(1, 7)]) 105 | ## superpixels 106 | #superpixels = np.fromfile("%s/%s.dat" % (ds_path, name), 107 | #dtype=np.int32) 108 | #superpixels = superpixels.reshape(img.shape[:-1][::-1]).T - 1 109 | #all_superpixels.append(superpixels) 110 | ## make horse and mountain to void 111 | #labels[labels == mountain_idx] = void_idx 112 | #labels[labels == horse_idx] = void_idx 113 | #Y.append(labels) 114 | #X.append(feat) 115 | #data = DataBunch(X, Y, file_names, all_superpixels) 116 | #data = add_edges(data, independent=independent) 117 | #return data 118 | 119 | 120 | def concatenate_datasets(data1, data2): 121 | X = data1.X + data2.X 122 | Y = data1.Y + data2.Y 123 | file_names = np.hstack([data1.file_names, data2.file_names]) 124 | superpixels = data1.superpixels + data2.superpixels 125 | return DataBunch(X, Y, file_names, superpixels) 126 | 127 | 128 | 129 | 130 | def load_kraehenbuehl(filename, which="train"): 131 | if which == "train": 132 | path = "/home/user/amueller/datasets/kraehenbuehl_potentials_msrc/out/" 133 | elif which == "trainval": 134 | path = ("/home/user/amueller/datasets/kraehenbuehl_potentials_msrc/" 135 | "textonboost_trainval/") 136 | elif which == "train_30px": 137 | path = ("/home/user/amueller/datasets/kraehenbuehl_potentials_msrc/" 138 | "train_30px/") 139 | elif which == "trainval_30px": 140 | path = ("/home/user/amueller/datasets/kraehenbuehl_potentials_msrc/" 141 | "trainval_30px/") 142 | else: 143 | raise ValueError("Unexpected which in load_kraehenbuehl: %s" % which) 144 | #path = "/home/local/datasets/MSRC_ObjCategImageDatabase_v2/asdf/" 145 | with open(path + filename + ".unary") as f: 146 | size = np.fromfile(f, dtype=np.uint32, count=3).byteswap() 147 | data = np.fromfile(f, dtype=np.float32).byteswap() 148 | img = data.reshape(size[1], size[0], size[2]) 149 | return img 150 | 151 | 152 | @memory.cache 153 | def get_kraehenbuehl_pot_sp(data, which="train"): 154 | feats = [] 155 | for x, filename, superpixels in zip(data.X, data.file_names, 156 | data.superpixels): 157 | probs = load_kraehenbuehl(filename, which=which) 158 | if which != "train": 159 | # softmax normalization 160 | probs -= np.max(probs, axis=-1)[:, :, np.newaxis] 161 | probs = np.exp(probs) 162 | probs /= probs.sum(axis=-1)[:, :, np.newaxis] 163 | 164 | # accumulate votes in superpixels 165 | # interleaved repeat 166 | class_indices = np.repeat(np.arange(21)[np.newaxis, :], 167 | superpixels.size, axis=0).ravel() 168 | # non-interleaved repeat 169 | superpixel_indices = np.repeat(superpixels.ravel(), 21) 170 | sp_probs = sparse.coo_matrix((probs.ravel(), (superpixel_indices, 171 | class_indices))) 172 | sp_probs = sp_probs.toarray() 173 | # renormalize (same as dividing by sp sizes) 174 | feats.append(sp_probs / sp_probs.sum(axis=-1)[:, np.newaxis]) 175 | return feats 176 | 177 | 178 | def sigm(x): 179 | return 1. / (1 + np.exp(-x)) 180 | 181 | 182 | def add_kraehenbuehl_features(data, which="train", replace=False): 183 | sp_probas = get_kraehenbuehl_pot_sp(data, which=which) 184 | if replace: 185 | X = [probas 186 | for probas in sp_probas] 187 | return DataBunch(X, data.Y, data.file_names, data.superpixels) 188 | if isinstance(data.X[0], np.ndarray): 189 | X = [np.hstack([x, probas]) 190 | for x, probas in zip(data.X, sp_probas)] 191 | else: 192 | X = [(np.hstack([x[0], probas]), x[1]) 193 | for x, probas in zip(data.X, sp_probas)] 194 | return DataBunch(X, data.Y, data.file_names, data.superpixels) 195 | 196 | 197 | def eval_on_pixels(data, sp_predictions, print_results=True): 198 | """Evaluate segmentation performance on pixel level. 199 | 200 | Parameters 201 | ---------- 202 | data : DataBunch Named tuple 203 | Contains superpixels, descriptors, superpixel gt and filenames. 204 | 205 | sp_predictions : list of arrays 206 | For each image, list of labels per superpixel 207 | 208 | print_results : bool, default=True 209 | Whether to print results to stdout. 210 | 211 | """ 212 | msrc = MSRC21Dataset() 213 | pixel_predictions = [sp_pred[sp] for sp_pred, sp in zip(sp_predictions, 214 | data.superpixels)] 215 | result = msrc.eval_pixel_performance(data.file_names, pixel_predictions, 216 | print_results=print_results) 217 | return result 218 | 219 | 220 | def plot_confusion_matrix(dataset, confusion, title=None): 221 | confusion_normalized = (confusion.astype(np.float) / 222 | confusion.sum(axis=1)[:, np.newaxis]) 223 | plt.matshow(confusion_normalized) 224 | plt.axis("off") 225 | plt.colorbar() 226 | for i, c in enumerate(dataset.classes): 227 | plt.text(i, -1, c, rotation=60, va='bottom') 228 | plt.text(-1, i, c, ha='right') 229 | if title: 230 | plt.title(title) 231 | -------------------------------------------------------------------------------- /msrc/msrc_svm.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from sklearn.svm import LinearSVC 4 | #from sklearn.linear_model import LogisticRegression 5 | 6 | from msrc_helpers import (discard_void, PixelwiseScorer, concatenate_datasets) 7 | from msrc_helpers import SimpleSplitCV, load_data 8 | from msrc_helpers import add_kraehenbuehl_features 9 | 10 | from IPython.core.debugger import Tracer 11 | tracer = Tracer() 12 | 13 | 14 | def train_svm(test=False, C=0.01, gamma=.1, grid=False): 15 | which = "piecewise" 16 | 17 | data_train = load_data(which=which) 18 | data_train = add_kraehenbuehl_features(data_train, which="train_30px") 19 | data_train = add_kraehenbuehl_features(data_train, which="train") 20 | data_train_novoid = discard_void(data_train, 21) 21 | if grid and test: 22 | raise ValueError("Don't you dare grid-search on the test-set!") 23 | 24 | svm = LinearSVC(C=C, class_weight='auto', multi_class='crammer_singer', 25 | dual=False) 26 | #svm = LogisticRegression(C=C, class_weight='auto') 27 | data_val = load_data('val', which=which) 28 | data_val = add_kraehenbuehl_features(data_val, which="train_30px") 29 | data_val = add_kraehenbuehl_features(data_val, which="train") 30 | data_val_novoid = discard_void(data_val, 21) 31 | 32 | if grid: 33 | n_samples_train = len(np.hstack(data_train_novoid.Y)) 34 | n_samples_val = len(np.hstack(data_val_novoid.Y)) 35 | cv = SimpleSplitCV(n_samples_train, n_samples_val) 36 | data_trainval = concatenate_datasets(data_train_novoid, 37 | data_val_novoid) 38 | 39 | from sklearn.grid_search import GridSearchCV 40 | #from sklearn.grid_search import RandomizedSearchCV 41 | #from scipy.stats import expon, gamma 42 | #param_grid = {'C': 10. ** np.arange(1, 4), 'gamma': 10. ** 43 | #np.arange(-3, 1)} 44 | param_grid = {'C': 10. ** np.arange(-6, 2)} 45 | scorer = PixelwiseScorer(data=data_val) 46 | grid = GridSearchCV(svm, param_grid=param_grid, verbose=10, n_jobs=-1, 47 | cv=cv, scoring=scorer, refit=False) 48 | grid.fit(np.vstack(data_trainval.X), 49 | np.hstack(data_trainval.Y)) 50 | print(grid.best_params_) 51 | print(grid.best_score_) 52 | else: 53 | print(svm) 54 | if test: 55 | data_train_novoid = concatenate_datasets(data_train_novoid, 56 | data_val_novoid) 57 | 58 | print(np.vstack(data_train_novoid.X).shape) 59 | svm.fit(np.vstack(data_train_novoid.X), np.hstack(data_train_novoid.Y)) 60 | if test: 61 | data_test = load_data("test", which=which) 62 | else: 63 | data_test = load_data("val", which=which) 64 | data_test = add_kraehenbuehl_features(data_test, which="train_30px") 65 | data_test = add_kraehenbuehl_features(data_test, which="train") 66 | scorer = PixelwiseScorer(data=data_test) 67 | scorer(svm, None, None) 68 | 69 | return svm 70 | 71 | 72 | if __name__ == "__main__": 73 | train_svm(grid=False, C=.1, test=False) 74 | -------------------------------------------------------------------------------- /msrc/parts.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | #from sklearn.preprocessing import StandardScaler 4 | 5 | #from datasets.msrc import MSRCDataset 6 | from pystruct import learners 7 | from pystruct.problems import GraphCRF, LatentGraphCRF 8 | 9 | from msrc_helpers import classes, load_data, plot_results 10 | 11 | from IPython.core.debugger import Tracer 12 | tracer = Tracer() 13 | 14 | 15 | def plot_parts(): 16 | from pystruct.problems.latent_graph_crf import kmeans_init 17 | car_idx = np.where(classes == "car")[0] 18 | data = load_data("train", independent=False) 19 | car_images = np.array([i for i, y in enumerate(data.Y) 20 | if np.any(y == car_idx)]) 21 | flat_X = [x[0] for x in data.X] 22 | edges = [[x[1]] for x in data.X] 23 | n_states_per_label = np.ones(22, dtype=np.int) 24 | n_states_per_label[car_idx] = 6 25 | 26 | H = kmeans_init(flat_X, data.Y, edges, n_labels=22, 27 | n_states_per_label=n_states_per_label, symmetric=True) 28 | X, Y, file_names, images, all_superpixels, H = zip(*[ 29 | (data.X[i], data.Y[i], data.file_names[i], data.images[i], 30 | data.all_superpixels[i], H[i]) 31 | for i in car_images]) 32 | plot_results(images, file_names, Y, H, all_superpixels, 33 | folder="test_parts", use_colors_predict=False) 34 | 35 | 36 | def train_car_parts(): 37 | car_idx = np.where(classes == "car")[0] 38 | data = load_data("train", independent=False) 39 | car_images = np.array([i for i, y in enumerate(data.Y) 40 | if np.any(y == car_idx)]) 41 | n_states_per_label = np.ones(22, dtype=np.int) 42 | n_states_per_label[car_idx] = 6 43 | 44 | X, Y, file_names, images, all_superpixels = zip(*[ 45 | (data.X[i], data.Y[i], data.file_names[i], data.images[i], 46 | data.all_superpixels[i]) for i in car_images]) 47 | problem = LatentGraphCRF(n_states_per_label=n_states_per_label, 48 | n_labels=22, inference_method='ad3', 49 | n_features=21 * 6) 50 | ssvm = learners.LatentSSVM( 51 | problem, verbose=2, C=10, max_iter=5000, n_jobs=-1, tol=0.0001, 52 | show_loss_every=10, base_svm='subgradient', inference_cache=50, 53 | latent_iter=5, learning_rate=0.001, decay_exponent=0.5) 54 | ssvm.fit(X, Y) 55 | plot_results(images, file_names, Y, ssvm.H_init_, all_superpixels, 56 | folder="parts_init", use_colors_predict=False) 57 | H = ssvm.predict_latent(X) 58 | plot_results(images, file_names, Y, H, all_superpixels, 59 | folder="parts_prediction", use_colors_predict=False) 60 | H_final = [problem.latent(x, y, ssvm.w) for x, y in zip(X, Y)] 61 | plot_results(images, file_names, Y, H_final, all_superpixels, 62 | folder="parts_final", use_colors_predict=False) 63 | tracer() 64 | 65 | 66 | def train_car(): 67 | car_idx = np.where(classes == "car")[0] 68 | data_train = load_data("train", independent=False) 69 | car_images = np.array([i for i, y in enumerate(data_train.Y) 70 | if np.any(y == car_idx)]) 71 | n_states_per_label = np.ones(22, dtype=np.int) 72 | n_states_per_label[car_idx] = 6 73 | 74 | X, Y, file_names, images, all_superpixels = zip(*[ 75 | (data_train.X[i], data_train.Y[i], data_train.file_names[i], 76 | data_train.images[i], data_train.superpixels[i]) 77 | for i in car_images]) 78 | problem = GraphCRF(n_states=22, inference_method='ad3', n_features=21 * 6) 79 | ssvm = learners.SubgradientStructuredSVM( 80 | problem, verbose=2, C=.001, max_iter=5000, n_jobs=-1, 81 | show_loss_every=10, learning_rate=0.0001, decay_exponent=0.5) 82 | ssvm.fit(X, Y) 83 | Y_pred = ssvm.predict(X) 84 | plot_results(images, file_names, Y, Y_pred, all_superpixels, 85 | folder="cars_only") 86 | 87 | data_val = load_data("val", independent=False) 88 | car_images_val = np.array([i for i, y in enumerate(data_val.Y) 89 | if np.any(y == car_idx)]) 90 | X_val, Y_val, file_names_val, images_val, all_superpixels_val = \ 91 | zip(*[(data_val.X[i], data_val.Y[i], data_val.file_names[i], 92 | data_val.images[i], data_val.superpixels[i]) for i in 93 | car_images_val]) 94 | Y_pred_val = ssvm.predict(X_val) 95 | plot_results(images_val, file_names_val, Y_val, Y_pred_val, 96 | all_superpixels_val, folder="cars_only_val") 97 | # C=10 98 | ## train: 99 | #0.92743060939680566V 100 | #> ssvm.score(X_val, Y_val) 101 | #0.52921719955898561 102 | # test 0.61693548387096775 103 | tracer() 104 | -------------------------------------------------------------------------------- /nyu/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amueller/segmentation/f6d252135d91985002fdaccc59214a7199dabed9/nyu/__init__.py -------------------------------------------------------------------------------- /nyu/nyu_baselines.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | #import matplotlib.pyplot as plt 3 | 4 | 5 | from sklearn.svm import LinearSVC 6 | 7 | from datasets.nyu import NYUSegmentation 8 | 9 | from nyu_helpers import load_nyu_pixelwise, load_nyu 10 | from latent_crf_experiments.utils import eval_on_pixels, eval_on_sp 11 | 12 | from IPython.core.debugger import Tracer 13 | 14 | tracer = Tracer() 15 | 16 | def eval_pixel_prediction(): 17 | dataset = NYUSegmentation() 18 | data = load_nyu_pixelwise('val') 19 | predictions = [np.argmax(x, axis=-1) for x in data.X] 20 | hamming, jaccard = eval_on_pixels(dataset, data.Y, predictions, print_results=True) 21 | 22 | 23 | def eval_sp_prediction(): 24 | dataset = NYUSegmentation() 25 | data = load_nyu('val', n_sp=500, sp='rgbd') 26 | predictions = [np.argmax(x, axis=-1) for x in data.X] 27 | #predictions = data.Y 28 | hamming, jaccard = eval_on_sp(dataset, data, predictions, print_results=True) 29 | 30 | 31 | def train_svm(C=0.1, grid=False): 32 | ds = NYUSegmentation() 33 | data_train = load_nyu("train", n_sp=500, sp='rgbd') 34 | svm = LinearSVC(C=C, dual=False, class_weight='auto') 35 | #N_train = [] 36 | #for f, sp in zip(data_train.file_names, data_train.superpixels): 37 | #normals = ds.get_pointcloud_normals(f)[:, :, 3:] 38 | #mean_normals = get_sp_normals(normals, sp) 39 | #N_train.append(mean_normals * .1) 40 | #N_flat_train = np.vstack(N_train) 41 | 42 | X, y = np.vstack(data_train.X), np.hstack(data_train.Y) 43 | #X = np.hstack([X, N_flat_train]) 44 | svm.fit(X, y) 45 | print(svm.score(X, y)) 46 | eval_on_sp(ds, data_train, [svm.predict(x) 47 | for x in data_train.X], 48 | print_results=True) 49 | 50 | data_val = load_nyu("val", n_sp=500, sp='rgbd') 51 | #N_val = [] 52 | #for f, sp in zip(data_val.file_names, data_val.superpixels): 53 | #normals = ds.get_pointcloud_normals(f)[:, :, 3:] 54 | #mean_normals = get_sp_normals(normals, sp) 55 | #N_val.append(mean_normals * .1) 56 | eval_on_sp(ds, data_val, [svm.predict(x) 57 | for x in data_val.X], 58 | print_results=True) 59 | 60 | if __name__ == "__main__": 61 | #eval_pixel_prediction() 62 | #eval_sp_prediction() 63 | train_svm(C=1) 64 | -------------------------------------------------------------------------------- /nyu/nyu_crf.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from pystruct import learners 4 | import pystruct.models as crfs 5 | from pystruct.utils import SaveLogger 6 | 7 | from datasets.nyu import NYUSegmentation 8 | from nyu_helpers import load_nyu 9 | from latent_crf_experiments.utils import discard_void, add_edge_features, add_edges 10 | #from hierarchical_helpers import load_data_global_probs 11 | #from msrc_helpers import SimpleSplitCV, concatenate_datasets 12 | 13 | 14 | from IPython.core.debugger import Tracer 15 | tracer = Tracer() 16 | 17 | 18 | def main(C=1): 19 | dataset = NYUSegmentation() 20 | # load training data 21 | data_train = load_nyu(n_sp=500, sp='rgbd') 22 | data_train = add_edges(data_train) 23 | data_train = add_edge_features(dataset, data_train, depth_diff=True, normal_angles=True) 24 | 25 | data_train = discard_void(dataset, data_train) 26 | 27 | n_states = 4. 28 | print("number of samples: %s" % len(data_train.X)) 29 | class_weights = 1. / np.bincount(np.hstack(data_train.Y)) 30 | class_weights *= n_states / np.sum(class_weights) 31 | #class_weights = np.ones(n_states) 32 | print(class_weights) 33 | #model = crfs.GraphCRF(n_states=n_states, 34 | #n_features=data_train.X[0][0].shape[1], 35 | #inference_method='qpbo', class_weight=class_weights) 36 | model = crfs.EdgeFeatureGraphCRF(inference_method='qpbo', 37 | class_weight=class_weights, 38 | n_edge_features=5, 39 | symmetric_edge_features=[0, 1]) 40 | experiment_name = "rgbd_test%f" % C 41 | ssvm = learners.OneSlackSSVM( 42 | model, verbose=2, C=C, max_iter=100000, n_jobs=-1, 43 | tol=0.001, show_loss_every=100, inference_cache=50, cache_tol='auto', 44 | logger=SaveLogger(experiment_name + ".pickle", save_every=100), 45 | inactive_threshold=1e-5, break_on_bad=False, inactive_window=50, 46 | switch_to=("ad3", {'branch_and_bound':True})) 47 | 48 | ssvm.fit(data_train.X, data_train.Y) 49 | print("fit finished!") 50 | return 51 | 52 | 53 | if __name__ == "__main__": 54 | #for C in [.1, 1]: 55 | #main(C) 56 | main(.1) 57 | -------------------------------------------------------------------------------- /nyu/nyu_helpers.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | #import os 3 | #from glob import glob 4 | 5 | from sklearn.externals.joblib import Memory, Parallel, delayed 6 | from scipy.misc import imread 7 | #import matplotlib.pyplot as plt 8 | 9 | from datasets.nyu import NYUSegmentation 10 | 11 | from slic_python import slic_n 12 | from latent_crf_experiments.utils import (DataBunchNoSP, DataBunch, gt_in_sp, 13 | probabilities_on_sp, get_sp_normals) 14 | from latent_crf_experiments.pascal.pascal_helpers import merge_small_sp 15 | from latent_crf_experiments.hierarchical_segmentation import \ 16 | make_hierarchy_edges, HierarchicalDataBunch 17 | from skimage.segmentation import slic 18 | from skimage import morphology 19 | 20 | from information_theoretic_mst import ITM 21 | 22 | memory = Memory(cachedir="/home/data/amueller/cache") 23 | 24 | 25 | def get_probabilities(file_name, path): 26 | probabilities = [] 27 | for label in xrange(1, 5): 28 | f = ("%s/prediction_all/%s_lab_image_label_%d.png" 29 | % (path, file_name, label)) 30 | probabilities.append(imread(f)[:, :, 0]) 31 | probabilities = np.dstack(probabilities).astype(np.float) 32 | return probabilities / 255. 33 | 34 | 35 | def load_single_file(dataset, file_name, n_sp=300, sp='rgb', reorder=None): 36 | print(file_name) 37 | image = dataset.get_image(file_name) 38 | if sp == 'rgb': 39 | sps = slic_n(image, n_superpixels=n_sp, compactness=10) 40 | elif sp == 'rgb-skimage': 41 | sps = slic(image, n_segments=n_sp, compactness=10, multichannel=True, sigma=0.1) 42 | sps = merge_small_sp(image, morphology.label(sps))[0] 43 | elif sp == 'rgbd': 44 | depth = dataset.get_depth(file_name) 45 | #depth -= depth.min() 46 | #depth /= depth.max() 47 | rgbd = np.dstack([image / 255., depth]) 48 | sps = slic(rgbd, n_segments=n_sp, compactness=.1, convert2lab=False, multichannel=True, sigma=0) 49 | sps = merge_small_sp(image, morphology.label(sps))[0] 50 | else: 51 | raise ValueError("Expected sp to be 'rgb' or 'rgbd' got %d" % sp) 52 | 53 | gt = gt_in_sp(dataset, file_name, sps) 54 | probs = get_probabilities(file_name, dataset.directory) 55 | if reorder is not None: 56 | probs = probs[:, :, reorder] 57 | probs_sp = probabilities_on_sp(dataset, probs, sps) 58 | return probs_sp, gt, sps 59 | 60 | 61 | @memory.cache 62 | def load_nyu(ds='train', n_sp=300, sp='rgb'): 63 | # trigger cache..... 64 | dataset = NYUSegmentation() 65 | file_names = dataset.get_split(ds) 66 | if ds == "test": 67 | reorder = np.array([2, 0, 3, 1]) 68 | else: 69 | reorder = None 70 | # load image to generate superpixels 71 | result = Parallel(n_jobs=-1)(delayed(load_single_file)(dataset, f, n_sp, sp, reorder=reorder) 72 | for f in file_names) 73 | X, Y, superpixels = zip(*result) 74 | 75 | return DataBunch(X, Y, file_names, superpixels) 76 | 77 | 78 | @memory.cache 79 | def load_nyu_pixelwise(ds='train'): 80 | if ds == "test": 81 | reorder = np.array([2, 0, 3, 1]) 82 | else: 83 | reorder = np.arange(4) 84 | # trigger cache. 85 | dataset = NYUSegmentation() 86 | file_names, X, Y = [], [], [] 87 | for file_name in dataset.get_split(ds): 88 | print(file_name) 89 | file_names.append(file_name) 90 | gt = dataset.get_ground_truth(file_name) 91 | prediction = get_probabilities(file_name, dataset.directory) 92 | Y.append(gt) 93 | X.append(prediction[:, :, reorder]) 94 | return DataBunchNoSP(X, Y, file_names) 95 | 96 | 97 | def compute_xyz_segments(dataset, data): 98 | segments = [] 99 | for image_name, superpixels in zip(data.file_names, data.superpixels): 100 | points_normals = dataset.get_pointcloud_normals(image_name) 101 | centers3d = [np.bincount(superpixels.ravel(), weights=c.ravel()) 102 | for c in points_normals[:, :, :3].reshape(-1, 3).T] 103 | centers3d = (np.vstack(centers3d) / np.bincount(superpixels.ravel())).T 104 | sp_normals = get_sp_normals(points_normals[:, :, 3:], superpixels) 105 | km = ITM(n_clusters=30) 106 | km.fit(np.hstack([centers3d, sp_normals])) 107 | segments.append(km.labels_) 108 | return segments 109 | 110 | def make_hierarchical_data(dataset, data): 111 | segments = compute_xyz_segments(dataset, data) 112 | hierarchy_edges = make_hierarchy_edges(segments, data.superpixels) 113 | X_stacked = [] 114 | for x, y, edges in zip(data.X, data.Y, hierarchy_edges): 115 | edges_stacked = np.vstack([x[1], edges]) 116 | # only add a single constant edge feature 117 | hierarchy_edge_features = np.zeros((edges.shape[0], x[2].shape[1])) 118 | hierarchy_edge_features[:, 0] = 1 119 | edge_features_stacked = np.vstack([x[2], hierarchy_edge_features]) 120 | n_hidden = np.max(edges) + 1 121 | x_stacked = (x[0], edges_stacked, edge_features_stacked, n_hidden) 122 | X_stacked.append(x_stacked) 123 | return HierarchicalDataBunch(X_stacked, data.Y, data.file_names, 124 | data.superpixels, segments) 125 | -------------------------------------------------------------------------------- /nyu/nyu_hierarchical.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from pystruct import learners 4 | import pystruct.models as crfs 5 | from pystruct.utils import SaveLogger 6 | 7 | from datasets.nyu import NYUSegmentation 8 | from nyu_helpers import load_nyu, make_hierarchical_data 9 | from latent_crf_experiments.utils import discard_void, add_edge_features, add_edges 10 | #from hierarchical_helpers import load_data_global_probs 11 | #from msrc_helpers import SimpleSplitCV, concatenate_datasets 12 | 13 | 14 | from IPython.core.debugger import Tracer 15 | tracer = Tracer() 16 | 17 | 18 | def main(C=1): 19 | dataset = NYUSegmentation() 20 | # load training data 21 | data_train = load_nyu('train', n_sp=500, sp='rgbd') 22 | data_train = add_edges(data_train) 23 | data_train = add_edge_features(dataset, data_train, depth_diff=True, normal_angles=True) 24 | data_train = make_hierarchical_data(dataset, data_train) 25 | data_train = discard_void(dataset, data_train) 26 | 27 | n_states = 4. 28 | print("number of samples: %s" % len(data_train.X)) 29 | class_weights = 1. / np.bincount(np.hstack(data_train.Y)) 30 | class_weights *= n_states / np.sum(class_weights) 31 | #class_weights = np.ones(n_states) 32 | print(class_weights) 33 | #model = crfs.GraphCRF(n_states=n_states, 34 | #n_features=data_train.X[0][0].shape[1], 35 | #inference_method='qpbo', class_weight=class_weights) 36 | model = crfs.EdgeFeatureLatentNodeCRF( 37 | n_hidden_states=5, n_edge_features=5, inference_method='qpbo', 38 | class_weight=class_weights, symmetric_edge_features=[0, 1], 39 | latent_node_features=False, n_labels=4) 40 | experiment_name = "rgbd_normal_angles_fold1_strong_reweight%f" % C 41 | base_ssvm = learners.OneSlackSSVM( 42 | model, verbose=2, C=C, max_iter=100000, n_jobs=1, 43 | tol=0.001, show_loss_every=100, inference_cache=50, cache_tol='auto', 44 | logger=SaveLogger(experiment_name + ".pickle", save_every=100), 45 | inactive_threshold=1e-5, break_on_bad=False, inactive_window=50, 46 | switch_to=("ad3", {'branch_and_bound':True})) 47 | latent_logger = SaveLogger("lssvm_" + experiment_name + 48 | "_%d.pickle", save_every=1) 49 | ssvm = learners.LatentSSVM(base_ssvm, logger=latent_logger, 50 | latent_iter=3) 51 | 52 | ssvm.fit(data_train.X, data_train.Y) 53 | print("fit finished!") 54 | return 55 | 56 | 57 | if __name__ == "__main__": 58 | #for C in [.1, 1]: 59 | #main(C) 60 | main(.1) 61 | -------------------------------------------------------------------------------- /pascal/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amueller/segmentation/f6d252135d91985002fdaccc59214a7199dabed9/pascal/__init__.py -------------------------------------------------------------------------------- /pascal/hierarchical_crf.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | import cPickle 4 | 5 | from sklearn.utils import shuffle 6 | #from sklearn.grid_search import GridSearchCV 7 | 8 | from pystruct import learners 9 | from pystruct.utils import SaveLogger 10 | from pystruct.models import LatentNodeCRF 11 | 12 | from datasets.pascal import PascalSegmentation 13 | from pascal_helpers import load_pascal, make_cpmc_hierarchy 14 | from latent_crf_experiments.utils import discard_void 15 | from IPython.core.debugger import Tracer 16 | tracer = Tracer() 17 | 18 | 19 | def svm_on_segments(C=.1, learning_rate=.001, subgradient=False): 20 | data_file = "data_train_XY.pickle" 21 | ds = PascalSegmentation() 22 | if os.path.exists(data_file): 23 | X_, Y_ = cPickle.load(open(data_file)) 24 | else: 25 | # load and prepare data 26 | data_train = load_pascal("train", sp_type="cpmc") 27 | data_train = make_cpmc_hierarchy(ds, data_train) 28 | data_train = discard_void(ds, data_train) 29 | X_, Y_ = data_train.X, data_train.Y 30 | cPickle.dump((X_, Y_), open(data_file, 'wb'), -1) 31 | 32 | 33 | 34 | class_weights = 1. / np.bincount(np.hstack(Y_)) 35 | class_weights *= 21. / np.sum(class_weights) 36 | experiment_name = ("latent_25_cpmc_%f_qpbo_n_slack_blub3" % C) 37 | logger = SaveLogger(experiment_name + ".pickle", save_every=10) 38 | model = LatentNodeCRF(n_hidden_states=25, 39 | inference_method='qpbo', 40 | class_weight=class_weights, 41 | latent_node_features=False) 42 | if subgradient: 43 | ssvm = learners.LatentSubgradientSSVM( 44 | model, C=C, verbose=1, show_loss_every=10, logger=logger, 45 | n_jobs=-1, learning_rate=learning_rate, decay_exponent=1, 46 | momentum=0., max_iter=100000, decay_t0=100) 47 | else: 48 | latent_logger = SaveLogger("lssvm_" + experiment_name + 49 | "_%d.pickle", save_every=1) 50 | #base_ssvm = learners.OneSlackSSVM( 51 | #model, verbose=2, C=C, max_iter=100, n_jobs=-1, tol=0.001, 52 | #show_loss_every=200, inference_cache=50, logger=logger, 53 | #cache_tol='auto', inactive_threshold=1e-5, break_on_bad=False, 54 | #switch_to=('ogm', {'alg': 'dd'})) 55 | base_ssvm = learners.NSlackSSVM( 56 | model, verbose=4, C=C, n_jobs=-1, tol=0.1, 57 | show_loss_every=20, logger=logger, inactive_threshold=1e-8, 58 | break_on_bad=False, batch_size=36, inactive_window=10, 59 | switch_to=('ad3', {'branch_and_bound': True})) 60 | ssvm = learners.LatentSSVM(base_ssvm, logger=latent_logger, 61 | latent_iter=3) 62 | #warm_start = True 63 | warm_start = False 64 | if warm_start: 65 | ssvm = logger.load() 66 | ssvm.logger = SaveLogger(experiment_name + "_retrain.pickle", 67 | save_every=10) 68 | ssvm.max_iter = 10000 69 | ssvm.decay_exponent = 1 70 | #ssvm.decay_t0 = 1000 71 | #ssvm.learning_rate = 0.00001 72 | #ssvm.momentum = 0 73 | 74 | X_, Y_ = shuffle(X_, Y_) 75 | #ssvm.fit(data_train.X, data_train.Y) 76 | ssvm.fit(X_, Y_) 77 | #H_init = [np.hstack([y, np.random.randint(21, 26)]) for y in Y_] 78 | #ssvm.fit(X_, Y_, H_init=H_init) 79 | print("fit finished!") 80 | 81 | 82 | if __name__ == "__main__": 83 | #for C in 10. ** np.arange(-5, 2): 84 | #for lr in 10. ** np.arange(-3, 2)[::-1]: 85 | #svm_on_segments(C=.01, learning_rate=lr) 86 | #svm_on_segments(C=.01, learning_rate=0.1) 87 | svm_on_segments(C=.01, subgradient=False) 88 | #plot_init() 89 | #plot_results() 90 | -------------------------------------------------------------------------------- /pascal/pascal_baselines.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | 4 | from skimage.segmentation import mark_boundaries 5 | from sklearn.svm import LinearSVC 6 | 7 | from sklearn.cross_validation import LeavePLabelOut 8 | from sklearn.grid_search import GridSearchCV 9 | from sklearn.utils import shuffle 10 | from sklearn.metrics import recall_score, Scorer 11 | 12 | from pascal_helpers import (load_pascal, load_kraehenbuehl, 13 | load_pascal_pixelwise) 14 | 15 | from latent_crf_experiments.hierarchical_segmentation import \ 16 | get_km_segments 17 | from latent_crf_experiments.utils import (eval_on_sp, eval_on_pixels, gt_in_sp, 18 | add_edges) 19 | 20 | from datasets.pascal import PascalSegmentation 21 | 22 | #from msrc_helpers import SimpleSplitCV 23 | 24 | from IPython.core.debugger import Tracer 25 | tracer = Tracer() 26 | np.set_printoptions(precision=2) 27 | 28 | 29 | def train_svm(C=0.1, grid=False): 30 | ds = PascalSegmentation() 31 | svm = LinearSVC(C=C, dual=False, class_weight='auto') 32 | 33 | if grid: 34 | data_train = load_pascal("kTrain") 35 | X, y = shuffle(data_train.X, data_train.Y) 36 | # prepare leave-one-label-out by assigning labels to images 37 | image_indicators = np.hstack([np.repeat(i, len(x)) for i, x in 38 | enumerate(X)]) 39 | # go down to only 5 "folds" 40 | labels = image_indicators % 5 41 | X, y = np.vstack(X), np.hstack(y) 42 | 43 | cv = LeavePLabelOut(labels=labels, p=1) 44 | param_grid = {'C': 10. ** np.arange(-3, 3)} 45 | scorer = Scorer(recall_score, average="macro") 46 | grid_search = GridSearchCV(svm, param_grid=param_grid, cv=cv, 47 | verbose=10, scoring=scorer, n_jobs=-1) 48 | grid_search.fit(X, y) 49 | else: 50 | data_train = load_pascal("train") 51 | X, y = np.vstack(data_train.X), np.hstack(data_train.Y) 52 | svm.fit(X, y) 53 | print(svm.score(X, y)) 54 | eval_on_sp(ds, data_train, [svm.predict(x) for x in data_train.X], 55 | print_results=True) 56 | 57 | data_val = load_pascal("val") 58 | eval_on_sp(ds, data_val, [svm.predict(x) for x in data_val.X], 59 | print_results=True) 60 | 61 | 62 | def visualize_pascal(plot_probabilities=False): 63 | data = load_pascal('val') 64 | ds = PascalSegmentation() 65 | for x, y, f, sps in zip(data.X, data.Y, data.file_names, data.superpixels): 66 | fig, ax = plt.subplots(2, 3) 67 | ax = ax.ravel() 68 | image = ds.get_image(f) 69 | y_pixel = ds.get_ground_truth(f) 70 | x_raw = load_kraehenbuehl(f) 71 | 72 | boundary_image = mark_boundaries(image, sps) 73 | 74 | ax[0].imshow(image) 75 | ax[1].imshow(y_pixel, cmap=ds.cmap) 76 | ax[2].imshow(boundary_image) 77 | ax[3].imshow(np.argmax(x_raw, axis=-1), cmap=ds.cmap, vmin=0, vmax=256) 78 | ax[4].imshow(y[sps], cmap=ds.cmap, vmin=0, vmax=256) 79 | ax[5].imshow(np.argmax(x, axis=-1)[sps], cmap=ds.cmap, vmin=0, 80 | vmax=256) 81 | for a in ax: 82 | a.set_xticks(()) 83 | a.set_yticks(()) 84 | plt.savefig("figures_pascal_val/%s.png" % f, bbox_inches='tight') 85 | plt.close() 86 | if plot_probabilities: 87 | fig, ax = plt.subplots(3, 7) 88 | for k in range(21): 89 | ax.ravel()[k].matshow(x[:, :, k], vmin=0, vmax=1) 90 | for a in ax.ravel(): 91 | a.set_xticks(()) 92 | a.set_yticks(()) 93 | plt.savefig("figures_pascal_val/%s_prob.png" % f, 94 | bbox_inches='tight') 95 | plt.close() 96 | tracer() 97 | 98 | 99 | def eval_pixel_prediction(): 100 | data = load_pascal_pixelwise('val') 101 | predictions = [np.argmax(x, axis=-1) for x in data.X] 102 | hamming, jaccard = eval_on_pixels(data.Y, predictions, print_results=True) 103 | tracer() 104 | 105 | 106 | def eval_sp_prediction(): 107 | data = load_pascal('val') 108 | predictions = [np.argmax(x, axis=-1) for x in data.X] 109 | hamming, jaccard = eval_on_sp(data, predictions, print_results=True) 110 | tracer() 111 | 112 | 113 | def eval_segment_best_possible(): 114 | ds = PascalSegmentation() 115 | print("loading") 116 | data = load_pascal('train') 117 | print("getting edges") 118 | data = add_edges(data) 119 | print("computing segments") 120 | segments = [get_km_segments(x, ds.get_image(image_name), sps, 121 | n_segments=25) for x, image_name, sps in 122 | zip(data.X, data.file_names, data.superpixels)] 123 | print("combining superpixels") 124 | segments = [seg[sp] for seg, sp in zip(segments, data.superpixels)] 125 | predictions = [gt_in_sp(ds, f, seg)[seg] 126 | for seg, f in zip(segments, data.file_names)] 127 | Y_true = [ds.get_ground_truth(f) for f in data.file_names] 128 | hamming, jaccard = eval_on_pixels(ds, Y_true, predictions, 129 | print_results=True) 130 | tracer() 131 | 132 | 133 | def eval_spixel_best_possible(): 134 | data = load_pascal('kTrain', sp_type='cpmc') 135 | pascal = PascalSegmentation() 136 | hamming, jaccard = eval_on_sp(pascal, data, data.Y, print_results=True) 137 | 138 | if __name__ == "__main__": 139 | #visualize_pascal() 140 | #eval_spixel_best_possible() 141 | #eval_pixel_prediction() 142 | #eval_sp_prediction() 143 | for C in [1, 100]: 144 | train_svm(C=C) 145 | #eval_segment_best_possible() 146 | -------------------------------------------------------------------------------- /pascal/pascal_bow.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from sklearn.svm import LinearSVC 4 | from sklearn.kernel_approximation import AdditiveChi2Sampler 5 | 6 | #from sklearn.cross_validation import LeavePLabelOut 7 | #from sklearn.grid_search import GridSearchCV 8 | #from sklearn.utils import shuffle 9 | #from sklearn.metrics import recall_score, Scorer 10 | from slic_python import slic_n 11 | 12 | from latent_crf_experiments.utils import eval_on_sp, add_global_descriptor 13 | from latent_crf_experiments.bow import SiftBOW 14 | 15 | from datasets.pascal import PascalSegmentation 16 | 17 | from IPython.core.debugger import Tracer 18 | tracer = Tracer() 19 | 20 | 21 | def train_svm(C=0.1, grid=False): 22 | pascal = PascalSegmentation() 23 | 24 | files_train = pascal.get_split("kTrain") 25 | superpixels = [slic_n(pascal.get_image(f), n_superpixels=100, 26 | compactness=10) 27 | for f in files_train] 28 | bow = SiftBOW(pascal, n_words=1000, color_sift=True) 29 | data_train = bow.fit_transform(files_train, superpixels) 30 | 31 | data_train = add_global_descriptor(data_train) 32 | 33 | svm = LinearSVC(C=C, dual=False, class_weight='auto') 34 | chi2 = AdditiveChi2Sampler() 35 | 36 | X, y = np.vstack(data_train.X), np.hstack(data_train.Y) 37 | X = chi2.fit_transform(X) 38 | svm.fit(X, y) 39 | print(svm.score(X, y)) 40 | eval_on_sp(pascal, data_train, [svm.predict(chi2.transform(x)) for x in 41 | data_train.X], print_results=True) 42 | 43 | files_val = pascal.get_split("kVal") 44 | superpixels_val = [slic_n(pascal.get_image(f), n_superpixels=100, 45 | compactness=10) for f in files_val] 46 | data_val = bow.transform(files_val, superpixels_val) 47 | data_val = add_global_descriptor(data_val) 48 | eval_on_sp(pascal, data_val, [svm.predict(chi2.transform(x)) for x in 49 | data_val.X], print_results=True) 50 | 51 | tracer() 52 | 53 | if __name__ == "__main__": 54 | train_svm(C=.1, grid=False) 55 | -------------------------------------------------------------------------------- /pascal/pascal_colors.txt: -------------------------------------------------------------------------------- 1 | 0.0000000e+00 0.0000000e+00 0.0000000e+00 2 | 5.0196078e-01 0.0000000e+00 0.0000000e+00 3 | 0.0000000e+00 5.0196078e-01 0.0000000e+00 4 | 5.0196078e-01 5.0196078e-01 0.0000000e+00 5 | 0.0000000e+00 0.0000000e+00 5.0196078e-01 6 | 5.0196078e-01 0.0000000e+00 5.0196078e-01 7 | 0.0000000e+00 5.0196078e-01 5.0196078e-01 8 | 5.0196078e-01 5.0196078e-01 5.0196078e-01 9 | 2.5098039e-01 0.0000000e+00 0.0000000e+00 10 | 7.5294118e-01 0.0000000e+00 0.0000000e+00 11 | 2.5098039e-01 5.0196078e-01 0.0000000e+00 12 | 7.5294118e-01 5.0196078e-01 0.0000000e+00 13 | 2.5098039e-01 0.0000000e+00 5.0196078e-01 14 | 7.5294118e-01 0.0000000e+00 5.0196078e-01 15 | 2.5098039e-01 5.0196078e-01 5.0196078e-01 16 | 7.5294118e-01 5.0196078e-01 5.0196078e-01 17 | 0.0000000e+00 2.5098039e-01 0.0000000e+00 18 | 5.0196078e-01 2.5098039e-01 0.0000000e+00 19 | 0.0000000e+00 7.5294118e-01 0.0000000e+00 20 | 5.0196078e-01 7.5294118e-01 0.0000000e+00 21 | 0.0000000e+00 2.5098039e-01 5.0196078e-01 22 | 5.0196078e-01 2.5098039e-01 5.0196078e-01 23 | 0.0000000e+00 7.5294118e-01 5.0196078e-01 24 | 5.0196078e-01 7.5294118e-01 5.0196078e-01 25 | 2.5098039e-01 2.5098039e-01 0.0000000e+00 26 | 7.5294118e-01 2.5098039e-01 0.0000000e+00 27 | 2.5098039e-01 7.5294118e-01 0.0000000e+00 28 | 7.5294118e-01 7.5294118e-01 0.0000000e+00 29 | 2.5098039e-01 2.5098039e-01 5.0196078e-01 30 | 7.5294118e-01 2.5098039e-01 5.0196078e-01 31 | 2.5098039e-01 7.5294118e-01 5.0196078e-01 32 | 7.5294118e-01 7.5294118e-01 5.0196078e-01 33 | 0.0000000e+00 0.0000000e+00 2.5098039e-01 34 | 5.0196078e-01 0.0000000e+00 2.5098039e-01 35 | 0.0000000e+00 5.0196078e-01 2.5098039e-01 36 | 5.0196078e-01 5.0196078e-01 2.5098039e-01 37 | 0.0000000e+00 0.0000000e+00 7.5294118e-01 38 | 5.0196078e-01 0.0000000e+00 7.5294118e-01 39 | 0.0000000e+00 5.0196078e-01 7.5294118e-01 40 | 5.0196078e-01 5.0196078e-01 7.5294118e-01 41 | 2.5098039e-01 0.0000000e+00 2.5098039e-01 42 | 7.5294118e-01 0.0000000e+00 2.5098039e-01 43 | 2.5098039e-01 5.0196078e-01 2.5098039e-01 44 | 7.5294118e-01 5.0196078e-01 2.5098039e-01 45 | 2.5098039e-01 0.0000000e+00 7.5294118e-01 46 | 7.5294118e-01 0.0000000e+00 7.5294118e-01 47 | 2.5098039e-01 5.0196078e-01 7.5294118e-01 48 | 7.5294118e-01 5.0196078e-01 7.5294118e-01 49 | 0.0000000e+00 2.5098039e-01 2.5098039e-01 50 | 5.0196078e-01 2.5098039e-01 2.5098039e-01 51 | 0.0000000e+00 7.5294118e-01 2.5098039e-01 52 | 5.0196078e-01 7.5294118e-01 2.5098039e-01 53 | 0.0000000e+00 2.5098039e-01 7.5294118e-01 54 | 5.0196078e-01 2.5098039e-01 7.5294118e-01 55 | 0.0000000e+00 7.5294118e-01 7.5294118e-01 56 | 5.0196078e-01 7.5294118e-01 7.5294118e-01 57 | 2.5098039e-01 2.5098039e-01 2.5098039e-01 58 | 7.5294118e-01 2.5098039e-01 2.5098039e-01 59 | 2.5098039e-01 7.5294118e-01 2.5098039e-01 60 | 7.5294118e-01 7.5294118e-01 2.5098039e-01 61 | 2.5098039e-01 2.5098039e-01 7.5294118e-01 62 | 7.5294118e-01 2.5098039e-01 7.5294118e-01 63 | 2.5098039e-01 7.5294118e-01 7.5294118e-01 64 | 7.5294118e-01 7.5294118e-01 7.5294118e-01 65 | 1.2549020e-01 0.0000000e+00 0.0000000e+00 66 | 6.2745098e-01 0.0000000e+00 0.0000000e+00 67 | 1.2549020e-01 5.0196078e-01 0.0000000e+00 68 | 6.2745098e-01 5.0196078e-01 0.0000000e+00 69 | 1.2549020e-01 0.0000000e+00 5.0196078e-01 70 | 6.2745098e-01 0.0000000e+00 5.0196078e-01 71 | 1.2549020e-01 5.0196078e-01 5.0196078e-01 72 | 6.2745098e-01 5.0196078e-01 5.0196078e-01 73 | 3.7647059e-01 0.0000000e+00 0.0000000e+00 74 | 8.7843137e-01 0.0000000e+00 0.0000000e+00 75 | 3.7647059e-01 5.0196078e-01 0.0000000e+00 76 | 8.7843137e-01 5.0196078e-01 0.0000000e+00 77 | 3.7647059e-01 0.0000000e+00 5.0196078e-01 78 | 8.7843137e-01 0.0000000e+00 5.0196078e-01 79 | 3.7647059e-01 5.0196078e-01 5.0196078e-01 80 | 8.7843137e-01 5.0196078e-01 5.0196078e-01 81 | 1.2549020e-01 2.5098039e-01 0.0000000e+00 82 | 6.2745098e-01 2.5098039e-01 0.0000000e+00 83 | 1.2549020e-01 7.5294118e-01 0.0000000e+00 84 | 6.2745098e-01 7.5294118e-01 0.0000000e+00 85 | 1.2549020e-01 2.5098039e-01 5.0196078e-01 86 | 6.2745098e-01 2.5098039e-01 5.0196078e-01 87 | 1.2549020e-01 7.5294118e-01 5.0196078e-01 88 | 6.2745098e-01 7.5294118e-01 5.0196078e-01 89 | 3.7647059e-01 2.5098039e-01 0.0000000e+00 90 | 8.7843137e-01 2.5098039e-01 0.0000000e+00 91 | 3.7647059e-01 7.5294118e-01 0.0000000e+00 92 | 8.7843137e-01 7.5294118e-01 0.0000000e+00 93 | 3.7647059e-01 2.5098039e-01 5.0196078e-01 94 | 8.7843137e-01 2.5098039e-01 5.0196078e-01 95 | 3.7647059e-01 7.5294118e-01 5.0196078e-01 96 | 8.7843137e-01 7.5294118e-01 5.0196078e-01 97 | 1.2549020e-01 0.0000000e+00 2.5098039e-01 98 | 6.2745098e-01 0.0000000e+00 2.5098039e-01 99 | 1.2549020e-01 5.0196078e-01 2.5098039e-01 100 | 6.2745098e-01 5.0196078e-01 2.5098039e-01 101 | 1.2549020e-01 0.0000000e+00 7.5294118e-01 102 | 6.2745098e-01 0.0000000e+00 7.5294118e-01 103 | 1.2549020e-01 5.0196078e-01 7.5294118e-01 104 | 6.2745098e-01 5.0196078e-01 7.5294118e-01 105 | 3.7647059e-01 0.0000000e+00 2.5098039e-01 106 | 8.7843137e-01 0.0000000e+00 2.5098039e-01 107 | 3.7647059e-01 5.0196078e-01 2.5098039e-01 108 | 8.7843137e-01 5.0196078e-01 2.5098039e-01 109 | 3.7647059e-01 0.0000000e+00 7.5294118e-01 110 | 8.7843137e-01 0.0000000e+00 7.5294118e-01 111 | 3.7647059e-01 5.0196078e-01 7.5294118e-01 112 | 8.7843137e-01 5.0196078e-01 7.5294118e-01 113 | 1.2549020e-01 2.5098039e-01 2.5098039e-01 114 | 6.2745098e-01 2.5098039e-01 2.5098039e-01 115 | 1.2549020e-01 7.5294118e-01 2.5098039e-01 116 | 6.2745098e-01 7.5294118e-01 2.5098039e-01 117 | 1.2549020e-01 2.5098039e-01 7.5294118e-01 118 | 6.2745098e-01 2.5098039e-01 7.5294118e-01 119 | 1.2549020e-01 7.5294118e-01 7.5294118e-01 120 | 6.2745098e-01 7.5294118e-01 7.5294118e-01 121 | 3.7647059e-01 2.5098039e-01 2.5098039e-01 122 | 8.7843137e-01 2.5098039e-01 2.5098039e-01 123 | 3.7647059e-01 7.5294118e-01 2.5098039e-01 124 | 8.7843137e-01 7.5294118e-01 2.5098039e-01 125 | 3.7647059e-01 2.5098039e-01 7.5294118e-01 126 | 8.7843137e-01 2.5098039e-01 7.5294118e-01 127 | 3.7647059e-01 7.5294118e-01 7.5294118e-01 128 | 8.7843137e-01 7.5294118e-01 7.5294118e-01 129 | 0.0000000e+00 1.2549020e-01 0.0000000e+00 130 | 5.0196078e-01 1.2549020e-01 0.0000000e+00 131 | 0.0000000e+00 6.2745098e-01 0.0000000e+00 132 | 5.0196078e-01 6.2745098e-01 0.0000000e+00 133 | 0.0000000e+00 1.2549020e-01 5.0196078e-01 134 | 5.0196078e-01 1.2549020e-01 5.0196078e-01 135 | 0.0000000e+00 6.2745098e-01 5.0196078e-01 136 | 5.0196078e-01 6.2745098e-01 5.0196078e-01 137 | 2.5098039e-01 1.2549020e-01 0.0000000e+00 138 | 7.5294118e-01 1.2549020e-01 0.0000000e+00 139 | 2.5098039e-01 6.2745098e-01 0.0000000e+00 140 | 7.5294118e-01 6.2745098e-01 0.0000000e+00 141 | 2.5098039e-01 1.2549020e-01 5.0196078e-01 142 | 7.5294118e-01 1.2549020e-01 5.0196078e-01 143 | 2.5098039e-01 6.2745098e-01 5.0196078e-01 144 | 7.5294118e-01 6.2745098e-01 5.0196078e-01 145 | 0.0000000e+00 3.7647059e-01 0.0000000e+00 146 | 5.0196078e-01 3.7647059e-01 0.0000000e+00 147 | 0.0000000e+00 8.7843137e-01 0.0000000e+00 148 | 5.0196078e-01 8.7843137e-01 0.0000000e+00 149 | 0.0000000e+00 3.7647059e-01 5.0196078e-01 150 | 5.0196078e-01 3.7647059e-01 5.0196078e-01 151 | 0.0000000e+00 8.7843137e-01 5.0196078e-01 152 | 5.0196078e-01 8.7843137e-01 5.0196078e-01 153 | 2.5098039e-01 3.7647059e-01 0.0000000e+00 154 | 7.5294118e-01 3.7647059e-01 0.0000000e+00 155 | 2.5098039e-01 8.7843137e-01 0.0000000e+00 156 | 7.5294118e-01 8.7843137e-01 0.0000000e+00 157 | 2.5098039e-01 3.7647059e-01 5.0196078e-01 158 | 7.5294118e-01 3.7647059e-01 5.0196078e-01 159 | 2.5098039e-01 8.7843137e-01 5.0196078e-01 160 | 7.5294118e-01 8.7843137e-01 5.0196078e-01 161 | 0.0000000e+00 1.2549020e-01 2.5098039e-01 162 | 5.0196078e-01 1.2549020e-01 2.5098039e-01 163 | 0.0000000e+00 6.2745098e-01 2.5098039e-01 164 | 5.0196078e-01 6.2745098e-01 2.5098039e-01 165 | 0.0000000e+00 1.2549020e-01 7.5294118e-01 166 | 5.0196078e-01 1.2549020e-01 7.5294118e-01 167 | 0.0000000e+00 6.2745098e-01 7.5294118e-01 168 | 5.0196078e-01 6.2745098e-01 7.5294118e-01 169 | 2.5098039e-01 1.2549020e-01 2.5098039e-01 170 | 7.5294118e-01 1.2549020e-01 2.5098039e-01 171 | 2.5098039e-01 6.2745098e-01 2.5098039e-01 172 | 7.5294118e-01 6.2745098e-01 2.5098039e-01 173 | 2.5098039e-01 1.2549020e-01 7.5294118e-01 174 | 7.5294118e-01 1.2549020e-01 7.5294118e-01 175 | 2.5098039e-01 6.2745098e-01 7.5294118e-01 176 | 7.5294118e-01 6.2745098e-01 7.5294118e-01 177 | 0.0000000e+00 3.7647059e-01 2.5098039e-01 178 | 5.0196078e-01 3.7647059e-01 2.5098039e-01 179 | 0.0000000e+00 8.7843137e-01 2.5098039e-01 180 | 5.0196078e-01 8.7843137e-01 2.5098039e-01 181 | 0.0000000e+00 3.7647059e-01 7.5294118e-01 182 | 5.0196078e-01 3.7647059e-01 7.5294118e-01 183 | 0.0000000e+00 8.7843137e-01 7.5294118e-01 184 | 5.0196078e-01 8.7843137e-01 7.5294118e-01 185 | 2.5098039e-01 3.7647059e-01 2.5098039e-01 186 | 7.5294118e-01 3.7647059e-01 2.5098039e-01 187 | 2.5098039e-01 8.7843137e-01 2.5098039e-01 188 | 7.5294118e-01 8.7843137e-01 2.5098039e-01 189 | 2.5098039e-01 3.7647059e-01 7.5294118e-01 190 | 7.5294118e-01 3.7647059e-01 7.5294118e-01 191 | 2.5098039e-01 8.7843137e-01 7.5294118e-01 192 | 7.5294118e-01 8.7843137e-01 7.5294118e-01 193 | 1.2549020e-01 1.2549020e-01 0.0000000e+00 194 | 6.2745098e-01 1.2549020e-01 0.0000000e+00 195 | 1.2549020e-01 6.2745098e-01 0.0000000e+00 196 | 6.2745098e-01 6.2745098e-01 0.0000000e+00 197 | 1.2549020e-01 1.2549020e-01 5.0196078e-01 198 | 6.2745098e-01 1.2549020e-01 5.0196078e-01 199 | 1.2549020e-01 6.2745098e-01 5.0196078e-01 200 | 6.2745098e-01 6.2745098e-01 5.0196078e-01 201 | 3.7647059e-01 1.2549020e-01 0.0000000e+00 202 | 8.7843137e-01 1.2549020e-01 0.0000000e+00 203 | 3.7647059e-01 6.2745098e-01 0.0000000e+00 204 | 8.7843137e-01 6.2745098e-01 0.0000000e+00 205 | 3.7647059e-01 1.2549020e-01 5.0196078e-01 206 | 8.7843137e-01 1.2549020e-01 5.0196078e-01 207 | 3.7647059e-01 6.2745098e-01 5.0196078e-01 208 | 8.7843137e-01 6.2745098e-01 5.0196078e-01 209 | 1.2549020e-01 3.7647059e-01 0.0000000e+00 210 | 6.2745098e-01 3.7647059e-01 0.0000000e+00 211 | 1.2549020e-01 8.7843137e-01 0.0000000e+00 212 | 6.2745098e-01 8.7843137e-01 0.0000000e+00 213 | 1.2549020e-01 3.7647059e-01 5.0196078e-01 214 | 6.2745098e-01 3.7647059e-01 5.0196078e-01 215 | 1.2549020e-01 8.7843137e-01 5.0196078e-01 216 | 6.2745098e-01 8.7843137e-01 5.0196078e-01 217 | 3.7647059e-01 3.7647059e-01 0.0000000e+00 218 | 8.7843137e-01 3.7647059e-01 0.0000000e+00 219 | 3.7647059e-01 8.7843137e-01 0.0000000e+00 220 | 8.7843137e-01 8.7843137e-01 0.0000000e+00 221 | 3.7647059e-01 3.7647059e-01 5.0196078e-01 222 | 8.7843137e-01 3.7647059e-01 5.0196078e-01 223 | 3.7647059e-01 8.7843137e-01 5.0196078e-01 224 | 8.7843137e-01 8.7843137e-01 5.0196078e-01 225 | 1.2549020e-01 1.2549020e-01 2.5098039e-01 226 | 6.2745098e-01 1.2549020e-01 2.5098039e-01 227 | 1.2549020e-01 6.2745098e-01 2.5098039e-01 228 | 6.2745098e-01 6.2745098e-01 2.5098039e-01 229 | 1.2549020e-01 1.2549020e-01 7.5294118e-01 230 | 6.2745098e-01 1.2549020e-01 7.5294118e-01 231 | 1.2549020e-01 6.2745098e-01 7.5294118e-01 232 | 6.2745098e-01 6.2745098e-01 7.5294118e-01 233 | 3.7647059e-01 1.2549020e-01 2.5098039e-01 234 | 8.7843137e-01 1.2549020e-01 2.5098039e-01 235 | 3.7647059e-01 6.2745098e-01 2.5098039e-01 236 | 8.7843137e-01 6.2745098e-01 2.5098039e-01 237 | 3.7647059e-01 1.2549020e-01 7.5294118e-01 238 | 8.7843137e-01 1.2549020e-01 7.5294118e-01 239 | 3.7647059e-01 6.2745098e-01 7.5294118e-01 240 | 8.7843137e-01 6.2745098e-01 7.5294118e-01 241 | 1.2549020e-01 3.7647059e-01 2.5098039e-01 242 | 6.2745098e-01 3.7647059e-01 2.5098039e-01 243 | 1.2549020e-01 8.7843137e-01 2.5098039e-01 244 | 6.2745098e-01 8.7843137e-01 2.5098039e-01 245 | 1.2549020e-01 3.7647059e-01 7.5294118e-01 246 | 6.2745098e-01 3.7647059e-01 7.5294118e-01 247 | 1.2549020e-01 8.7843137e-01 7.5294118e-01 248 | 6.2745098e-01 8.7843137e-01 7.5294118e-01 249 | 3.7647059e-01 3.7647059e-01 2.5098039e-01 250 | 8.7843137e-01 3.7647059e-01 2.5098039e-01 251 | 3.7647059e-01 8.7843137e-01 2.5098039e-01 252 | 8.7843137e-01 8.7843137e-01 2.5098039e-01 253 | 3.7647059e-01 3.7647059e-01 7.5294118e-01 254 | 8.7843137e-01 3.7647059e-01 7.5294118e-01 255 | 3.7647059e-01 8.7843137e-01 7.5294118e-01 256 | 8.7843137e-01 8.7843137e-01 7.5294118e-01 257 | -------------------------------------------------------------------------------- /pascal/pascal_crf.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from pystruct import learners 4 | import pystruct.models as crfs 5 | from pystruct.utils import SaveLogger 6 | 7 | from datasets.pascal import PascalSegmentation 8 | from pascal_helpers import load_pascal 9 | from latent_crf_experiments.utils import (discard_void, add_edges, 10 | add_edge_features, eval_on_sp) 11 | 12 | 13 | from IPython.core.debugger import Tracer 14 | tracer = Tracer() 15 | 16 | 17 | def main(C=1, test=False): 18 | ds = PascalSegmentation() 19 | # load training data 20 | edge_type = "pairwise" 21 | if test: 22 | which = "train" 23 | else: 24 | which = "kTrain" 25 | data_train = load_pascal(which=which, sp_type="cpmc") 26 | 27 | data_train = add_edges(data_train, edge_type) 28 | data_train = add_edge_features(ds, data_train) 29 | data_train = discard_void(ds, data_train, ds.void_label) 30 | 31 | print("number of samples: %s" % len(data_train.X)) 32 | class_weights = 1. / np.bincount(np.hstack(data_train.Y)) 33 | class_weights *= 21. / np.sum(class_weights) 34 | print(class_weights) 35 | #model = crfs.GraphCRF(n_states=n_states, 36 | #n_features=data_train.X[0][0].shape[1], 37 | #inference_method='qpbo', class_weight=class_weights) 38 | model = crfs.EdgeFeatureGraphCRF(inference_method='qpbo', 39 | class_weight=class_weights, 40 | symmetric_edge_features=[0, 1], 41 | antisymmetric_edge_features=[2]) 42 | experiment_name = "cpmc_edge_features_trainval_new_%f" % C 43 | #warm_start = True 44 | warm_start = False 45 | ssvm = learners.OneSlackSSVM( 46 | model, verbose=2, C=C, max_iter=100000, n_jobs=-1, 47 | tol=0.0001, show_loss_every=50, inference_cache=50, cache_tol='auto', 48 | logger=SaveLogger(experiment_name + ".pickle", save_every=100), 49 | inactive_threshold=1e-5, break_on_bad=False, inactive_window=50, 50 | switch_to=None) 51 | #ssvm = learners.SubgradientSSVM( 52 | #model, verbose=3, C=C, max_iter=10000, n_jobs=-1, show_loss_every=10, 53 | #logger=SaveLogger(experiment_name + ".pickle", save_every=10), 54 | #momentum=0, learning_rate=0.1, decay_exponent=1, decay_t0=100) 55 | 56 | if warm_start: 57 | ssvm = SaveLogger(experiment_name + ".pickle").load() 58 | ssvm.logger = SaveLogger( 59 | file_name=experiment_name + "_refit.pickle", 60 | save_every=10) 61 | #ssvm.learning_rate = 0.000001 62 | 63 | ssvm.model.inference_method = 'ad3bb' 64 | #ssvm.n_jobs = 1 65 | 66 | ssvm.fit(data_train.X, data_train.Y, warm_start=warm_start) 67 | return 68 | 69 | print("fit finished!") 70 | if test: 71 | data_val = load_pascal('val') 72 | else: 73 | data_val = load_pascal('kVal') 74 | 75 | data_val = add_edges(data_val, edge_type) 76 | data_val = add_edge_features(ds, data_val, more_colors=True) 77 | eval_on_sp(ds, data_val, ssvm.predict(data_val.X), print_results=True) 78 | 79 | if __name__ == "__main__": 80 | #for C in 10. ** np.arange(-4, 2): 81 | #main(C) 82 | main(0.01, test=True) 83 | -------------------------------------------------------------------------------- /pascal/pascal_helpers.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from scipy.io import loadmat 3 | 4 | from sklearn.externals.joblib import Memory, Parallel, delayed 5 | from skimage import morphology 6 | from skimage.segmentation import boundaries 7 | #from skimage.measure import regionprops 8 | from skimage.filter import sobel 9 | from skimage.color import rgb2gray 10 | from slic_python import slic_n 11 | 12 | from datasets.pascal import PascalSegmentation 13 | from latent_crf_experiments.utils import (gt_in_sp, region_graph, 14 | get_mean_colors, DataBunch, 15 | DataBunchNoSP, probabilities_on_sp) 16 | from latent_crf_experiments.hierarchical_segmentation \ 17 | import HierarchicalDataBunch 18 | 19 | 20 | memory = Memory(cachedir="/tmp/cache") 21 | pascal_path = "/home/local/datasets/VOC2011/TrainVal/VOCdevkit/VOC2011" 22 | segments_path = ("/home/user/amueller/tools/cpmc_new/" 23 | "cpmc_release1/data/MySegmentsMat") 24 | 25 | 26 | 27 | def load_kraehenbuehl(filename): 28 | path = "/home/user/amueller/local/voc_potentials_kraehenbuehl/unaries/" 29 | with open(path + filename + ".unary") as f: 30 | size = np.fromfile(f, dtype=np.uint32, count=3).byteswap() 31 | data = np.fromfile(f, dtype=np.float32).byteswap() 32 | img = data.reshape(size[1], size[0], size[2]) 33 | return img 34 | 35 | 36 | @memory.cache 37 | def load_pascal_pixelwise(which='train', year="2010"): 38 | pascal = PascalSegmentation() 39 | if which not in ["train", "val"]: 40 | raise ValueError("Expected 'which' to be 'train' or 'val', got %s." % 41 | which) 42 | split_file = pascal_path + "/ImageSets/Segmentation/%s.txt" % which 43 | files = np.loadtxt(split_file, dtype=np.str) 44 | files = [f for f in files if f.split("_")[0] <= year] 45 | X, Y = [], [] 46 | for f in files: 47 | X.append(load_kraehenbuehl(f)) 48 | Y.append(pascal.get_ground_truth(f)) 49 | 50 | return DataBunchNoSP(X, Y, files) 51 | 52 | 53 | def load_pascal_single(f, sp_type, which, pascal): 54 | print(f) 55 | image = pascal.get_image(f) 56 | if sp_type == "slic": 57 | sp = slic_n(image, n_superpixels=100, compactness=10) 58 | segments = None 59 | elif sp_type == "cpmc": 60 | segments, sp = superpixels_segments(f) 61 | sp, _ = merge_small_sp(image, sp) 62 | sp = morphological_clean_sp(image, sp, 4) 63 | else: 64 | raise ValueError("Expected sp to be 'slic' or 'cpmc', got %s" % 65 | sp_type) 66 | x = get_kraehenbuehl_pot_sp(f, sp) 67 | if which != "test": 68 | y = gt_in_sp(pascal, f, sp) 69 | return x, y, sp, segments 70 | 71 | 72 | @memory.cache 73 | def load_pascal(which='train', year="2010", sp_type="slic", n_jobs=-1): 74 | pascal = PascalSegmentation() 75 | files = pascal.get_split(which=which, year=year) 76 | results = Parallel(n_jobs=n_jobs)(delayed(load_pascal_single)( 77 | f, which=which, sp_type=sp_type, pascal=pascal) for f in files) 78 | X, Y, superpixels, segments = zip(*results) 79 | if sp_type == "slic": 80 | return DataBunch(X, Y, files, superpixels) 81 | else: 82 | return HierarchicalDataBunch(X, Y, files, superpixels, segments) 83 | 84 | 85 | def get_kraehenbuehl_pot_sp(filename, superpixels): 86 | probs = load_kraehenbuehl(filename) 87 | ds = PascalSegmentation() 88 | return probabilities_on_sp(ds, probs, superpixels) 89 | 90 | 91 | def superpixels_segments(filename): 92 | mat_file = segments_path + "/" + filename 93 | segments = loadmat(mat_file)['top_masks'] 94 | n_segments = segments.shape[2] 95 | if n_segments > 100: 96 | raise ValueError("Need to rewrite... float only holds so many values.") 97 | # 2**50 + 1 is different from 2 ** 5. But 2 ** 50 + 2 ** -50 is not 98 | # different from 2 ** 50 so we do this in two stages 99 | added = (segments[:, :, :50] * 100 | 2. ** np.arange(min(50, n_segments))).sum(axis=-1) 101 | _, added = np.unique(added, return_inverse=True) 102 | 103 | if n_segments > 50: 104 | added2 = (segments[:, :, 50:] * 105 | 2. ** np.arange(n_segments - 50)).sum(axis=-1) 106 | _, added2 = np.unique(added2, return_inverse=True) 107 | added = added + (np.max(added) + 1) * added2 108 | _, added = np.unique(added, return_inverse=True) 109 | 110 | labels = morphology.label(added.reshape(segments.shape[:2]), neighbors=4) 111 | 112 | return segments, labels 113 | 114 | 115 | def get_pb(filename): 116 | pb = loadmat(segments_path[:-13] + "PB/" + filename + 117 | "_PB.mat")['gPb_thin'] 118 | return pb 119 | 120 | 121 | def merge_small_sp(image, regions, min_size=None): 122 | if min_size is None: 123 | min_size = np.prod(image.shape[:2]) / float(np.max(regions) + 1) 124 | shape = regions.shape 125 | _, regions = np.unique(regions, return_inverse=True) 126 | regions = regions.reshape(shape[:2]) 127 | edges = region_graph(regions) 128 | mean_colors = get_mean_colors(image, regions) 129 | mask = np.bincount(regions.ravel()) < min_size 130 | # mapping of old labels to new labels 131 | new_labels = np.arange(len(np.unique(regions))) 132 | for r in np.where(mask)[0]: 133 | # get neighbors: 134 | where_0 = edges[:, 0] == r 135 | where_1 = edges[:, 1] == r 136 | neighbors1 = edges[where_0, 1] 137 | neighbors2 = edges[where_1, 0] 138 | neighbors = np.concatenate([neighbors1, neighbors2]) 139 | neighbors = neighbors[neighbors != r] 140 | # get closest in color 141 | distances = np.sum((mean_colors[r] - mean_colors[neighbors]) ** 2, 142 | axis=-1) 143 | nearest = np.argmin(distances) 144 | # merge 145 | new = neighbors[nearest] 146 | new_labels[new_labels == r] = new 147 | edges[where_0, 0] = new 148 | edges[where_1, 1] = new 149 | regions = new_labels[regions] 150 | _, regions = np.unique(regions, return_inverse=True) 151 | regions = regions.reshape(shape[:2]) 152 | grr = np.bincount(regions.ravel()) < min_size 153 | if np.any(grr): 154 | from IPython.core.debugger import Tracer 155 | Tracer()() 156 | return regions, new_labels 157 | 158 | 159 | def morphological_clean_sp(image, segments, diameter=4): 160 | # remove small / thin segments by morphological closing + watershed 161 | # extract boundaries 162 | boundary = boundaries.find_boundaries(segments) 163 | closed = morphology.binary_closing(boundary, np.ones((diameter, diameter))) 164 | # extract regions 165 | labels = morphology.label(closed, neighbors=4, background=1) 166 | # watershed to get rid of boundaries 167 | # interestingly we can't use gPb here. It is to sharp. 168 | edge_image = sobel(rgb2gray(image)) 169 | result = morphology.watershed(edge_image, labels + 1) 170 | # we want them to start at zero! 171 | return result - 1 172 | 173 | 174 | def create_segment_sp_graph(segments, superpixels): 175 | #n_segments = segments.shape[2] 176 | n_superpixels = len(np.unique(superpixels)) 177 | assert(n_superpixels == np.max(superpixels) + 1) 178 | edges = [] 179 | for sp in range(n_superpixels): 180 | sp_indicator = superpixels == sp 181 | overlaps = segments[sp_indicator].sum(axis=0) 182 | includes = overlaps > np.sum(sp_indicator) / 2. 183 | for i in np.where(includes)[0]: 184 | edges.append([sp, i]) 185 | return np.array(edges) 186 | 187 | 188 | @memory.cache 189 | def make_cpmc_hierarchy(dataset, data): 190 | X_new = [] 191 | all_edges = Parallel(n_jobs=-1)( 192 | delayed(create_segment_sp_graph)(segments, superpixels) 193 | for superpixels, segments in zip(data.superpixels, data.segments)) 194 | for x, superpixels, segments, edges in zip(data.X, data.superpixels, 195 | data.segments, all_edges): 196 | n_superpixels = len(np.unique(superpixels)) 197 | n_segments = segments.shape[2] 198 | edges[:, 1] += n_superpixels 199 | X_new.append((x, edges, n_segments)) 200 | return HierarchicalDataBunch(X_new, data.Y, data.file_names, 201 | data.superpixels, data.segments) 202 | -------------------------------------------------------------------------------- /pascal/tests_helpers.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from datasets.pascal import PascalSegmentation 4 | from pascal_helpers import superpixels_segments, merge_small_sp 5 | 6 | 7 | def test_remove_small_segments(): 8 | pascal = PascalSegmentation() 9 | train_files = pascal.get_split() 10 | 11 | idx = 10 12 | image = pascal.get_image(train_files[idx]) 13 | segments, superpixels = superpixels_segments(train_files[idx]) 14 | new_regions, correspondences = merge_small_sp(image, superpixels) 15 | new_counts = np.bincount(new_regions.ravel()) 16 | if np.any(new_counts < 50): 17 | raise ValueError("Stupid thing!") 18 | -------------------------------------------------------------------------------- /pascal/visualize_segment_sps.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | 4 | 5 | from scipy.misc import imsave 6 | from datasets.pascal import PascalSegmentation 7 | from latent_crf_experiments.utils import get_superpixel_centers 8 | from pascal_helpers import (superpixels_segments, merge_small_sp, 9 | morphological_clean_sp, create_segment_sp_graph) 10 | 11 | from skimage.segmentation import mark_boundaries 12 | from skimage.measure import regionprops 13 | 14 | 15 | def visualize_sps(): 16 | pascal = PascalSegmentation() 17 | train_files = pascal.get_split() 18 | 19 | for image_file in train_files: 20 | print(image_file) 21 | image = pascal.get_image(image_file) 22 | segments, superpixels = superpixels_segments(image_file) 23 | new_regions, correspondences = merge_small_sp(image, superpixels) 24 | clean_regions = morphological_clean_sp(image, new_regions, 4) 25 | imsave("segment_sp_fixed/%s.png" 26 | % image_file, mark_boundaries(image, clean_regions)) 27 | 28 | 29 | def visualize_segments(): 30 | pascal = PascalSegmentation() 31 | train_files = pascal.get_split() 32 | 33 | for image_file in train_files: 34 | print(image_file) 35 | image = pascal.get_image(image_file) 36 | segments, superpixels = superpixels_segments(image_file) 37 | new_regions, correspondences = merge_small_sp(image, superpixels) 38 | clean_regions = morphological_clean_sp(image, new_regions, 4) 39 | new_regions, correspondences = merge_small_sp(image, superpixels) 40 | clean_regions = morphological_clean_sp(image, new_regions, 4) 41 | marked = mark_boundaries(image, clean_regions) 42 | edges = create_segment_sp_graph(segments, clean_regions) 43 | edges = np.array(edges) 44 | n_segments = segments.shape[2] 45 | segment_centers = [regionprops(segments.astype(np.int)[:, :, i], 46 | ['Centroid'])[0]['Centroid'] for i in 47 | range(n_segments)] 48 | segment_centers = np.vstack(segment_centers)[:, ::-1] 49 | superpixel_centers = get_superpixel_centers(clean_regions) 50 | grr = min(n_segments, 10) 51 | fig, axes = plt.subplots(3, grr // 3, figsize=(30, 30)) 52 | 53 | for i, ax in enumerate(axes.ravel()): 54 | ax.imshow(mark_boundaries(marked, segments[:, :, i], (1, 0, 0))) 55 | ax.scatter(segment_centers[:, 0], segment_centers[:, 1], 56 | color='red') 57 | ax.scatter(superpixel_centers[:, 0], superpixel_centers[:, 1], 58 | color='blue') 59 | this_edges = edges[edges[:, 1] == i] 60 | for edge in this_edges: 61 | ax.plot([superpixel_centers[edge[0]][0], 62 | segment_centers[edge[1]][0]], 63 | [superpixel_centers[edge[0]][1], 64 | segment_centers[edge[1]][1]], c='black') 65 | ax.set_xlim(0, superpixels.shape[1]) 66 | ax.set_ylim(superpixels.shape[0], 0) 67 | ax.set_xticks(()) 68 | ax.set_yticks(()) 69 | #plt.show() 70 | #imsave("segments_test/%s.png" % image_file, marked) 71 | plt.savefig("segments_test/%s.png" % image_file) 72 | plt.close() 73 | 74 | if __name__ == "__main__": 75 | visualize_sps() 76 | #visualize_segments() 77 | -------------------------------------------------------------------------------- /plotting.py: -------------------------------------------------------------------------------- 1 | import os 2 | import matplotlib.pyplot as plt 3 | import numpy as np 4 | 5 | from skimage.segmentation import mark_boundaries 6 | 7 | from utils import get_superpixel_centers 8 | 9 | 10 | def plot_results(dataset, data, Y_pred, folder="figures", 11 | use_colors_predict=True): 12 | if not os.path.exists(folder): 13 | os.mkdir(folder) 14 | import matplotlib.colors as cl 15 | np.random.seed(0) 16 | random_colormap = cl.ListedColormap(np.random.uniform(size=(100, 3))) 17 | for image_name, superpixels, y, y_pred in zip(data.file_names, 18 | data.superpixels, data.Y, 19 | Y_pred): 20 | image = dataset.get_image(image_name) 21 | fig, axes = plt.subplots(2, 3, figsize=(12, 6)) 22 | axes[0, 0].imshow(image) 23 | axes[0, 1].set_title("ground truth") 24 | axes[0, 1].imshow(image) 25 | gt = dataset.get_ground_truth(image_name) 26 | axes[0, 1].imshow(gt, alpha=.7, cmap=dataset.cmap) 27 | axes[1, 0].set_title("sp ground truth") 28 | axes[1, 0].imshow(image) 29 | axes[1, 0].imshow(y[superpixels], vmin=0, vmax=dataset.cmap.N, 30 | alpha=.7, cmap=dataset.cmap) 31 | 32 | axes[1, 1].set_title("prediction") 33 | axes[1, 1].imshow(image) 34 | if use_colors_predict: 35 | axes[1, 1].imshow(y_pred[superpixels], alpha=.7, vmin=0, 36 | vmax=dataset.cmap.N, cmap=dataset.cmap) 37 | else: 38 | vmax = np.max(np.hstack(Y_pred)) 39 | axes[1, 1].imshow(y_pred[superpixels], vmin=0, vmax=vmax, alpha=.9, 40 | cmap=random_colormap) 41 | if use_colors_predict: 42 | present_y = np.unique(np.hstack([y, y_pred])) 43 | else: 44 | present_y = np.unique(y) 45 | present_y = present_y[present_y != dataset.void_label] 46 | axes[0, 2].imshow(present_y[:, np.newaxis], interpolation='nearest', 47 | cmap=dataset.cmap, vmin=0, vmax=dataset.cmap.N) 48 | for i, c in enumerate(present_y): 49 | axes[0, 2].text(1, i, dataset.classes[c]) 50 | for ax in axes.ravel(): 51 | ax.set_xticks(()) 52 | ax.set_yticks(()) 53 | axes[1, 2].set_visible(False) 54 | fig.savefig(folder + "/%s.png" % image_name, bbox_inches="tight") 55 | plt.close(fig) 56 | 57 | 58 | def plot_sp_graph(image, superpixels, edges, colors='black'): 59 | plt.figure(figsize=(10, 10)) 60 | bounary_image = mark_boundaries(image, superpixels) 61 | plt.imshow(bounary_image) 62 | centers = get_superpixel_centers(superpixels) 63 | 64 | for i, edge in enumerate(edges): 65 | e0, e1 = edge 66 | if len(colors) == len(edges): 67 | color = colors[i] 68 | else: 69 | color = colors 70 | plt.plot([centers[e0][0], centers[e1][0]], 71 | [centers[e0][1], centers[e1][1]], 72 | c=color) 73 | plt.scatter(centers[:, 0], centers[:, 1], s=100) 74 | plt.tight_layout() 75 | plt.xlim(0, superpixels.shape[1]) 76 | plt.ylim(superpixels.shape[0], 0) 77 | plt.axis("off") 78 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amueller/segmentation/f6d252135d91985002fdaccc59214a7199dabed9/tests/__init__.py -------------------------------------------------------------------------------- /tests/test_ignore_void_crf.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from numpy.testing.utils import assert_array_equal 4 | from nose.tools import assert_almost_equal, assert_true 5 | 6 | from pystruct.learners import SubgradientStructuredSVM 7 | from pystruct.utils import find_constraint, exhaustive_inference 8 | 9 | from ..ignore_void_crf import IgnoreVoidCRF 10 | 11 | from IPython.core.debugger import Tracer 12 | tracer = Tracer() 13 | 14 | # make simple binary problem 15 | X = np.random.uniform(size=(20, 3, 2)) 16 | Y = (X[:, :, 0] > .5).astype(np.int) 17 | # add void around decision boundary 18 | Y[np.abs(X[:, :, 0] - 0.5) < 0.1] = 2 19 | X = [(x, np.empty(shape=(0, 2), dtype=np.int)) for x in X] 20 | 21 | 22 | def test_inference(): 23 | crf = IgnoreVoidCRF(n_states=3, n_features=2, void_label=2, 24 | inference_method='lp') 25 | # set some random weights, do inference and check that everything is ok 26 | np.random.seed(0) 27 | for x, y in zip(X, Y): 28 | w = np.random.normal(size=crf.size_psi) 29 | y_hat, energy = crf.inference(x, w, relaxed=True, 30 | return_energy=True) 31 | energy_svm = np.dot(w, crf.psi(x, y_hat)) 32 | assert_almost_equal(energy_svm, -energy) 33 | y_hat_exhaustive = exhaustive_inference(crf, x, w) 34 | 35 | y_hat = crf.inference(x, w) 36 | assert_array_equal(y_hat, y_hat_exhaustive) 37 | 38 | 39 | def test_loss_augmented_inference(): 40 | crf = IgnoreVoidCRF(n_states=3, n_features=2, void_label=2, 41 | inference_method='lp') 42 | np.random.seed(0) 43 | for x, y in zip(X, Y): 44 | w = np.random.normal(size=crf.size_psi) 45 | y_hat, energy = crf.loss_augmented_inference(x, y, w, relaxed=True, 46 | return_energy=True) 47 | assert_almost_equal(np.dot(crf.psi(x, y_hat), w) + 48 | crf.continuous_loss(y, y_hat[0]), -energy) 49 | 50 | 51 | def test_learning(): 52 | crf = IgnoreVoidCRF(n_states=3, n_features=2, void_label=2, 53 | inference_method='lp') 54 | ssvm = SubgradientStructuredSVM(crf, verbose=10, C=100, n_jobs=1, 55 | max_iter=50, learning_rate=0.01) 56 | ssvm.fit(X, Y) 57 | 58 | for x in X: 59 | y_hat_exhaustive = exhaustive_inference(crf, x, ssvm.w) 60 | y_hat = crf.inference(x, ssvm.w) 61 | assert_array_equal(y_hat, y_hat_exhaustive) 62 | 63 | constr = [find_constraint(crf, x, y, ssvm.w, y_hat=y_hat) 64 | for x, y, y_hat in zip(X, Y, ssvm.predict(X))] 65 | losses = [c[3] for c in constr] 66 | slacks = [c[2] for c in constr] 67 | assert_true(np.all(np.array(slacks) >= np.array(losses))) 68 | -------------------------------------------------------------------------------- /toy_experiments/directional_bars.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | 4 | from sklearn.cross_validation import train_test_split 5 | 6 | from pystruct.problems import LatentDirectionalGridCRF 7 | from pystruct.learners import LatentSSVM 8 | 9 | import pystruct.toy_datasets as toy 10 | 11 | from IPython.core.debugger import Tracer 12 | tracer = Tracer() 13 | 14 | 15 | def main(): 16 | # get some data 17 | X, Y = toy.generate_bars(n_samples=40, noise=10, total_size=10) 18 | X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=.5) 19 | 20 | # train a latent grid crf 21 | n_labels = len(np.unique(Y_train)) 22 | crf = LatentDirectionalGridCRF(n_labels=n_labels, n_states_per_label=3, 23 | inference_method='lp') 24 | clf = LatentSSVM(problem=crf, max_iter=50, C=10., verbose=2, 25 | check_constraints=True, n_jobs=-1, break_on_bad=True, 26 | plot=True) 27 | clf.fit(X_train, Y_train) 28 | 29 | # the rest is plotting 30 | for X_, Y_, H, name in [[X_train, Y_train, clf.H_init_, "train"], 31 | [X_test, Y_test, [None] * len(X_test), "test"]]: 32 | Y_pred = clf.predict(X_) 33 | i = 0 34 | loss = 0 35 | for x, y, h_init, y_pred in zip(X_, Y_, H, Y_pred): 36 | loss += np.sum(y != y_pred / crf.n_states_per_label) 37 | fig, ax = plt.subplots(3, 2) 38 | ax[0, 0].matshow(y * crf.n_states_per_label, 39 | vmin=0, vmax=crf.n_states - 1) 40 | ax[0, 0].set_title("ground truth") 41 | unary_pred = np.argmax(x, axis=-1) * crf.n_states_per_label 42 | ax[0, 1].matshow(unary_pred, vmin=0, vmax=crf.n_states - 1) 43 | ax[0, 1].set_title("unaries only") 44 | if h_init is None: 45 | ax[1, 0].set_visible(False) 46 | else: 47 | ax[1, 0].matshow(h_init, vmin=0, vmax=crf.n_states - 1) 48 | ax[1, 0].set_title("latent initial") 49 | ax[1, 1].matshow(crf.latent(x, y, clf.w), 50 | vmin=0, vmax=crf.n_states - 1) 51 | ax[1, 1].set_title("latent final") 52 | ax[2, 0].matshow(y_pred, vmin=0, vmax=crf.n_states - 1) 53 | ax[2, 0].set_title("prediction") 54 | ax[2, 1].matshow((y_pred // crf.n_states_per_label) 55 | * crf.n_states_per_label, 56 | vmin=0, vmax=crf.n_states - 1) 57 | ax[2, 1].set_title("prediction") 58 | for a in ax.ravel(): 59 | a.set_xticks(()) 60 | a.set_yticks(()) 61 | fig.savefig("data_%s_%03d.png" % (name, i), bbox_inches="tight") 62 | i += 1 63 | print("loss %s set: %f" % (name, loss)) 64 | #print(clf.w) 65 | 66 | if __name__ == "__main__": 67 | main() 68 | -------------------------------------------------------------------------------- /toy_experiments/directional_bars_joint.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | 4 | from sklearn.cross_validation import train_test_split 5 | 6 | from pystruct.problems import LatentDirectionalGridCRF 7 | from pystruct.learners import LatentSSVM 8 | 9 | import pystruct.toy_datasets as toy 10 | 11 | from IPython.core.debugger import Tracer 12 | tracer = Tracer() 13 | 14 | 15 | def main(): 16 | # get some data 17 | X, Y = toy.generate_bars(n_samples=40, noise=10, total_size=10, 18 | separate_labels=False) 19 | X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=.5) 20 | 21 | # train a latent grid crf 22 | n_labels = len(np.unique(Y_train)) 23 | crf = LatentDirectionalGridCRF(n_labels=n_labels, 24 | n_states_per_label=[1, 6], 25 | inference_method='lp') 26 | clf = LatentSSVM(problem=crf, max_iter=50, C=10., verbose=2, 27 | check_constraints=True, n_jobs=-1, break_on_bad=False, 28 | tol=-10, base_svm='1-slack') 29 | clf.fit(X_train, Y_train) 30 | 31 | # the rest is plotting 32 | for X_, Y_, H, name in [[X_train, Y_train, clf.H_init_, "train"], 33 | [X_test, Y_test, [None] * len(X_test), "test"]]: 34 | Y_pred = clf.predict(X_) 35 | i = 0 36 | loss = 0 37 | for x, y, h_init, y_pred in zip(X_, Y_, H, Y_pred): 38 | loss += np.sum(y != y_pred) 39 | fig, ax = plt.subplots(3, 2) 40 | ax[0, 0].matshow(y, 41 | vmin=0, vmax=crf.n_labels - 1) 42 | ax[0, 0].set_title("ground truth") 43 | unary_pred = np.argmax(x, axis=-1) 44 | ax[0, 1].matshow(unary_pred, vmin=0, vmax=crf.n_labels - 1) 45 | ax[0, 1].set_title("unaries only") 46 | if h_init is None: 47 | ax[1, 0].set_visible(False) 48 | else: 49 | ax[1, 0].matshow(h_init, vmin=0, vmax=crf.n_states - 1) 50 | ax[1, 0].set_title("latent initial") 51 | ax[1, 1].matshow(crf.latent(x, y, clf.w), 52 | vmin=0, vmax=crf.n_states - 1) 53 | ax[1, 1].set_title("latent final") 54 | ax[2, 0].matshow(crf.inference(x, clf.w), 55 | vmin=0, vmax=crf.n_states - 1) 56 | ax[2, 0].set_title("prediction") 57 | ax[2, 1].matshow(y_pred, vmin=0, vmax=crf.n_labels - 1) 58 | ax[2, 1].set_title("prediction") 59 | for a in ax.ravel(): 60 | a.set_xticks(()) 61 | a.set_yticks(()) 62 | fig.savefig("data_%s_%03d.png" % (name, i), bbox_inches="tight") 63 | i += 1 64 | print("loss %s set: %f" % (name, loss)) 65 | #print(clf.w) 66 | 67 | if __name__ == "__main__": 68 | main() 69 | -------------------------------------------------------------------------------- /toy_experiments/harder_crosses.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | 4 | from sklearn.cross_validation import train_test_split 5 | 6 | from pystruct.problems import LatentGridCRF 7 | from pystruct.learners import LatentSSVM 8 | 9 | import pystruct.toy_datasets as toy 10 | 11 | from IPython.core.debugger import Tracer 12 | tracer = Tracer() 13 | 14 | 15 | def main(): 16 | X, Y = toy.generate_crosses(n_samples=40, noise=8, n_crosses=2, 17 | total_size=10) 18 | X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=.5) 19 | n_labels = len(np.unique(Y_train)) 20 | crf = LatentGridCRF(n_labels=n_labels, n_states_per_label=2, 21 | inference_method='lp') 22 | clf = LatentSSVM(problem=crf, max_iter=50, C=1000., verbose=2, 23 | check_constraints=True, n_jobs=-1, break_on_bad=True, 24 | plot=True) 25 | clf.fit(X_train, Y_train) 26 | 27 | for X_, Y_, H, name in [[X_train, Y_train, clf.H_init_, "train"], 28 | [X_test, Y_test, [None] * len(X_test), "test"]]: 29 | Y_pred = clf.predict(X_) 30 | i = 0 31 | loss = 0 32 | for x, y, h_init, y_pred in zip(X_, Y_, H, Y_pred): 33 | loss += np.sum(y != y_pred / crf.n_states_per_label) 34 | fig, ax = plt.subplots(3, 2) 35 | ax[0, 0].matshow(y * crf.n_states_per_label, 36 | vmin=0, vmax=crf.n_states - 1) 37 | ax[0, 0].set_title("ground truth") 38 | unary_params = np.repeat(np.eye(2), 2, axis=1) 39 | pairwise_params = np.zeros(10) 40 | w_unaries_only = np.hstack([unary_params.ravel(), 41 | pairwise_params.ravel()]) 42 | unary_pred = crf.inference(x, w_unaries_only) 43 | ax[0, 1].matshow(unary_pred, vmin=0, vmax=crf.n_states - 1) 44 | ax[0, 1].set_title("unaries only") 45 | if h_init is None: 46 | ax[1, 0].set_visible(False) 47 | else: 48 | ax[1, 0].matshow(h_init, vmin=0, vmax=crf.n_states - 1) 49 | ax[1, 0].set_title("latent initial") 50 | ax[1, 1].matshow(crf.latent(x, y, clf.w), 51 | vmin=0, vmax=crf.n_states - 1) 52 | ax[1, 1].set_title("latent final") 53 | ax[2, 0].matshow(y_pred, vmin=0, vmax=crf.n_states - 1) 54 | ax[2, 0].set_title("prediction") 55 | ax[2, 1].matshow((y_pred // crf.n_states_per_label) 56 | * crf.n_states_per_label, 57 | vmin=0, vmax=crf.n_states - 1) 58 | ax[2, 1].set_title("prediction") 59 | for a in ax.ravel(): 60 | a.set_xticks(()) 61 | a.set_yticks(()) 62 | fig.savefig("data_%s_%03d.png" % (name, i), bbox_inches="tight") 63 | i += 1 64 | print("loss %s set: %f" % (name, loss)) 65 | print(clf.w) 66 | 67 | if __name__ == "__main__": 68 | main() 69 | -------------------------------------------------------------------------------- /toy_experiments/simple_crosses.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | 4 | from sklearn.cross_validation import train_test_split 5 | 6 | from pystruct.problems import LatentGridCRF 7 | from pystruct.learners import LatentSSVM 8 | 9 | import pystruct.toy_datasets as toy 10 | 11 | from IPython.core.debugger import Tracer 12 | tracer = Tracer() 13 | 14 | 15 | def main(): 16 | X, Y = toy.generate_crosses(n_samples=20, noise=5, n_crosses=1, 17 | total_size=8) 18 | X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=.5) 19 | n_labels = len(np.unique(Y_train)) 20 | crf = LatentGridCRF(n_labels=n_labels, n_states_per_label=[1, 2], 21 | inference_method='lp') 22 | clf = LatentSSVM(problem=crf, max_iter=50, C=1000., verbose=2, 23 | check_constraints=True, n_jobs=-1, break_on_bad=True) 24 | clf.fit(X_train, Y_train) 25 | 26 | i = 0 27 | for X_, Y_, H, name in [[X_train, Y_train, clf.H_init_, "train"], 28 | [X_test, Y_test, [None] * len(X_test), "test"]]: 29 | Y_pred = clf.predict(X_) 30 | score = clf.score(X_, Y_) 31 | for x, y, h_init, y_pred in zip(X_, Y_, H, Y_pred): 32 | fig, ax = plt.subplots(4, 1) 33 | ax[0].matshow(y, vmin=0, vmax=crf.n_labels - 1) 34 | ax[0].set_title("Ground truth") 35 | ax[1].matshow(np.argmax(x, axis=-1), vmin=0, vmax=crf.n_labels - 1) 36 | ax[1].set_title("Unaries only") 37 | #if h_init is None: 38 | #ax[1, 0].set_visible(False) 39 | #else: 40 | #ax[1, 0].matshow(h_init, vmin=0, vmax=crf.n_states - 1) 41 | #ax[1, 0].set_title("latent initial") 42 | #ax[2].matshow(crf.latent(x, y, clf.w), 43 | #vmin=0, vmax=crf.n_states - 1) 44 | #ax[2].set_title("latent final") 45 | ax[2].matshow(crf.inference(x, clf.w), vmin=0, vmax=crf.n_states 46 | - 1) 47 | ax[2].set_title("Prediction for h") 48 | ax[3].matshow(y_pred, vmin=0, vmax=crf.n_labels - 1) 49 | ax[3].set_title("Prediction for y") 50 | for a in ax.ravel(): 51 | a.set_xticks(()) 52 | a.set_yticks(()) 53 | plt.subplots_adjust(hspace=.5) 54 | fig.savefig("data_%s_%03d.png" % (name, i), bbox_inches="tight", 55 | dpi=400) 56 | i += 1 57 | print("score %s set: %f" % (name, score)) 58 | print(clf.w) 59 | 60 | if __name__ == "__main__": 61 | main() 62 | -------------------------------------------------------------------------------- /toy_experiments/square_with_hole.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | 4 | from sklearn.cross_validation import train_test_split 5 | 6 | from pystruct.problems import LatentDirectionalGridCRF 7 | from pystruct.learners import LatentSSVM 8 | 9 | import pystruct.toy_datasets as toy 10 | 11 | from IPython.core.debugger import Tracer 12 | tracer = Tracer() 13 | 14 | 15 | def main(): 16 | # get some data 17 | X, Y = toy.generate_square_with_hole(n_samples=80, noise=5, total_size=7) 18 | X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=.5) 19 | 20 | # train a latent grid crf 21 | n_labels = len(np.unique(Y_train)) 22 | crf = LatentDirectionalGridCRF(n_labels=n_labels, n_states_per_label=8, 23 | inference_method='lp') 24 | clf = LatentSSVM(problem=crf, max_iter=50, C=100., verbose=2, 25 | check_constraints=True, n_jobs=-1, break_on_bad=True, 26 | plot=True) 27 | clf.fit(X_train, Y_train) 28 | 29 | # the rest is plotting 30 | for X_, Y_, H, name in [[X_train, Y_train, clf.H_init_, "train"], 31 | [X_test, Y_test, [None] * len(X_test), "test"]]: 32 | Y_pred = clf.predict(X_) 33 | i = 0 34 | loss = 0 35 | for x, y, h_init, y_pred in zip(X_, Y_, H, Y_pred): 36 | loss += np.sum(y != y_pred / crf.n_states_per_label) 37 | fig, ax = plt.subplots(3, 2) 38 | ax[0, 0].matshow(y * crf.n_states_per_label, 39 | vmin=0, vmax=crf.n_states - 1) 40 | ax[0, 0].set_title("ground truth") 41 | unary_pred = np.argmax(x, axis=-1) * crf.n_states_per_label 42 | ax[0, 1].matshow(unary_pred, vmin=0, vmax=crf.n_states - 1) 43 | ax[0, 1].set_title("unaries only") 44 | if h_init is None: 45 | ax[1, 0].set_visible(False) 46 | else: 47 | ax[1, 0].matshow(h_init, vmin=0, vmax=crf.n_states - 1) 48 | ax[1, 0].set_title("latent initial") 49 | ax[1, 1].matshow(crf.latent(x, y, clf.w), 50 | vmin=0, vmax=crf.n_states - 1) 51 | ax[1, 1].set_title("latent final") 52 | ax[2, 0].matshow(y_pred, vmin=0, vmax=crf.n_states - 1) 53 | ax[2, 0].set_title("prediction") 54 | ax[2, 1].matshow((y_pred // crf.n_states_per_label) 55 | * crf.n_states_per_label, 56 | vmin=0, vmax=crf.n_states - 1) 57 | ax[2, 1].set_title("prediction") 58 | for a in ax.ravel(): 59 | a.set_xticks(()) 60 | a.set_yticks(()) 61 | fig.savefig("data_%s_%03d.png" % (name, i), bbox_inches="tight") 62 | i += 1 63 | print("loss %s set: %f" % (name, loss)) 64 | #print(clf.w) 65 | 66 | if __name__ == "__main__": 67 | main() 68 | -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | from collections import namedtuple 2 | import numbers 3 | import itertools 4 | 5 | import numpy as np 6 | from scipy import sparse 7 | 8 | from sklearn.kernel_approximation import AdditiveChi2Sampler 9 | from sklearn.externals.joblib import Memory, Parallel, delayed 10 | from pystruct.utils import make_grid_edges 11 | 12 | DataBunch = namedtuple('DataBunch', 'X, Y, file_names, superpixels') 13 | DataBunchNoSP = namedtuple('DataBunchNoSP', 'X, Y, file_names') 14 | 15 | memory = Memory(cachedir="/home/data/amueller/cache", verbose=1) 16 | 17 | 18 | @memory.cache 19 | def transform_chi2(data): 20 | chi2 = AdditiveChi2Sampler(sample_steps=2) 21 | if isinstance(data.X[0], np.ndarray): 22 | X_new = [chi2.fit_transform(x).astype(np.float32) for x in data.X] 23 | elif len(data.X[0]) == 2: 24 | X_new = [(chi2.fit_transform(x[0]), x[1]) for x in data.X] 25 | elif len(data.X[0]) == 3: 26 | X_new = [(chi2.fit_transform(x[0]), x[1], x[2]) for x in data.X] 27 | else: 28 | raise ValueError("len(x) is weird: %d" % len(data.X[0])) 29 | 30 | return DataBunch(X_new, data.Y, data.file_names, data.superpixels) 31 | 32 | 33 | @memory.cache 34 | def discard_void(dataset, data, latent_features=False): 35 | if isinstance(data.X[0], np.ndarray): 36 | X_new = [x[y != dataset.void_label] for x, y in zip(data.X, data.Y)] 37 | Y_new = [y[y != dataset.void_label] for y in data.Y] 38 | return DataBunch(X_new, Y_new, data.file_names, 39 | data.superpixels) 40 | X_new, Y_new = [], [] 41 | for x, y in zip(data.X, data.Y): 42 | mask = y != dataset.void_label 43 | voids = np.where(~mask)[0] 44 | 45 | if len(x) == 2: 46 | features, edges = x 47 | elif len(x) == 3: 48 | if isinstance(x[2], numbers.Integral): 49 | features, edges, n_hidden = x 50 | mask = np.hstack([mask, np.ones(n_hidden, dtype=np.bool)]) 51 | else: 52 | features, edges, edge_features = x 53 | edge_features_new = edge_features 54 | elif len(x) == 4: 55 | features, edges, edge_features, n_hidden = x 56 | edge_features_new = edge_features 57 | mask = np.hstack([mask, np.ones(n_hidden, dtype=np.bool)]) 58 | else: 59 | raise ValueError("len(x) is weird: %d" % len(data.X[0])) 60 | 61 | edges_new = edges 62 | if edges_new.shape[0] > 0: 63 | # if there are no edges, don't need to filter them 64 | # also, below code would break ;) 65 | for void_node in voids: 66 | involves_void_node = np.any(edges_new == void_node, axis=1) 67 | edges_new = edges_new[~involves_void_node] 68 | if (len(x) == 3 and not isinstance(x[2], numbers.Integral) or 69 | len(x) == 4): 70 | edge_features_new = edge_features_new[~involves_void_node] 71 | 72 | reindex_edges = np.zeros(len(mask), dtype=np.int) 73 | reindex_edges[mask] = np.arange(np.sum(mask)) 74 | edges_new = reindex_edges[edges_new] 75 | if len(x) == 2: 76 | X_new.append((features[mask], edges_new)) 77 | Y_new.append(y[mask]) 78 | elif len(x) == 3: 79 | if isinstance(x[2], numbers.Integral): 80 | n_hidden_new = np.max(edges_new) - np.sum(mask[:-n_hidden]) + 1 81 | if latent_features: 82 | X_new.append((features[mask], edges_new, n_hidden_new)) 83 | else: 84 | X_new.append((features[mask[:-n_hidden]], edges_new, 85 | n_hidden_new)) 86 | Y_new.append(y[mask[:-n_hidden]]) 87 | #X_new.append((features[mask], edges_new, n_hidden_new)) 88 | #Y_new.append(y[mask[:-n_hidden]]) 89 | else: 90 | X_new.append((features[mask], edges_new, edge_features_new)) 91 | Y_new.append(y[mask]) 92 | else: 93 | n_hidden_new = np.max(edges_new) - np.sum(mask[:-n_hidden]) + 1 94 | X_new.append((features[mask[:-n_hidden]], edges_new, 95 | edge_features_new, n_hidden_new)) 96 | Y_new.append(y[mask[:-n_hidden]]) 97 | return DataBunch(X_new, Y_new, data.file_names, data.superpixels) 98 | 99 | 100 | @memory.cache 101 | def add_edges(data, kind="pairwise"): 102 | # generate graph 103 | if kind == "independent": 104 | X_new = [(x, np.empty((0, 2), dtype=np.int)) for x in data.X] 105 | 106 | elif kind == "extended": 107 | X_new = [(x, extend_edges(region_graph(sp), length=3)) 108 | for x, sp in zip(data.X, data.superpixels)] 109 | 110 | elif kind == "fully_connected": 111 | X_new = [(x, np.vstack([e for e in 112 | itertools.combinations(np.arange(len(x)), 2)])) 113 | for x in data.X] 114 | elif kind == "pairwise": 115 | X_new = [(x, region_graph(sp)) 116 | for x, sp in zip(data.X, data.superpixels)] 117 | else: 118 | raise ValueError("Parameter 'kind' should be one of 'independent'" 119 | ",'fully_connected' or 'pairwise', got %s" 120 | % kind) 121 | 122 | return DataBunch(X_new, data.Y, data.file_names, data.superpixels) 123 | 124 | 125 | def radius_graph(superpixels, eps=40): 126 | n_vertices = np.max(superpixels) + 1 127 | centers = np.empty((n_vertices, 2)) 128 | gridx, gridy = np.mgrid[:superpixels.shape[0], :superpixels.shape[1]] 129 | 130 | for v in xrange(n_vertices): 131 | centers[v] = [gridy[superpixels == v].mean(), 132 | gridx[superpixels == v].mean()] 133 | edges = [] 134 | for e in itertools.combinations(np.arange(n_vertices), 2): 135 | if np.linalg.norm(centers[e[0]] - centers[e[1]]) < eps: 136 | edges.append(e) 137 | return np.vstack(edges) 138 | 139 | 140 | def region_graph(regions): 141 | edges = make_grid_edges(regions) 142 | n_vertices = np.max(regions) + 1 143 | 144 | crossings = edges[regions.ravel()[edges[:, 0]] 145 | != regions.ravel()[edges[:, 1]]] 146 | edges = regions.ravel()[crossings] 147 | edges = np.sort(edges, axis=1) 148 | crossing_hash = (edges[:, 0] + n_vertices * edges[:, 1]) 149 | # find unique connections 150 | unique_hash = np.unique(crossing_hash) 151 | # undo hashing 152 | unique_crossings = np.c_[unique_hash % n_vertices, 153 | unique_hash // n_vertices] 154 | return unique_crossings 155 | 156 | 157 | def extend_edges(edges, length=2): 158 | # returns all paths of length one or two in the graph given by edges 159 | n_vertices = np.max(edges) + 1 160 | graph = sparse.coo_matrix((np.ones(len(edges)), edges.T), 161 | shape=(n_vertices, n_vertices)) 162 | neighborhood = graph 163 | for i in range(length - 1): 164 | graph = graph + neighborhood * graph 165 | return np.c_[graph.nonzero()] 166 | 167 | 168 | def get_mean_colors(image, superpixels): 169 | r = np.bincount(superpixels.ravel(), weights=image[:, :, 0].ravel()) 170 | g = np.bincount(superpixels.ravel(), weights=image[:, :, 1].ravel()) 171 | b = np.bincount(superpixels.ravel(), weights=image[:, :, 2].ravel()) 172 | mean_colors = (np.vstack([r, g, b]) 173 | / np.bincount(superpixels.ravel())).T / 255. 174 | return mean_colors 175 | 176 | 177 | def get_edge_contrast(edges, image, superpixels, gamma=10): 178 | mean_colors = get_mean_colors(image, superpixels) 179 | contrasts = [np.exp(-gamma * np.linalg.norm(mean_colors[e[0]] 180 | - mean_colors[e[1]])) 181 | for e in edges] 182 | return np.vstack(contrasts) 183 | 184 | 185 | def get_edge_depth_diff(edges, depth, superpixels): 186 | mean_depth = np.bincount(superpixels.ravel(), weights=depth.ravel()) 187 | mean_depth = mean_depth / np.bincount(superpixels.ravel()).T 188 | #depth_diff = [np.exp(-gamma * np.linalg.norm(mean_depth[e[0]] 189 | #- mean_depth[e[1]])) 190 | #for e in edges] 191 | depth_diff = [(mean_depth[e[0]] - mean_depth[e[1]]) 192 | for e in edges] 193 | depth_diff = np.vstack(depth_diff) 194 | return depth_diff / np.abs(depth_diff).max() 195 | 196 | 197 | 198 | def get_superpixel_centers(superpixels): 199 | n_vertices = np.max(superpixels) + 1 200 | centers = np.empty((n_vertices, 2)) 201 | gridx, gridy = np.mgrid[:superpixels.shape[0], :superpixels.shape[1]] 202 | for v in xrange(n_vertices): 203 | centers[v] = [gridy[superpixels == v].mean(), 204 | gridx[superpixels == v].mean()] 205 | return centers 206 | 207 | 208 | def get_center_distances(edges, superpixels): 209 | centers = get_superpixel_centers(superpixels) 210 | distances = np.sum((centers[edges[:, 0]] - centers[edges[:, 1]]) ** 2, 211 | axis=1) 212 | distances -= distances.min() 213 | distances /= distances.max() 214 | return np.exp(-distances[:, np.newaxis] * 2.) 215 | 216 | 217 | def get_edge_directions(edges, superpixels): 218 | centers = get_superpixel_centers(superpixels) 219 | 220 | directions = [] 221 | for edge in edges: 222 | e0, e1 = edge 223 | diff = centers[e0] - centers[e1] 224 | diff /= np.linalg.norm(diff) 225 | directions.append(np.arcsin(diff[1])) 226 | return np.vstack(directions) 227 | 228 | 229 | def get_sp_normals(normals, superpixels): 230 | normals_flat = normals.reshape(-1, 3) 231 | n_sp = np.max(superpixels) + 1 232 | mask = np.isfinite(normals_flat.sum(axis=1)) 233 | superpixels = superpixels.ravel()[mask] 234 | normals_flat = normals_flat[mask] 235 | x = np.bincount(superpixels, weights=normals_flat[:, 0], minlength=n_sp) 236 | y = np.bincount(superpixels, weights=normals_flat[:, 1], minlength=n_sp) 237 | z = np.bincount(superpixels, weights=normals_flat[:, 2], minlength=n_sp) 238 | mean_normals = np.vstack([x, y, z]).T 239 | lengths = np.maximum(1e-3, np.sqrt(np.sum(mean_normals ** 2, axis=1))) 240 | mean_normals /= lengths[:, np.newaxis] 241 | return mean_normals 242 | 243 | 244 | def get_normal_angles(edges, normals, superpixels): 245 | mean_normals = get_sp_normals(normals, superpixels) 246 | 247 | product = np.sum(mean_normals[edges[:, 0]] * mean_normals[edges[:, 1]], axis=1) 248 | product[~np.isfinite(product)] = 0 249 | return (1 - np.arccos(np.abs(product)) * 2. / np.pi)[:, np.newaxis] 250 | 251 | 252 | def edge_features_single(dataset, x, superpixels, file_name, more_colors=False, 253 | depth_diff=False, center_distances=False, normal_angles=False): 254 | edges = x[1] 255 | features = [np.ones((edges.shape[0], 1))] 256 | image = dataset.get_image(file_name) 257 | if more_colors: 258 | features.append(get_edge_contrast(edges, image, superpixels, 259 | gamma=5)) 260 | features.append(get_edge_contrast(edges, image, superpixels, 261 | gamma=10)) 262 | features.append(get_edge_contrast(edges, image, superpixels, 263 | gamma=20)) 264 | features.append(get_edge_contrast(edges, image, superpixels, 265 | gamma=100)) 266 | else: 267 | features.append(get_edge_contrast(edges, image, superpixels, 268 | gamma=10)) 269 | if depth_diff: 270 | depth = dataset.get_depth(file_name) 271 | features.append(get_edge_depth_diff(edges, depth, superpixels)) 272 | 273 | if center_distances: 274 | features.append(get_center_distances(edges, superpixels)) 275 | 276 | if normal_angles: 277 | normals = dataset.get_pointcloud_normals(file_name)[:, :, 3:] 278 | features.append(get_normal_angles(edges, normals, superpixels)) 279 | 280 | features.append(get_edge_directions(edges, superpixels)) 281 | return np.hstack(features) 282 | 283 | 284 | @memory.cache 285 | def add_edge_features(dataset, data, more_colors=False, center_distances=False, 286 | depth_diff=False, normal_angles=False): 287 | # trigger cache .. 288 | all_edge_features = Parallel(n_jobs=1, verbose=4)( 289 | delayed(edge_features_single)(dataset, x, superpixels, file_name, 290 | more_colors=more_colors, 291 | center_distances=center_distances, 292 | depth_diff=depth_diff, normal_angles=normal_angles) 293 | for x, superpixels, file_name in zip(data.X, data.superpixels, 294 | data.file_names)) 295 | X = [(x[0], x[1], features) for x, features in zip(data.X, all_edge_features)] 296 | return DataBunch(X, data.Y, data.file_names, data.superpixels) 297 | 298 | 299 | def gt_in_sp(dataset, filename, superpixels): 300 | y = dataset.get_ground_truth(filename) 301 | votes = sparse.coo_matrix((np.ones(superpixels.size), 302 | (y.ravel(), superpixels.ravel()))) 303 | return np.argmax(votes.toarray(), axis=0) 304 | 305 | 306 | def eval_on_pixels(dataset, Y_true, Y_pred, print_results=False): 307 | n_classes = len(dataset.classes) - 1 # -1 for void 308 | tp, tn, fp, fn = [np.zeros(n_classes) for i in xrange(4)] 309 | correct, total = 0, 0 310 | for y_true, y_pred in zip(Y_true, Y_pred): 311 | mask = y_true != dataset.void_label 312 | y_true, y_pred = y_true[mask], y_pred[mask] 313 | correct += np.sum(y_true == y_pred) 314 | total += len(y_true) 315 | for k in range(n_classes): 316 | tp[k] += np.sum((y_true == k) * (y_pred == k)) 317 | tn[k] += np.sum((y_true != k) * (y_pred != k)) 318 | fp[k] += np.sum((y_true != k) * (y_pred == k)) 319 | fn[k] += np.sum((y_true == k) * (y_pred != k)) 320 | jaccard = tp / (fp + fn + tp) * 100 321 | hamming = tp / (tp + fn) * 100 322 | if print_results: 323 | np.set_printoptions(precision=2) 324 | print("Jaccard") 325 | print(jaccard) 326 | print("Hamming") 327 | print(hamming) 328 | print("Mean Jaccard: %.1f Mean Hamming: %.1f" 329 | % (np.mean(jaccard), np.mean(hamming))) 330 | print("Total : %.1f" % (correct / float(total) * 100)) 331 | 332 | return hamming, jaccard 333 | 334 | 335 | def eval_on_sp(dataset, data, Y_pred, print_results=False): 336 | Y_pred_pixels = [y_pred[sp] for sp, y_pred in zip(data.superpixels, 337 | Y_pred)] 338 | Y_true = [dataset.get_ground_truth(f) for f in data.file_names] 339 | return eval_on_pixels(dataset, Y_true, Y_pred_pixels, 340 | print_results=print_results) 341 | 342 | 343 | @memory.cache 344 | def add_global_descriptor(data): 345 | # adds to each superpixel a feature consisting of the average over the 346 | # image 347 | global_descs = map(lambda x: x.sum(axis=0) / x.shape[0], data.X) 348 | X_new = [np.hstack([x, np.repeat(d[np.newaxis, :], x.shape[0], axis=0)]) 349 | for d, x in zip(global_descs, data.X)] 350 | return DataBunch(X_new, data.Y, data.file_names, data.superpixels) 351 | 352 | 353 | def probabilities_on_sp(ds, probabilities, superpixels, add_covariance=False): 354 | # accumulate votes in superpixels 355 | # interleaved repeat 356 | n_classes = len(ds.classes) - 1 357 | class_indices = np.repeat(np.arange(n_classes)[np.newaxis, :], 358 | superpixels.size, axis=0).ravel() 359 | # non-interleaved repeat 360 | superpixel_indices = np.repeat(superpixels.ravel(), n_classes) 361 | sp_probs = sparse.coo_matrix((probabilities.ravel(), 362 | (superpixel_indices, class_indices))) 363 | sp_probs = sp_probs.toarray().astype(np.float) 364 | # renormalize (same as dividing by sp sizes) 365 | sp_probs = sp_probs / sp_probs.sum(axis=-1)[:, np.newaxis] 366 | if add_covariance: 367 | covars = [] 368 | for i in np.unique(superpixels): 369 | covars.append(np.cov(probabilities[superpixels == i].T).ravel()) 370 | return np.hstack([sp_probs, np.array(covars)]) 371 | return sp_probs 372 | 373 | -------------------------------------------------------------------------------- /visualize_edge_features.py: -------------------------------------------------------------------------------- 1 | import matplotlib 2 | matplotlib.use('WxAgg') 3 | matplotlib.interactive(False) 4 | 5 | import mayavi.mlab as mv 6 | import numpy as np 7 | import matplotlib.pyplot as plt 8 | 9 | from skimage.segmentation import mark_boundaries 10 | 11 | from datasets.nyu import NYUSegmentation 12 | from nyu.nyu_helpers import load_nyu 13 | 14 | #from utils import get_edge_depth_diff, add_edges 15 | from utils import get_sp_normals, add_edges, get_superpixel_centers 16 | 17 | #from IPython.core.debugger import Tracer 18 | #tracer = Tracer() 19 | 20 | 21 | def crazy_visual(): 22 | dataset = NYUSegmentation() 23 | # load training data 24 | data = load_nyu(n_sp=500) 25 | data = add_edges(data) 26 | 27 | for x, image_name, superpixels, y in zip(data.X, data.file_names, 28 | data.superpixels, data.Y): 29 | print(image_name) 30 | if int(image_name) != 11: 31 | continue 32 | image = dataset.get_image(image_name) 33 | plt.figure(figsize=(20, 20)) 34 | bounary_image = mark_boundaries(image, superpixels) 35 | plt.imshow(bounary_image) 36 | gridx, gridy = np.mgrid[:superpixels.shape[0], :superpixels.shape[1]] 37 | 38 | edges = x[1] 39 | points_normals = dataset.get_pointcloud_normals(image_name) 40 | centers2d = get_superpixel_centers(superpixels) 41 | centers3d = [np.bincount(superpixels.ravel(), weights=c.ravel()) 42 | for c in points_normals[:, :, :3].reshape(-1, 3).T] 43 | centers3d = (np.vstack(centers3d) / np.bincount(superpixels.ravel())).T 44 | sp_normals = get_sp_normals(points_normals[:, :, 3:], superpixels) 45 | offset = centers3d[edges[:, 0]] - centers3d[edges[:, 1]] 46 | offset = offset / np.sqrt(np.sum(offset ** 2, axis=1))[:, np.newaxis] 47 | #mean_normal = (sp_normals[edges[:, 0]] + sp_normals[edges[:, 1]]) / 2. 48 | mean_normal = sp_normals[edges[:, 0]] 49 | #edge_features = np.arccos(np.abs((offset * mean_normal).sum(axis=1))) * 2. / np.pi 50 | edge_features = 1 - np.abs((offset * mean_normal).sum(axis=1)) 51 | no_normals = (np.all(sp_normals[edges[:, 0]] == 0, axis=1) 52 | + np.all(sp_normals[edges[:, 1]] == 0, axis=1)) 53 | edge_features[no_normals] = 0 # nan normals 54 | 55 | if True: 56 | coords = points_normals[:, :, :3].reshape(-1, 3) 57 | perm = np.random.permutation(superpixels.max()+1) 58 | mv.points3d(coords[:,0], coords[:, 1], coords[:, 2], perm[superpixels.ravel()], mode='point') 59 | #mv.points3d(centers3d[:, 0], centers3d[:, 1], centers3d[:, 2], scale_factor=.04) 60 | mv.quiver3d(centers3d[:, 0], centers3d[:, 1], centers3d[:, 2], sp_normals[:, 0], sp_normals[:, 1], sp_normals[:, 2]) 61 | mv.show() 62 | from IPython.core.debugger import Tracer 63 | Tracer()() 64 | 65 | 66 | for i, edge in enumerate(edges): 67 | e0, e1 = edge 68 | #color = (dataset.colors[y[e0]] + dataset.colors[y[e1]]) / (2. * 255.) 69 | #f = edge_features[i] 70 | #if f < 0: 71 | #e0, e1 = e1, e0 72 | #f = -f 73 | 74 | #plt.arrow(centers[e0][0], centers[e0][1], 75 | #centers[e1][0] - centers[e0][0], centers[e1][1] - centers[e0][1], 76 | #width=f * 5 77 | #) 78 | color = "black" 79 | plt.plot([centers2d[e0][0], centers2d[e1][0]], 80 | [centers2d[e0][1], centers2d[e1][1]], 81 | c=color, 82 | linewidth=edge_features[i] * 5 83 | ) 84 | plt.scatter(centers2d[:, 0], centers2d[:, 1], s=100) 85 | plt.tight_layout() 86 | plt.xlim(0, superpixels.shape[1]) 87 | plt.ylim(superpixels.shape[0], 0) 88 | plt.axis("off") 89 | plt.savefig("figures/normal_relative/%s.png" % image_name, 90 | bbox_inches="tight") 91 | plt.close() 92 | 93 | if __name__ == "__main__": 94 | crazy_visual() 95 | -------------------------------------------------------------------------------- /visualize_new_gt.py: -------------------------------------------------------------------------------- 1 | 2 | import numpy as np 3 | import matplotlib.pyplot as plt 4 | 5 | from datasets.msrc import MSRCDataset, colors, classes 6 | 7 | from IPython.core.debugger import Tracer 8 | tracer = Tracer() 9 | 10 | 11 | def main(): 12 | msrc = MSRCDataset() 13 | images = msrc.get_split() 14 | for image_name in images: 15 | image = msrc.get_image(image_name) 16 | fig, axes = plt.subplots(1, 4, figsize=(12, 6)) 17 | axes[0].imshow(image) 18 | axes[1].set_title("ground truth") 19 | axes[1].imshow(image) 20 | gt = msrc.get_ground_truth(image_name) 21 | axes[1].imshow(colors[gt], alpha=.7) 22 | axes[2].set_title("new ground_truth") 23 | gt_new = msrc.get_ground_truth(image_name, ds="new") 24 | axes[2].imshow(image) 25 | axes[2].imshow(colors[gt_new], vmin=0, vmax=23, alpha=.7) 26 | present_y = np.unique(np.hstack([gt.ravel(), gt_new.ravel()])) 27 | axes[3].imshow(colors[present_y, :][:, np.newaxis, :], 28 | interpolation='nearest') 29 | for i, c in enumerate(present_y): 30 | axes[3].text(1, i, classes[c]) 31 | for ax in axes.ravel(): 32 | ax.set_xticks(()) 33 | ax.set_yticks(()) 34 | fig.savefig("new_gt/%s.png" % image_name, bbox_inches="tight") 35 | plt.close(fig) 36 | 37 | 38 | if __name__ == "__main__": 39 | main() 40 | --------------------------------------------------------------------------------