├── README.md ├── SGR_layer.py ├── adj_mat_coco.pickle ├── assets └── adj_mat.png ├── build_adj_mat.py └── embed_mat_cocov2_300.pickle /README.md: -------------------------------------------------------------------------------- 1 | # SGRLayer 2 | Pure Tensorflow implementation of the SGR layer as proposed in "Symbolic Graph Reasoning Meets Convolutions". This work makes several assumptions as to how the authors thought about certain implementation details. Therefore, in no way is this implementation guaranteed to produce correct or similar results as in the paper. 3 | 4 | We additionally add the fasttext embeddings and adjacency matrix used in our experiments, 5 | The image below shows the adjacency matrix we used. 6 | ![adj_mat](https://github.com/julianschoep/SGRLayer/blob/master/assets/adj_mat.png) 7 | 8 | Liang, Xiaodan, et al. "Symbolic graph reasoning meets convolutions." Advances in Neural Information Processing Systems. 2018. 9 | -------------------------------------------------------------------------------- /SGR_layer.py: -------------------------------------------------------------------------------- 1 | """DeepLab v3 models based on slim library.""" 2 | 3 | from __future__ import absolute_import 4 | from __future__ import division 5 | from __future__ import print_function 6 | 7 | import tensorflow as tf 8 | 9 | from tensorflow.contrib.slim.nets import resnet_v2 10 | from tensorflow.contrib import layers as layers_lib 11 | from tensorflow.contrib.framework.python.ops import arg_scope 12 | from tensorflow.contrib.layers.python.layers import layers 13 | 14 | from utils import preprocessing 15 | import numpy as np 16 | import horovod.tensorflow as hvd 17 | import os 18 | 19 | 20 | import memory_saving_gradients 21 | # monkey patch tf.gradients to point to our custom version, with automatic checkpoint selection 22 | tf.__dict__["gradients"] = memory_saving_gradients.gradients_memory 23 | 24 | # from SGR import SGRLayer 25 | 26 | _BATCH_NORM_DECAY = 0.9997 27 | _WEIGHT_DECAY = 5e-4 28 | 29 | M = 204 # no of symbolic entities in knowledge graph 30 | Dc = 256 # symbolic feature length 31 | Dl = 256 # input layer depth 32 | K = 300 # fasttext embedding vector size 33 | 34 | 35 | 36 | def histogram(name, scalar): 37 | # TF summary histogram wrapper to see how the various layers in the SGR progress. 38 | 39 | def clip_fn(t,min_val, max_val): 40 | return tf.clip_by_value(t, min_val, max_val) 41 | clip_dict = { 42 | 'input_features':(0,10), 43 | 'output_features':(-2,2), 44 | 'applied_mapping':(-50,50), 45 | 'compatibility':(-10,60), 46 | 'evolved_feat':(-10,10), 47 | 'encoded_concat_feat':(-150,150), 48 | 'visual_feat':(0,250) 49 | } 50 | 51 | if 'feat' in name: 52 | tf.summary.histogram(name+'_clip50',clip_fn(scalar,-50,50)) 53 | tf.summary.histogram(name+'_clip10',clip_fn(scalar,-10,10)) 54 | if 'softmax' in name: 55 | tf.summary.histogram(name+'_clip.001', clip_fn(scalar,0,0.001)) 56 | tf.summary.histogram(name+'_clip.01', clip_fn(scalar,0,0.01)) 57 | tf.summary.histogram(name+'_clip.4', clip_fn(scalar,0,0.4)) 58 | 59 | else:#name in zooms: 60 | tf.summary.histogram(name+'_clip1', clip_fn(scalar,-2,2)) 61 | tf.summary.histogram(name+'_clip1', clip_fn(scalar,-1,1)) 62 | tf.summary.histogram(name+'_clip5', clip_fn(scalar,-5,5)) 63 | 64 | return tf.summary.histogram(name, scalar) 65 | 66 | 67 | def row_normalize(mat): 68 | Q = tf.reduce_sum(mat, axis=-1) # take sum along rows # input mat 1,H,W # 33x33 69 | Q = tf.cast(Q, tf.float32) # conver tot float 70 | # take max with espilon to avoid inf/nan in rsqrt 71 | # _EPS = tf.constant(1e-7) 72 | # Q = tf.map_fn(lambda x: tf.maximum(_EPS, x), Q) 73 | # Inverse sqrt reciprocal (i.e. negative half power) 74 | sQ = tf.rsqrt(Q) 75 | sQ = tf.matrix_diag(sQ) # make diagonal mat 76 | # do symmetric normalization 77 | return tf.matmul(sQ, tf.matmul(mat, sQ, name='right_side_norm'), name='left_side_norm') 78 | 79 | 80 | def norm_adjacency(adjacency_matrix): 81 | # Adds identity connection to adjacency matrix, converts to tf constant 82 | # and normalize 83 | 84 | # Adjecency matrix can be of any shape. 85 | a = adjacency_matrix # MxM 86 | assert a.shape[0] == a.shape[1], 'adjacency matrix should be symmetirc' 87 | # row normalize the adjacency matrix 88 | a += np.identity(a.shape[0]) # add identity connections 89 | tf_adj = tf.constant(a, dtype=tf.float32) 90 | norm_adj = row_normalize(tf_adj) 91 | return norm_adj 92 | 93 | 94 | def SGRLayer(inputs, adj_matrix, embeddings, batch_norm_decay, is_training): 95 | histogram('SGR_input_features', inputs) 96 | to_inspect = {} # layer activations that we want to visualize later 97 | with tf.variable_scope('SGR', reuse=tf.AUTO_REUSE): 98 | 99 | # For the experiment without using fasttext embeddings. 100 | _fasttext_embeddings = tf.Variable(tf.random_normal( 101 | (M,K), 102 | mean=-0.0061, 103 | stddev=0.223, 104 | dtype=tf.float32 105 | ),dtype=tf.float32, 106 | trainable=False) 107 | 108 | fasttext_embeddings = tf.Variable(embeddings, dtype=tf.float32,trainable=False,name='fasttext_embeddings') 109 | norm_adj = norm_adjacency(adj_matrix) 110 | 111 | INPUT_SHAPE = tf.shape(inputs)# 112 | 113 | x = inputs 114 | 115 | ### LOCAL TO SEMANTIC MODULE ### 116 | with tf.variable_scope('voting'): 117 | tf.logging.info('Building voting module') 118 | 119 | with tf.variable_scope('calc_votes'): 120 | 121 | # will have shape [?,H,W,M] 122 | votes = layers_lib.conv2d( # 123 | inputs, M, (1, 1), stride=1, 124 | normalizer_fn=tf.nn.softmax, 125 | normalizer_params={'axis':-1, 'name':'voting_softmax_target'} , 126 | activation_fn=None, # NO RELU IS USED IN THE PAPER HERE 127 | scope='L2S_Wa') # uses Xavier initialization 128 | 129 | to_inspect['softmax_vote_activations'] = votes 130 | 131 | # shape [?,H,W,M] 132 | histogram('vote_softmax', votes) 133 | votes = tf.reshape( 134 | votes, [INPUT_SHAPE[0], INPUT_SHAPE[1] * INPUT_SHAPE[2], M]) # shape [?,H*W,M] 135 | votes = tf.transpose(votes, [0, 2, 1]) # shape [?,M, H*W] # every pixel Xi distributes 136 | # voting over the M nodes. np.sum(0,:,0]) must be 1 137 | 138 | with tf.variable_scope('assign_votes'): 139 | in_feat = layers_lib.conv2d( # transform each local feature into length Dc 140 | inputs, Dc, (1, 1), stride=1, 141 | biases_initializer=None, 142 | weights_initializer=tf.glorot_normal_initializer(), 143 | activation_fn=None, # NO RELU IS USED IN THE PAPER HERE 144 | scope='L2S_Wps') # shape [?,H,W,Dc] 145 | activation_summary(in_feat) 146 | # shape [H*W,Dc] 147 | in_feat = tf.reshape(in_feat, [ 148 | INPUT_SHAPE[0], INPUT_SHAPE[1] * INPUT_SHAPE[2], Dc]) # shape [?,H*W,Dc] 149 | in_feat = tf.matmul( 150 | votes, in_feat, name='applied_votes') # shape [?,M,Dc] 151 | visual_features = tf.nn.relu( 152 | in_feat, name='visual_feat') # shape [?,M,Dc] This relu is not mentioned in the text, but is shown 153 | # in figure 2 in the paper. 154 | histogram('visual_features', visual_features) 155 | to_inspect['visual_features'] = visual_features 156 | 157 | ### GRAPH REASONING MODULE ### 158 | with tf.variable_scope('reasoning'): 159 | tf.logging.info('Building reasoning module') 160 | 161 | # Fasttext embeddings should be externally supplied 162 | # Fasttext embedding has shape [M,K], tile em for the features each batch slice 163 | 164 | fasttext_embeddings = tf.expand_dims(fasttext_embeddings, 0) 165 | 166 | fasttext_embeddings = tf.tile( 167 | fasttext_embeddings, [INPUT_SHAPE[0], 1, 1]) # sh [b,M,K] 168 | 169 | concat_feat = tf.concat([visual_features, fasttext_embeddings], 170 | axis=-1, name='concat_embed') # shape [b,M,Dc+K] 171 | tf.summary.histogram('concat_features', concat_feat) 172 | to_inspect['embed_concat_feat'] = concat_feat 173 | activation_summary(concat_feat) 174 | 175 | 176 | # Encode into dc length vector (e.g. [M,(Dc+k)] to [M,(Dc)])) 177 | # transformed_feat = tf.layers.conv2d(inputs=concat_feat, 178 | # filters=Dc,kernel_size=(1),name='encode_concat_feats') # encode 179 | # back to shape [M,Dc] 180 | # This operation without RELU somehow forces the features to become very negative, which causes 181 | # the output of the SGR layer to be zero due to the RELU activation at the end. Lets try to put a bias 182 | # and relu activation here to force the output to be positive. 183 | # concat_feat has shape [b,M,Dc+K] 184 | # make shape [b,M,Dc] 185 | with tf.variable_scope('encode_concat'): 186 | concat_feat = tf.reshape(concat_feat,[-1,Dc+K]) # shape [b*M,Dc+K] 187 | Wg= tf.get_variable(name="Wg", dtype=tf.float32, 188 | shape=(Dc+K, Dc), 189 | initializer=tf.glorot_normal_initializer(), 190 | trainable=True) 191 | tf.summary.histogram('Wg_encode_concat', Wg) 192 | transformed_concat_feat = tf.matmul(concat_feat,Wg)# shape [b*M,Dc] @ square weight to force positive 193 | transformed_concat_feat = tf.reshape(transformed_concat_feat,[INPUT_SHAPE[0],M, Dc]) 194 | 195 | histogram('encoded_concat_feat', transformed_concat_feat) 196 | 197 | tf.logging.info('Creating second log module') 198 | to_inspect['embed_transformed_feat'] = transformed_concat_feat 199 | 200 | # norm_adj has shape MxM 201 | # evolved feat has shape [b,M,Dc] 202 | # Approach one: Mulitply with row normalized adjacency matrix, for every batch 203 | # Approach two: Stack batch features along columns 204 | # multiply norm_adj with this [Mxb*Dc] tensor. 205 | # The resulting tensor will be the 'evolved' batches still stacked along the cols. 206 | # Reshape back to stack batches back along the first dimension. 207 | with tf.variable_scope('graph_convolution'): 208 | tile_norm_adj = tf.tile(norm_adj[tf.newaxis,:,:],[INPUT_SHAPE[0],1,1]) # shape [b,M,M] @ [b,M,Dc] --> [b,M,Dc] # approach two 209 | evolved_feat = tf.matmul(tile_norm_adj,transformed_concat_feat,name='matmul_adj_feat') # [b,M,Dc] 210 | #evolved_feat = tf.map_fn(lambda vis_feat: tf.matmul(norm_adj, vis_feat),transformed_feat, name='matmul_adj_feat') # shape [b,M,Dc] # approach one 211 | histogram('evolved_feat',evolved_feat) 212 | 213 | to_inspect['embed_evolved_feat'] = evolved_feat 214 | 215 | histogram('evolved_feat_before_relu', evolved_feat) 216 | evolved_feat = tf.nn.relu( 217 | evolved_feat, name='evolved_feats') # shape [b,M,Dc] 218 | 219 | histogram('evolved_feat', evolved_feat) 220 | ## SEMANTIC TO LOCAL MODULE ### 221 | with tf.variable_scope('mapping'): 222 | tf.logging.info('Building mapping module') 223 | with tf.variable_scope('calc_mapping'): 224 | tf.logging.info('Setup calc mapping graph') 225 | 226 | # use alternative that uses nested map_fn functions. 227 | # VERIFIED Works. Has memory of 950mb, still doesnt enable OS8, but alternatives are difficult 228 | # to do without nested map_fns, which in turn have some gradient computation errors (Nonetype has no attribute 'op') 229 | # This seems like a good middle ground between memory usage and speed, and it works 230 | 231 | def _compute_compat_batches(inp,evolved): 232 | batch_data = (inp,evolved) 233 | def compute_compat_batch(batch_x, batch_feat): 234 | # batch_x has shape [H,W,Dl] 235 | # batch_feat has shape [M,Dc] 236 | sh = tf.shape(batch_x) 237 | batch_x = tf.reshape(batch_x,[sh[0]*sh[1],sh[2]]) # shape [H*W,Dl] 238 | batch_x = tf.tile(tf.expand_dims(batch_x,1),[1,M,1]) # has shape [H*W,M,Dl] 239 | 240 | batch_feat = tf.tile(tf.expand_dims(batch_feat,0),[sh[0]*sh[1],1,1]) # sh [H*W,M,Dc] 241 | 242 | compare_concat = tf.concat([batch_feat, batch_x], axis=-1, 243 | name='concat_inp') # (H*WxMx(Dc+Dl)) 244 | compare_concat.set_shape([None, None, Dc+Dc]) 245 | compat = layers_lib.conv2d( # Ws in paper 246 | compare_concat[tf.newaxis,:,:,:], 247 | 1, (1, 1), 248 | stride=1, 249 | biases_initializer=None, 250 | activation_fn=None, 251 | scope='Ws') # has shape (H*W,M,1) 252 | 253 | return tf.reshape(compat,[sh[0]*sh[1],M]) 254 | 255 | return tf.map_fn(lambda batch: compute_compat_batch(*batch), batch_data,dtype=tf.float32) 256 | 257 | compat = _compute_compat_batches(inputs, evolved_feat) # will have shape [b,H*W,M] 258 | tf.summary.scalar('compat/sparsity', tf.nn.zero_fraction(compat)) 259 | activation_summary(compat) 260 | histogram('compatibility',compat) 261 | to_inspect['compatibility'] = compat 262 | 263 | mapping = tf.nn.softmax( 264 | compat, axis=1, name='mapping_softmax') # (b,H*WxM) ! note axis, we want to take softmax w.r.t. other pixels, 265 | tf.summary.scalar('mapping/sparsity', tf.nn.zero_fraction(mapping)) 266 | 267 | to_inspect['mapping_target'] = mapping 268 | histogram('mapping_softmax', mapping) 269 | 270 | # NOT over other visual nodes. So a visual feature gets distributed to output nodes according to how compatible its value is 271 | # w.r.t. to other pixels, 272 | 273 | # They say they do this in the paper? Sec. 3.4. doing symmetric row normalization of possibly asymetric (or even 3d?) matrix 274 | # within the computational graph of tensorflow is difficult. 275 | # Not shown in figure 2, but implemented anyway. 276 | 277 | # Instead we use tf build int normalization function. 278 | # Faster than our own implementation and supports arbitrareliy sized tensors, which is useful for eval. 279 | with tf.variable_scope('row_normalize_mapping'): 280 | #mapping has shape [b,H*W,M] 281 | mapping_norm = tf.norm(mapping,ord=1,axis=-1) # has shape [b,H*W] holding sums along M dim 282 | mapping_norm = tf.tile(tf.expand_dims(mapping_norm,axis=-1),[1,1,M]) # copy sums along M dim for element wise divide shape [b,H*W,M] 283 | mapping = mapping / mapping_norm # l1 normalized mapping, where the M dim sums up to 1. 284 | #mapping = tf.nn.l2_normalize(mapping,axis=2) # does l2 normalization along the rows. 285 | histogram('mapping_softmax_row_norm', mapping) 286 | to_inspect['mapping_row_norm'] = mapping # shape [b,H,W,M] 287 | 288 | 289 | with tf.variable_scope('apply_mapping'): 290 | tf.logging.info('Setup apply mapping graph') 291 | 292 | Wsp = tf.get_variable(name="Wsp", dtype=tf.float32, 293 | shape=(Dc, Dl), 294 | initializer=tf.glorot_normal_initializer(), 295 | trainable=True) 296 | 297 | tf.summary.histogram('W_transf_evolved', Wsp) 298 | # evolved_feat # shape [b,M,Dc] 299 | 300 | # merge batch with M again for efficient mult with Wsp 301 | evolved_feat = tf.reshape(evolved_feat,[-1,Dc]) # shape [b*M,Dc] 302 | 303 | transf_evolved = tf.matmul(evolved_feat,Wsp) #[b*M,Dl] 304 | transf_evolved = tf.reshape(transf_evolved,[INPUT_SHAPE[0],M,Dl]) # [b,M,Dl] 305 | 306 | #transf_evolved = tf.map_fn(lambda batch_evolved_feat: tf.matmul(batch_evolved_feat, Wsp),evolved_feat) # shape [b,M,Dl] 307 | activation_summary(transf_evolved) 308 | to_inspect['transf_evolved'] = transf_evolved 309 | # merge back 310 | histogram('transf_evolved', transf_evolved) 311 | 312 | # Not mentioned in paper, but before applying mapping, reshape 313 | # row-normalized Ag (e.g. mapping) back to H*W x M so we can matmul 314 | # it! 315 | mapping = tf.reshape( 316 | mapping, [INPUT_SHAPE[0], INPUT_SHAPE[1] * INPUT_SHAPE[2], M]) # shape b,H*WxM 317 | 318 | # Distribute evolved features to output feature map according to 319 | # softmax mapping 320 | applied_mapping = tf.matmul( 321 | mapping, transf_evolved, name='applied_mapping') # bx(H*W)xDl 322 | to_inspect['applied_mapping_before_relu'] = applied_mapping 323 | histogram('applied_mapping_before_relu', applied_mapping) 324 | applied_mapping = tf.nn.relu( 325 | applied_mapping, name='mapping_relu') # bx(H*W)xDl 326 | applied_mapping = tf.reshape(applied_mapping, [INPUT_SHAPE[0], INPUT_SHAPE[1], INPUT_SHAPE[2], Dl], 327 | name='mapped_feat_target') # b,HxWxDl # Look at feature maps! (there's a lot so maybe difficult) 328 | 329 | histogram('SGR_output_features', applied_mapping) 330 | to_inspect['applied_mapping'] = applied_mapping 331 | to_inspect['SGR_input'] = inputs 332 | tf.summary.scalar('applied_mapping/sparsity', tf.nn.zero_fraction(applied_mapping)) 333 | 334 | return (inputs+applied_mapping),to_inspect 335 | 336 | -------------------------------------------------------------------------------- /adj_mat_coco.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/julianschoep/SGRLayer/d2c8bae2f19e541c371b92046cd7a63cd11c6c7f/adj_mat_coco.pickle -------------------------------------------------------------------------------- /assets/adj_mat.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/julianschoep/SGRLayer/d2c8bae2f19e541c371b92046cd7a63cd11c6c7f/assets/adj_mat.png -------------------------------------------------------------------------------- /build_adj_mat.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | import time 4 | import matplotlib.pyplot as plt 5 | 6 | """ 7 | This is basically a dump of the jupyter notebook I used to generate the adjacency matrix used in this layer. 8 | This therefore contains the used hierarchy. 9 | """ 10 | 11 | 12 | things_dict = { 'things': 'things', 13 | 'indoor-super-things': 'things', 14 | 'outdoor-super-things': 'things', 15 | 'person-things': 'outdoor-super-things', 16 | 'vehicle-things': 'outdoor-super-things', 17 | 'outdoor-things': 'outdoor-super-things', 18 | 'animal-things': 'outdoor-super-things', 19 | 'accessory-things': 'outdoor-super-things', 20 | 'sports-things': 'outdoor-super-things', 21 | 'kitchen-things': 'indoor-super-things', 22 | 'food-things': 'indoor-super-things', 23 | 'furniture-things': 'indoor-super-things', 24 | 'electronic-things': 'indoor-super-things', 25 | 'appliance-things': 'indoor-super-things', 26 | 'indoor-things': 'indoor-super-things', 27 | 'person': 'person-things', 28 | 'bicycle': 'vehicle-things', 29 | 'car': 'vehicle-things', 30 | 'motorcycle': 'vehicle-things', 31 | 'airplane': 'vehicle-things', 32 | 'bus': 'vehicle-things', 33 | 'train': 'vehicle-things', 34 | 'truck': 'vehicle-things', 35 | 'boat': 'vehicle-things', 36 | 'traffic light': 'outdoor-things', 37 | 'fire hydrant': 'outdoor-things', 38 | 'street sign': 'outdoor-things', 39 | 'stop sign': 'outdoor-things', 40 | 'parking meter': 'outdoor-things', 41 | 'bench': 'outdoor-things', 42 | 'bird': 'animal-things', 43 | 'cat': 'animal-things', 44 | 'dog': 'animal-things', 45 | 'horse': 'animal-things', 46 | 'sheep': 'animal-things', 47 | 'cow': 'animal-things', 48 | 'elephant': 'animal-things', 49 | 'bear': 'animal-things', 50 | 'zebra': 'animal-things', 51 | 'giraffe': 'animal-things', 52 | 'hat': 'accessory-things', 53 | 'backpack': 'accessory-things', 54 | 'umbrella': 'accessory-things', 55 | 'shoe': 'accessory-things', 56 | 'eye glasses': 'accessory-things', 57 | 'handbag': 'accessory-things', 58 | 'tie': 'accessory-things', 59 | 'suitcase': 'accessory-things', 60 | 'frisbee': 'sports-things', 61 | 'skis': 'sports-things', 62 | 'snowboard': 'sports-things', 63 | 'sports ball': 'sports-things', 64 | 'kite': 'sports-things', 65 | 'baseball bat': 'sports-things', 66 | 'baseball glove': 'sports-things', 67 | 'skateboard': 'sports-things', 68 | 'surfboard': 'sports-things', 69 | 'tennis racket': 'sports-things', 70 | 'bottle': 'kitchen-things', 71 | 'plate': 'kitchen-things', 72 | 'wine glass': 'kitchen-things', 73 | 'cup': 'kitchen-things', 74 | 'fork': 'kitchen-things', 75 | 'knife': 'kitchen-things', 76 | 'spoon': 'kitchen-things', 77 | 'bowl': 'kitchen-things', 78 | 'banana': 'food-things', 79 | 'apple': 'food-things', 80 | 'sandwich': 'food-things', 81 | 'orange': 'food-things', 82 | 'broccoli': 'food-things', 83 | 'carrot': 'food-things', 84 | 'hot dog': 'food-things', 85 | 'pizza': 'food-things', 86 | 'donut': 'food-things', 87 | 'cake': 'food-things', 88 | 'chair': 'furniture-things', 89 | 'couch': 'furniture-things', 90 | 'potted plant': 'furniture-things', 91 | 'bed': 'furniture-things', 92 | 'mirror': 'furniture-things', 93 | 'dining table': 'furniture-things', 94 | 'window': 'furniture-things', 95 | 'desk': 'furniture-things', 96 | 'toilet': 'furniture-things', 97 | 'door': 'furniture-things', 98 | 'tv': 'electronic-things', 99 | 'laptop': 'electronic-things', 100 | 'mouse': 'electronic-things', 101 | 'remote': 'electronic-things', 102 | 'keyboard': 'electronic-things', 103 | 'cell phone': 'electronic-things', 104 | 'microwave': 'appliance-things', 105 | 'oven': 'appliance-things', 106 | 'toaster': 'appliance-things', 107 | 'sink': 'appliance-things', 108 | 'refrigerator': 'appliance-things', 109 | 'blender': 'appliance-things', 110 | 'book': 'indoor-things', 111 | 'clock': 'indoor-things', 112 | 'vase': 'indoor-things', 113 | 'scissors': 'indoor-things', 114 | 'teddy bear': 'indoor-things', 115 | 'hair drier': 'indoor-things', 116 | 'toothbrush': 'indoor-things', 117 | 'hair brush': 'indoor-things' 118 | } 119 | stuff_dict = { 120 | 'stuff': 'stuff', 121 | 'indoor-super-stuff': 'stuff', 122 | 'outdoor-super-stuff': 'stuff', 123 | 'rawmaterial-stuff': 'indoor-super-stuff', 124 | 'wall-stuff': 'indoor-super-stuff', 125 | 'ceiling-stuff': 'indoor-super-stuff', 126 | 'floor-stuff': 'indoor-super-stuff', 127 | 'window-stuff': 'indoor-super-stuff', 128 | 'furniture-stuff': 'indoor-super-stuff', 129 | 'textile-stuff': 'indoor-super-stuff', 130 | 'food-stuff': 'indoor-super-stuff', 131 | 'building-stuff': 'outdoor-super-stuff', 132 | 'structural-stuff': 'outdoor-super-stuff', 133 | 'plant-stuff': 'outdoor-super-stuff', 134 | 'sky-stuff': 'outdoor-super-stuff', 135 | 'solid-stuff': 'outdoor-super-stuff', 136 | 'ground-stuff': 'outdoor-super-stuff', 137 | 'water-stuff': 'outdoor-super-stuff', 138 | 'cardboard': 'rawmaterial-stuff', 139 | 'paper': 'rawmaterial-stuff', 140 | 'plastic': 'rawmaterial-stuff', 141 | 'metal': 'rawmaterial-stuff', 142 | 'wall-tile': 'wall-stuff', 143 | 'wall-panel': 'wall-stuff', 144 | 'wall-wood': 'wall-stuff', 145 | 'wall-brick': 'wall-stuff', 146 | 'wall-stone': 'wall-stuff', 147 | 'wall-concrete': 'wall-stuff', 148 | 'wall-other': 'wall-stuff', 149 | 'ceiling-tile': 'ceiling-stuff', 150 | 'ceiling-other': 'ceiling-stuff', 151 | 'carpet': 'floor-stuff', 152 | 'floor-tile': 'floor-stuff', 153 | 'floor-wood': 'floor-stuff', 154 | 'floor-marble': 'floor-stuff', 155 | 'floor-stone': 'floor-stuff', 156 | 'floor-other': 'floor-stuff', 157 | 'window-blind': 'window-stuff', 158 | 'window-other': 'window-stuff', 159 | 'door-stuff': 'furniture-stuff', 160 | 'desk-stuff': 'furniture-stuff', 161 | 'table': 'furniture-stuff', 162 | 'shelf': 'furniture-stuff', 163 | 'cabinet': 'furniture-stuff', 164 | 'cupboard': 'furniture-stuff', 165 | 'mirror-stuff': 'furniture-stuff', 166 | 'counter': 'furniture-stuff', 167 | 'light': 'furniture-stuff', 168 | 'stairs': 'furniture-stuff', 169 | 'furniture-other': 'furniture-stuff', 170 | 'rug': 'textile-stuff', 171 | 'mat': 'textile-stuff', 172 | 'towel': 'textile-stuff', 173 | 'napkin': 'textile-stuff', 174 | 'clothes': 'textile-stuff', 175 | 'cloth': 'textile-stuff', 176 | 'curtain': 'textile-stuff', 177 | 'blanket': 'textile-stuff', 178 | 'pillow': 'textile-stuff', 179 | 'banner': 'textile-stuff', 180 | 'textile-other': 'textile-stuff', 181 | 'fruit': 'food-stuff', 182 | 'salad': 'food-stuff', 183 | 'vegetable': 'food-stuff', 184 | 'food-other': 'food-stuff', 185 | 'house': 'building-stuff', 186 | 'skyscraper': 'building-stuff', 187 | 'bridge': 'building-stuff', 188 | 'tent': 'building-stuff', 189 | 'roof': 'building-stuff', 190 | 'building-other': 'building-stuff', 191 | 'fence': 'structural-stuff', 192 | 'cage': 'structural-stuff', 193 | 'net': 'structural-stuff', 194 | 'railing': 'structural-stuff', 195 | 'structural-other': 'structural-stuff', 196 | 'grass': 'plant-stuff', 197 | 'tree': 'plant-stuff', 198 | 'bush': 'plant-stuff', 199 | 'leaves': 'plant-stuff', 200 | 'flower': 'plant-stuff', 201 | 'branch': 'plant-stuff', 202 | 'moss': 'plant-stuff', 203 | 'straw': 'plant-stuff', 204 | 'plant-other': 'plant-stuff', 205 | 'clouds': 'sky-stuff', 206 | 'sky-other': 'sky-stuff', 207 | 'wood': 'solid-stuff', 208 | 'rock': 'solid-stuff', 209 | 'stone': 'solid-stuff', 210 | 'mountain': 'solid-stuff', 211 | 'hill': 'solid-stuff', 212 | 'solid-other': 'solid-stuff', 213 | 'sand': 'ground-stuff', 214 | 'snow': 'ground-stuff', 215 | 'dirt': 'ground-stuff', 216 | 'mud': 'ground-stuff', 217 | 'gravel': 'ground-stuff', 218 | 'road': 'ground-stuff', 219 | 'pavement': 'ground-stuff', 220 | 'railroad': 'ground-stuff', 221 | 'platform': 'ground-stuff', 222 | 'playingfield': 'ground-stuff', 223 | 'ground-other': 'ground-stuff', 224 | 'fog': 'water-stuff', 225 | 'river': 'water-stuff', 226 | 'sea': 'water-stuff', 227 | 'waterdrops': 'water-stuff', 228 | 'water-other': 'water-stuff', 229 | } 230 | 231 | all_dict = { 232 | 'stuff': 'stuff', 233 | 'indoor': 'stuff', 234 | 'outdoor': 'stuff', 235 | 'rawmaterial': 'indoor', 236 | 'wall': 'indoor', 237 | 'ceiling': 'indoor', 238 | 'floor': 'indoor', 239 | 'window': 'indoor', 240 | 'furniture': 'indoor', 241 | 'textile': 'indoor', 242 | 'food': 'indoor', 243 | 'building': 'outdoor', 244 | 'structural': 'outdoor', 245 | 'plant': 'outdoor', 246 | 'sky': 'outdoor', 247 | 'solid': 'outdoor', 248 | 'ground': 'outdoor', 249 | 'water': 'outdoor', 250 | 'cardboard': 'rawmaterial', 251 | 'paper': 'rawmaterial', 252 | 'plastic': 'rawmaterial', 253 | 'metal': 'rawmaterial', 254 | 'wall-tile': 'wall', 255 | 'wall-panel': 'wall', 256 | 'wall-wood': 'wall', 257 | 'wall-brick': 'wall', 258 | 'wall-stone': 'wall', 259 | 'wall-concrete': 'wall', 260 | 'wall-other': 'wall', 261 | 'ceiling-tile': 'ceiling', 262 | 'ceiling-other': 'ceiling', 263 | 'carpet': 'floor', 264 | 'floor-tile': 'floor', 265 | 'floor-wood': 'floor', 266 | 'floor-marble': 'floor', 267 | 'floor-stone': 'floor', 268 | 'floor-other': 'floor', 269 | 'window-blind': 'window', 270 | 'window-other': 'window', 271 | 'door': 'furniture', 272 | 'desk': 'furniture', 273 | 'table': 'furniture', 274 | 'shelf': 'furniture', 275 | 'cabinet': 'furniture', 276 | 'cupboard': 'furniture', 277 | 'mirror': 'furniture', 278 | 'counter': 'furniture', 279 | 'light': 'furniture', 280 | 'stairs': 'furniture', 281 | 'furniture-other': 'furniture', 282 | 'rug': 'textile', 283 | 'mat': 'textile', 284 | 'towel': 'textile', 285 | 'napkin': 'textile', 286 | 'clothes': 'textile', 287 | 'cloth': 'textile', 288 | 'curtain': 'textile', 289 | 'blanket': 'textile', 290 | 'pillow': 'textile', 291 | 'banner': 'textile', 292 | 'textile-other': 'textile', 293 | 'fruit': 'food', 294 | 'salad': 'food', 295 | 'vegetable': 'food', 296 | 'food-other': 'food', 297 | 'house': 'building', 298 | 'skyscraper': 'building', 299 | 'bridge': 'building', 300 | 'tent': 'building', 301 | 'roof': 'building', 302 | 'building-other': 'building', 303 | 'fence': 'structural', 304 | 'cage': 'structural', 305 | 'net': 'structural', 306 | 'railing': 'structural', 307 | 'structural-other': 'structural', 308 | 'grass': 'plant', 309 | 'tree': 'plant', 310 | 'bush': 'plant', 311 | 'leaves': 'plant', 312 | 'flower': 'plant', 313 | 'branch': 'plant', 314 | 'moss': 'plant', 315 | 'straw': 'plant', 316 | 'plant-other': 'plant', 317 | 'clouds': 'sky', 318 | 'sky-other': 'sky', 319 | 'wood': 'solid', 320 | 'rock': 'solid', 321 | 'stone': 'solid', 322 | 'mountain': 'solid', 323 | 'hill': 'solid', 324 | 'solid-other': 'solid', 325 | 'sand': 'ground', 326 | 'snow': 'ground', 327 | 'dirt': 'ground', 328 | 'mud': 'ground', 329 | 'gravel': 'ground', 330 | 'road': 'ground', 331 | 'pavement': 'ground', 332 | 'railroad': 'ground', 333 | 'platform': 'ground', 334 | 'playingfield': 'ground', 335 | 'ground-other': 'ground', 336 | 'fog': 'water', 337 | 'river': 'water', 338 | 'sea': 'water', 339 | 'waterdrops': 'water', 340 | 'water-other': 'water', 341 | 'things': 'things', 342 | 'indoor': 'things', 343 | 'outdoor': 'things', 344 | 'person': 'outdoor', 345 | 'vehicle': 'outdoor', 346 | 'outdoor': 'outdoor', 347 | 'animal': 'outdoor', 348 | 'accessory': 'outdoor', 349 | 'sports': 'outdoor', 350 | 'kitchen': 'indoor', 351 | 'food': 'indoor', 352 | 'furniture': 'indoor', 353 | 'electronic': 'indoor', 354 | 'appliance': 'indoor', 355 | 'indoor': 'indoor', 356 | 'person': 'person', 357 | 'bicycle': 'vehicle', 358 | 'car': 'vehicle', 359 | 'motorcycle': 'vehicle', 360 | 'airplane': 'vehicle', 361 | 'bus': 'vehicle', 362 | 'train': 'vehicle', 363 | 'truck': 'vehicle', 364 | 'boat': 'vehicle', 365 | 'traffic light': 'outdoor', 366 | 'fire hydrant': 'outdoor', 367 | 'street sign': 'outdoor', 368 | 'stop sign': 'outdoor', 369 | 'parking meter': 'outdoor', 370 | 'bench': 'outdoor', 371 | 'bird': 'animal', 372 | 'cat': 'animal', 373 | 'dog': 'animal', 374 | 'horse': 'animal', 375 | 'sheep': 'animal', 376 | 'cow': 'animal', 377 | 'elephant': 'animal', 378 | 'bear': 'animal', 379 | 'zebra': 'animal', 380 | 'giraffe': 'animal', 381 | 'hat': 'accessory', 382 | 'backpack': 'accessory', 383 | 'umbrella': 'accessory', 384 | 'shoe': 'accessory', 385 | 'eye glasses': 'accessory', 386 | 'handbag': 'accessory', 387 | 'tie': 'accessory', 388 | 'suitcase': 'accessory', 389 | 'frisbee': 'sports', 390 | 'skis': 'sports', 391 | 'snowboard': 'sports', 392 | 'sports ball': 'sports', 393 | 'kite': 'sports', 394 | 'baseball bat': 'sports', 395 | 'baseball glove': 'sports', 396 | 'skateboard': 'sports', 397 | 'surfboard': 'sports', 398 | 'tennis racket': 'sports', 399 | 'bottle': 'kitchen', 400 | 'plate': 'kitchen', 401 | 'wine glass': 'kitchen', 402 | 'cup': 'kitchen', 403 | 'fork': 'kitchen', 404 | 'knife': 'kitchen', 405 | 'spoon': 'kitchen', 406 | 'bowl': 'kitchen', 407 | 'banana': 'food', 408 | 'apple': 'food', 409 | 'sandwich': 'food', 410 | 'orange': 'food', 411 | 'broccoli': 'food', 412 | 'carrot': 'food', 413 | 'hot dog': 'food', 414 | 'pizza': 'food', 415 | 'donut': 'food', 416 | 'cake': 'food', 417 | 'chair': 'furniture', 418 | 'couch': 'furniture', 419 | 'potted plant': 'furniture', 420 | 'bed': 'furniture', 421 | 'mirror': 'furniture', 422 | 'dining table': 'furniture', 423 | 'window': 'furniture', 424 | 'desk': 'furniture', 425 | 'toilet': 'furniture', 426 | 'door': 'furniture', 427 | 'tv': 'electronic', 428 | 'laptop': 'electronic', 429 | 'mouse': 'electronic', 430 | 'remote': 'electronic', 431 | 'keyboard': 'electronic', 432 | 'cell phone': 'electronic', 433 | 'microwave': 'appliance', 434 | 'oven': 'appliance', 435 | 'toaster': 'appliance', 436 | 'sink': 'appliance', 437 | 'refrigerator': 'appliance', 438 | 'blender': 'appliance', 439 | 'book': 'indoor', 440 | 'clock': 'indoor', 441 | 'vase': 'indoor', 442 | 'scissors': 'indoor', 443 | 'teddy bear': 'indoor', 444 | 'hair drier': 'indoor', 445 | 'toothbrush': 'indoor', 446 | 'hair brush': 'indoor' 447 | 448 | } 449 | 450 | # Dictionary has structure {leaf : parent} 451 | # Also includes a self connection stuff-> stuff at the top parent, 452 | # so the leaves contain ALL the entities 453 | 454 | def make_adjacency_matrix(connection_dict, directed=True): 455 | # Connection is from row TO collumn. 456 | all_words = np.unique(list(connection_dict.keys())) # Assumption! See text above 457 | M = len(all_words) 458 | 459 | 460 | word2idx = {word:i for (i,word) in enumerate(all_words)} 461 | idx2word = {i:word for (i,word) in enumerate(all_words)} 462 | adj_m = np.zeros((M,M)) 463 | for leaf in connection_dict.keys(): 464 | parent = connection_dict[leaf] 465 | adj_m[word2idx[parent],word2idx[leaf]] = 1 # Adds connection from parent to child 466 | if not directed: 467 | adj_m[word2idx[leaf],word2idx[parent]] = 1 # Adds connection back from child to parent 468 | return adj_m, word2idx, idx2word, all_words 469 | 470 | # Adjacency matrix A 471 | # word2idx, idx2word maps words to integers, ordered as 'all_words' 472 | A, word2idx,idx2word,all_words = make_adjacency_matrix(all_dict) 473 | plt.figure(figsize=(20,20)) 474 | plt.xticks(np.arange(len(all_words[:50])), all_words, rotation='vertical') 475 | plt.yticks(np.arange(len(all_words[:50])), all_words) 476 | plt.grid() 477 | plt.imshow(A[:50,:50]) 478 | plt.show() 479 | -------------------------------------------------------------------------------- /embed_mat_cocov2_300.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/julianschoep/SGRLayer/d2c8bae2f19e541c371b92046cd7a63cd11c6c7f/embed_mat_cocov2_300.pickle --------------------------------------------------------------------------------