├── .codeclimate.yml ├── .coveragerc ├── .gitignore ├── .mdlrc ├── .travis.yml ├── LICENSE ├── README.md ├── cifar_conv2d.py ├── cifar_graph.py ├── lib ├── __init__.py ├── datasets │ ├── __init__.py │ ├── augment.py │ ├── augment_test.py │ ├── cifar_10.py │ ├── cifar_10_test.py │ ├── dataset.py │ ├── dataset_test.py │ ├── download.py │ ├── download_test.py │ ├── mnist.py │ ├── mnist_test.py │ ├── pascal_voc.py │ ├── pascal_voc_test.py │ ├── queue.py │ └── queue_test.py ├── graph │ ├── __init__.py │ ├── adjacency.py │ ├── adjacency_test.py │ ├── clustering.py │ ├── clustering_test.py │ ├── coarsening.py │ ├── coarsening_test.py │ ├── distortion.py │ ├── distortion_test.py │ ├── grid.py │ ├── grid_test.py │ ├── spatial.py │ └── spatial_test.py ├── layer │ ├── __init__.py │ ├── average_pool.py │ ├── average_pool_test.py │ ├── chebyshev_gcnn.py │ ├── chebyshev_gcnn_test.py │ ├── conv2d.py │ ├── conv2d_test.py │ ├── embedded_gcnn.py │ ├── embedded_gcnn_test.py │ ├── fc.py │ ├── fc_test.py │ ├── fire.py │ ├── fire_test.py │ ├── gcnn.py │ ├── gcnn_test.py │ ├── image_augment.py │ ├── image_augment_test.py │ ├── inits.py │ ├── inits_test.py │ ├── layer.py │ ├── layer_test.py │ ├── max_pool.py │ ├── max_pool_test.py │ ├── spatial.py │ ├── spatial_test.py │ ├── var_layer.py │ └── var_layer_test.py ├── model │ ├── __init__.py │ ├── metrics.py │ ├── metrics_test.py │ ├── model.py │ ├── placeholder.py │ └── train.py ├── pipeline │ ├── __init__.py │ ├── augment.py │ ├── augment_test.py │ ├── dataset.py │ ├── dataset_test.py │ ├── file_queue.py │ ├── file_queue_test.py │ └── preprocess.py ├── segmentation │ ├── __init__.py │ ├── adjacency.py │ ├── adjacency_test.py │ ├── algorithm.py │ ├── algorithm_test.py │ ├── feature_extraction.py │ ├── feature_extraction_test.py │ ├── form_feature_extraction.py │ ├── form_feature_extraction_test.py │ ├── form_feature_selection.py │ └── form_feature_selection_test.py └── tf │ ├── __init__.py │ ├── bspline.py │ ├── bspline_test.py │ ├── convert.py │ ├── convert_test.py │ ├── laplacian.py │ ├── laplacian_test.py │ ├── math.py │ └── math_test.py ├── mnist_graph.py ├── mnist_spatial.py ├── notebooks ├── form_feature_extraction_speed_test.ipynb ├── form_feature_selection.ipynb ├── mnist_quickshift_node_removal.ipynb ├── pipeline_speed_test.ipynb ├── spatial_pipeline_speed_test.ipynb ├── spectrum.ipynb ├── speed_comparison.ipynb └── superpixel.ipynb ├── pascal_conv2d.py ├── pascal_graph.py ├── requirements.txt ├── requirements_test.txt └── test_data └── VOCtrainval_11-May-2012.tar /.codeclimate.yml: -------------------------------------------------------------------------------- 1 | engines: 2 | duplication: 3 | enabled: true 4 | exclude_paths: 5 | - "**_test.py" 6 | - "**pascal_voc.py" 7 | - "pascal_conv2d.py" 8 | - "cifar_conv2d.py" 9 | - "mnist_spatial.py" 10 | config: 11 | languages: 12 | python: 13 | mass_threshold: 120 14 | radon: 15 | enabled: true 16 | config: 17 | threshold: "C" 18 | exclude_paths: 19 | - "**train.py" 20 | fixme: 21 | enabled: true 22 | exclude_paths: 23 | - "notebooks/" 24 | pep8: 25 | enabled: true 26 | markdownlint: 27 | enabled: true 28 | shellcheck: 29 | enabled: true 30 | ratings: 31 | paths: 32 | - "**.py" 33 | - "**.sh" 34 | - "**.md" 35 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [report] 2 | exclude_lines = 3 | pragma: no cover 4 | raise NotImplementedError 5 | raise AssertionError 6 | raise ValueError 7 | except ImportError 8 | 9 | [run] 10 | omit = 11 | # These files aren't really testable :( 12 | lib/model/model.py 13 | lib/model/train.py 14 | lib/model/placeholder.py 15 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | tags 2 | .DS_Store 3 | __pycache__/ 4 | *.py[cod] 5 | .ipynb_checkpoints/ 6 | .coverage 7 | data/ 8 | test_data/VOCDevKit 9 | test_data/tiny-imagenet-200 10 | -------------------------------------------------------------------------------- /.mdlrc: -------------------------------------------------------------------------------- 1 | rules "~MD013" 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | sudo: required 3 | dist: trusty 4 | cache: data 5 | matrix: 6 | include: 7 | - python: 2.7 8 | - python: 3.4 9 | - python: 3.5 10 | - python: 3.6 11 | install: 12 | - pip install -r requirements_test.txt 13 | - pip install codecov 14 | script: 15 | - pep8 . 16 | - flake8 . 17 | - nosetests --with-coverage --nologcapture 18 | after_success: 19 | - codecov 20 | notifications: 21 | email: false 22 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2017 Matthias Fey 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Embedded Graph Convolutional Neural Networks 2 | 3 | [![Build Status][build-image]][build-url] 4 | [![Code Coverage][coverage-image]][coverage-url] 5 | [![Requirements Status][requirements-image]][requirements-url] 6 | [![Code Climate][code-climate-image]][code-climate-url] 7 | [![Code Climate Issues][code-climate-issues-image]][code-climate-issues-url] 8 | 9 | [build-image]: https://travis-ci.org/rusty1s/embedded_gcnn.svg?branch=master 10 | [build-url]: https://travis-ci.org/rusty1s/embedded_gcnn 11 | [coverage-image]: https://codecov.io/gh/rusty1s/embedded_gcnn/branch/master/graph/badge.svg 12 | [coverage-url]: https://codecov.io/github/rusty1s/embedded_gcnn?branch=master 13 | [requirements-image]: https://requires.io/github/rusty1s/embedded_gcnn/requirements.svg?branch=master 14 | [requirements-url]: https://requires.io/github/rusty1s/embedded_gcnn/requirements/?branch=master 15 | [code-climate-image]: https://codeclimate.com/github/rusty1s/embedded_gcnn/badges/gpa.svg 16 | [code-climate-url]: https://codeclimate.com/github/rusty1s/embedded_gcnn 17 | [code-climate-issues-image]: https://codeclimate.com/github/rusty1s/embedded_gcnn/badges/issue_count.svg 18 | [code-climate-issues-url]: https://codeclimate.com/github/rusty1s/embedded_gcnn/issues 19 | 20 | ![Neural Network Approach](https://user-images.githubusercontent.com/6945922/28239620-a72734d0-6970-11e7-9253-8a1e027efd78.png) 21 | 22 | This is a TensorFlow implementation of my mastersthesis on [Graph-based Image 23 | Classification](https://github.com/rusty1s/deep-learning-on-graphs/tree/master/masterthesis) 24 | *(german)*. 25 | 26 | **Embedded graph convolutional neural networks (EGCNN)** aim to make significant improvements to learning on graphs where nodes are positioned on a twodimensional euclidean plane and thus possess an orientation (like up, down, right and left). 27 | As proof, we implemented an image classification on embedded graphs by first segmenting the image into superpixels with the use of [SLIC](https://infoscience.epfl.ch/record/177415/files/Superpixel_PAMI2011-2.pdf) or [Quickshift](http://vision.cs.ucla.edu/papers/vedaldiS08quick.pdf), converting this representation into a graph and inputting these to the neural network. 28 | 29 | ![SlIC and Quickshift Segmentation](https://user-images.githubusercontent.com/6945922/27761633-61569a56-5e60-11e7-96d6-5a0507d26cf8.jpg) 30 | 31 | Graphs are trained on three different datasets and are automatically downloaded by running the corresponding train scripts: 32 | 33 | * [MNIST](http://yann.lecun.com/exdb/mnist/) (run `python mnist_graph.py` and `python mnist_spatial.py`) 34 | * [Cifar-10](https://www.cs.toronto.edu/~kriz/cifar.html) (run `python cifar_graph.py` and `python cifar_conv2d.py`) 35 | * [PascalVOC](http://host.robots.ox.ac.uk/pascal/VOC/) (run `python pascal_graph.py` and `python pascal_conv2d.py`) 36 | 37 | This repository also includes layer implementations of alternative approaches such as [SGCNN](https://arxiv.org/abs/1312.6203) and [GCN](https://arxiv.org/abs/1609.02907) for graphs and the Fire module of [SqueezeNet](https://arxiv.org/abs/1602.07360) for images to validate the results. 38 | 39 | ## Results 40 | 41 | | Dataset | SLIC | Quickshift | 42 | | ---------- | ------:| ----------:| 43 | | MNIST | 97.405 | 98.025 | 44 | | Cifar-10 | 74.218 | 75.230 | 45 | | Pascal VOC | 54.473 | 54.516 | 46 | 47 | ## Requirements 48 | 49 | To install the required python packages, run: 50 | 51 | ```bash 52 | pip install -r requirements.txt 53 | ``` 54 | 55 | ## Running tests 56 | 57 | Install the test requirements 58 | 59 | ```bash 60 | pip install -r requirements_test.txt 61 | ``` 62 | 63 | and run the test suite: 64 | 65 | ```bash 66 | nosetests --nologcapture 67 | ``` 68 | 69 | ## Cite 70 | 71 | Please cite my master thesis if you use this code in your own work: 72 | 73 | ``` 74 | @mastersthesis{Fey2017, 75 | title={{Convolutional Neural Networks auf Graphrepr{\"a}sentationen von Bildern}}, 76 | author={Matthias Fey}, 77 | school={Technische Universit{\"a}t Dortmund}, 78 | year={2017}, 79 | } 80 | ``` 81 | -------------------------------------------------------------------------------- /cifar_conv2d.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from __future__ import division 3 | 4 | from six.moves import xrange 5 | import time 6 | 7 | import tensorflow as tf 8 | 9 | from lib.datasets import Cifar10 as Data 10 | from lib.model import Model as BaseModel 11 | from lib.layer import ImageAugment, Conv2d, MaxPool, FC 12 | 13 | DATA_DIR = 'data/cifar_10' 14 | 15 | LEARNING_RATE = 0.001 16 | NUM_STEPS_PER_DECAY = 5000 17 | TRAIN_DIR = None 18 | LOG_DIR = 'data/summaries/cifar_conv2d' 19 | SAVE_STEP = 250 20 | 21 | DROPOUT = 0.5 22 | BATCH_SIZE = 64 23 | MAX_STEPS = 60000 24 | DISPLAY_STEP = 10 25 | 26 | data = Data(DATA_DIR) 27 | 28 | placeholders = { 29 | 'features': 30 | tf.placeholder(tf.float32, 31 | [None, data.width, data.height, 32 | data.num_channels], 'features'), 33 | 'labels': 34 | tf.placeholder(tf.uint8, [None, data.num_classes], 'labels'), 35 | 'dropout': 36 | tf.placeholder(tf.float32, [], 'dropout'), 37 | } 38 | 39 | 40 | class Model(BaseModel): 41 | def _build(self): 42 | augment = ImageAugment() 43 | conv_1_1 = Conv2d(data.num_channels, 64, size=3, logging=self.logging) 44 | conv_1_2 = Conv2d(64, 64, size=3, logging=self.logging) 45 | max_pool_1 = MaxPool(2) 46 | conv_2 = Conv2d(64, 128, size=3, logging=self.logging) 47 | max_pool_2 = MaxPool(2) 48 | conv_3 = Conv2d(128, 256, size=3, logging=self.logging) 49 | max_pool_3 = MaxPool(2) 50 | fc_1 = FC(4 * 4 * 256, 256, weight_decay=0.004, logging=self.logging) 51 | fc_2 = FC(256, 128, weight_decay=0.004, logging=self.logging) 52 | fc_3 = FC( 53 | 128, 54 | data.num_classes, 55 | act=lambda x: x, 56 | bias=False, 57 | dropout=self.placeholders['dropout'], 58 | logging=self.logging) 59 | 60 | self.layers = [ 61 | augment, conv_1_1, conv_1_2, max_pool_1, conv_2, max_pool_2, 62 | conv_3, max_pool_3, fc_1, fc_2, fc_3 63 | ] 64 | 65 | 66 | model = Model( 67 | placeholders=placeholders, 68 | learning_rate=LEARNING_RATE, 69 | num_steps_per_decay=NUM_STEPS_PER_DECAY, 70 | train_dir=TRAIN_DIR, 71 | log_dir=LOG_DIR) 72 | 73 | model.build() 74 | global_step = model.initialize() 75 | 76 | 77 | def feed_dict_with_batch(images, labels, dropout=0): 78 | return { 79 | placeholders['features']: images, 80 | placeholders['labels']: labels, 81 | placeholders['dropout']: DROPOUT, 82 | } 83 | 84 | 85 | try: 86 | for step in xrange(global_step, MAX_STEPS): 87 | t_pre = time.process_time() 88 | images, labels = data.train.next_batch(BATCH_SIZE, shuffle=True) 89 | feed_dict = feed_dict_with_batch(images, labels, DROPOUT) 90 | t_pre = time.process_time() - t_pre 91 | 92 | t_train = model.train(feed_dict, step) 93 | 94 | if step % DISPLAY_STEP == 0: 95 | # Evaluate on training and validation set with zero dropout. 96 | feed_dict.update({model.placeholders['dropout']: 0}) 97 | train_info = model.evaluate(feed_dict, step, 'train') 98 | images, labels = data.val.next_batch(BATCH_SIZE, shuffle=True) 99 | feed_dict = feed_dict_with_batch(images, labels) 100 | val_info = model.evaluate(feed_dict, step, 'val') 101 | 102 | log = 'step={}, '.format(step) 103 | log += 'time={:.2f}s + {:.2f}s, '.format(t_pre, t_train) 104 | log += 'train_loss={:.5f}, '.format(train_info[0]) 105 | log += 'train_acc={:.5f}, '.format(train_info[1]) 106 | log += 'val_loss={:.5f}, '.format(val_info[0]) 107 | log += 'val_acc={:.5f}'.format(val_info[1]) 108 | print(log) 109 | 110 | if step % SAVE_STEP == 0: 111 | model.save() 112 | 113 | except KeyboardInterrupt: 114 | print() 115 | 116 | print('Optimization finished!') 117 | print('Evaluate on test set. This can take a few minutes.') 118 | 119 | try: 120 | num_steps = data.test.num_examples // BATCH_SIZE 121 | test_info = [0, 0] 122 | 123 | for i in xrange(num_steps): 124 | images, labels = data.test.next_batch(BATCH_SIZE, shuffle=False) 125 | feed_dict = feed_dict_with_batch(images, labels) 126 | 127 | batch_info = model.evaluate(feed_dict) 128 | test_info = [a + b for a, b in zip(test_info, batch_info)] 129 | 130 | log = 'Test results: ' 131 | log += 'loss={:.5f}, '.format(test_info[0] / num_steps) 132 | log += 'acc={:.5f}, '.format(test_info[1] / num_steps) 133 | 134 | print(log) 135 | 136 | except KeyboardInterrupt: 137 | print() 138 | print('Test evaluation aborted.') 139 | -------------------------------------------------------------------------------- /cifar_graph.py: -------------------------------------------------------------------------------- 1 | from lib.datasets import Cifar10 as Data 2 | from lib.model import Model as BaseModel, generate_placeholders, train 3 | from lib.segmentation import extract_features_fixed 4 | # from lib.segmentation import slic_fixed 5 | from lib.segmentation import quickshift_fixed 6 | from lib.pipeline import preprocess_pipeline_fixed 7 | from lib.layer import EmbeddedGCNN as Conv, MaxPool, AveragePool, FC 8 | 9 | # SLIC_FEATURES = [0, 2, 5, 7, 8, 9, 19, 21, 22] 10 | QUICKSHIFT_FEATURES = [2, 3, 4, 5, 7, 8, 19, 21, 22] 11 | 12 | DATA_DIR = 'data/cifar_10' 13 | 14 | PREPROCESS_FIRST = None 15 | # PREPROCESS_FIRST = 'data/cifar_10/slic' 16 | # PREPROCESS_FIRST = 'data/cifar_10/quickshift' 17 | 18 | LEVELS = 4 19 | CONNECTIVITY = 8 20 | SCALE_INVARIANCE = False 21 | STDDEV = 1 22 | 23 | LEARNING_RATE = 0.001 24 | NUM_STEPS_PER_DECAY = 5000 25 | TRAIN_DIR = None 26 | # LOG_DIR = 'data/summaries/cifar_slic_graph' 27 | LOG_DIR = 'data/summaries/cifar_quickshift_graph' 28 | 29 | AUGMENT_TRAIN_EXAMPLES = True 30 | DROPOUT = 0.5 31 | BATCH_SIZE = 64 32 | MAX_STEPS = 60000 33 | DISPLAY_STEP = 10 34 | # FORM_FEATURES = SLIC_FEATURES 35 | FORM_FEATURES = QUICKSHIFT_FEATURES 36 | NUM_FEATURES = len(FORM_FEATURES) + 3 37 | 38 | data = Data(DATA_DIR) 39 | 40 | # segmentation_algorithm = slic_fixed( 41 | # num_segments=200, compactness=5, max_iterations=10, sigma=0) 42 | segmentation_algorithm = quickshift_fixed( 43 | ratio=1, kernel_size=1, max_dist=5, sigma=0) 44 | 45 | feature_extraction_algorithm = extract_features_fixed(FORM_FEATURES) 46 | 47 | preprocess_algorithm = preprocess_pipeline_fixed( 48 | segmentation_algorithm, feature_extraction_algorithm, LEVELS, CONNECTIVITY, 49 | SCALE_INVARIANCE, STDDEV) 50 | 51 | 52 | class Model(BaseModel): 53 | def _build(self): 54 | conv_1_1 = Conv( 55 | NUM_FEATURES, 56 | 64, 57 | adjs_dist=self.placeholders['adj_dist_1'], 58 | adjs_rad=self.placeholders['adj_rad_1'], 59 | logging=self.logging) 60 | conv_1_2 = Conv( 61 | 64, 62 | 64, 63 | adjs_dist=self.placeholders['adj_dist_1'], 64 | adjs_rad=self.placeholders['adj_rad_1'], 65 | logging=self.logging) 66 | max_pool_1 = MaxPool(size=2) 67 | conv_2 = Conv( 68 | 64, 69 | 128, 70 | adjs_dist=self.placeholders['adj_dist_2'], 71 | adjs_rad=self.placeholders['adj_rad_2'], 72 | logging=self.logging) 73 | max_pool_2 = MaxPool(size=2) 74 | conv_3 = Conv( 75 | 128, 76 | 256, 77 | adjs_dist=self.placeholders['adj_dist_3'], 78 | adjs_rad=self.placeholders['adj_rad_3'], 79 | logging=self.logging) 80 | max_pool_3 = MaxPool(size=2) 81 | conv_4 = Conv( 82 | 256, 83 | 512, 84 | adjs_dist=self.placeholders['adj_dist_4'], 85 | adjs_rad=self.placeholders['adj_rad_4'], 86 | logging=self.logging) 87 | average_pool = AveragePool() 88 | fc_1 = FC(512, 256, weight_decay=0.004, logging=self.logging) 89 | fc_2 = FC(256, 128, weight_decay=0.004, logging=self.logging) 90 | fc_3 = FC( 91 | 128, 92 | data.num_classes, 93 | act=lambda x: x, 94 | bias=False, 95 | dropout=self.placeholders['dropout'], 96 | logging=self.logging) 97 | 98 | self.layers = [ 99 | conv_1_1, conv_1_2, max_pool_1, conv_2, max_pool_2, conv_3, 100 | max_pool_3, conv_4, average_pool, fc_1, fc_2, fc_3 101 | ] 102 | 103 | 104 | placeholders = generate_placeholders(BATCH_SIZE, LEVELS, NUM_FEATURES, 105 | data.num_classes) 106 | 107 | model = Model( 108 | placeholders=placeholders, 109 | learning_rate=LEARNING_RATE, 110 | num_steps_per_decay=NUM_STEPS_PER_DECAY, 111 | train_dir=TRAIN_DIR, 112 | log_dir=LOG_DIR) 113 | 114 | train(model, data, preprocess_algorithm, BATCH_SIZE, DROPOUT, 115 | AUGMENT_TRAIN_EXAMPLES, MAX_STEPS, PREPROCESS_FIRST, DISPLAY_STEP) 116 | -------------------------------------------------------------------------------- /lib/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rusty1s/embedded_gcnn/06db3799e794d6ebcd9db023ebd8b0937587df94/lib/__init__.py -------------------------------------------------------------------------------- /lib/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | from .mnist import MNIST 2 | from .cifar_10 import Cifar10 3 | from .pascal_voc import PascalVOC 4 | from .queue import PreprocessQueue 5 | 6 | __all__ = ['MNIST', 'Cifar10', 'PascalVOC', 'PreprocessQueue'] 7 | -------------------------------------------------------------------------------- /lib/datasets/augment.py: -------------------------------------------------------------------------------- 1 | import random 2 | import numpy as np 3 | 4 | 5 | def flip_left_right_image(image): 6 | return np.fliplr(image) 7 | 8 | 9 | def random_flip_left_right_image(image, rand=None): 10 | rand = bool(random.getrandbits(1)) if rand is None else rand 11 | return flip_left_right_image(image) if rand else image 12 | 13 | 14 | def adjust_brightness(image, delta): 15 | image = image + delta 16 | image = np.clip(image, 0, 1) 17 | return image 18 | 19 | 20 | def random_brightness(image, max_delta): 21 | rand = random.uniform(-max_delta, max_delta) 22 | return adjust_brightness(image, rand) 23 | 24 | 25 | def adjust_contrast(image, delta): 26 | mean = image.mean(axis=(0, 1)) 27 | image = (image - mean) * (1 + delta) + mean 28 | image = np.clip(image, 0, 1) 29 | return image 30 | 31 | 32 | def random_contrast(image, max_delta): 33 | rand = random.uniform(-max_delta, max_delta) 34 | return adjust_contrast(image, rand) 35 | -------------------------------------------------------------------------------- /lib/datasets/augment_test.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | import numpy as np 4 | from numpy.testing import assert_equal, assert_almost_equal 5 | 6 | from .augment import (flip_left_right_image, random_flip_left_right_image, 7 | adjust_brightness, random_brightness, adjust_contrast, 8 | random_contrast) 9 | 10 | 11 | class AugmentTest(TestCase): 12 | def test_flip_left_right_image(self): 13 | image = np.array([ 14 | [[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]], 15 | [[0.6, 0.5, 0.4], [0.3, 0.2, 0.1]], 16 | ]) 17 | 18 | expected = [ 19 | [[0.4, 0.5, 0.6], [0.1, 0.2, 0.3]], 20 | [[0.3, 0.2, 0.1], [0.6, 0.5, 0.4]], 21 | ] 22 | 23 | assert_equal(flip_left_right_image(image), expected) 24 | 25 | assert_equal(random_flip_left_right_image(image, True), expected) 26 | assert_equal(random_flip_left_right_image(image, False), image) 27 | 28 | random = random_flip_left_right_image(image) 29 | self.assertTrue( 30 | np.array_equal(random, image) or np.array_equal(random, expected)) 31 | 32 | def test_adjust_brightness(self): 33 | image = np.array([ 34 | [[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]], 35 | [[0.6, 0.5, 0.4], [0.3, 0.2, 0.1]], 36 | ]) 37 | 38 | expected = [ 39 | [[0.6, 0.7, 0.8], [0.9, 1.0, 1.0]], 40 | [[1.0, 1.0, 0.9], [0.8, 0.7, 0.6]], 41 | ] 42 | 43 | assert_equal(adjust_brightness(image, 0.5), expected) 44 | 45 | self.assertGreaterEqual(random_brightness(image, 0.5).min(), 0) 46 | self.assertLessEqual(random_brightness(image, 0.5).max(), 1) 47 | 48 | def test_adjust_contrast(self): 49 | image = np.array([ 50 | [[0.1, 0.4, 0.3], [0.4, 0.5, 0.6]], 51 | [[0.6, 0.5, 0.4], [0.3, 0.2, 0.5]], 52 | ]) 53 | 54 | expected = [ 55 | [[0, 0.4, 0.225], [0.425, 0.55, 0.675]], 56 | [[0.725, 0.55, 0.375], [0.275, 0.1, 0.525]], 57 | ] 58 | 59 | assert_almost_equal(adjust_contrast(image, 0.5), expected) 60 | 61 | self.assertGreaterEqual(random_contrast(image, 0.5).min(), 0) 62 | self.assertLessEqual(random_contrast(image, 0.5).max(), 1) 63 | -------------------------------------------------------------------------------- /lib/datasets/cifar_10.py: -------------------------------------------------------------------------------- 1 | from __future__ import division 2 | 3 | import os 4 | import sys 5 | from six.moves import xrange 6 | 7 | import numpy as np 8 | 9 | from .dataset import Datasets, Dataset 10 | from .download import maybe_download_and_extract 11 | 12 | try: 13 | import cPickle as pickle 14 | except ImportError: 15 | import pickle 16 | 17 | URL = 'http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz' 18 | 19 | 20 | class Cifar10(Datasets): 21 | def __init__(self, data_dir, val_size=5000): 22 | maybe_download_and_extract(URL, data_dir) 23 | self.data_dir = os.path.join(data_dir, 'cifar-10-batches-py') 24 | 25 | images = np.zeros( 26 | (0, self.width * self.height * self.num_channels), 27 | dtype=np.float32) 28 | labels = np.zeros((0), dtype=np.uint8) 29 | 30 | for i in xrange(1, 6): 31 | batch = self._load_batch('data_batch_{}'.format(i)) 32 | images = np.concatenate((images, batch[0]), axis=0) 33 | labels = np.concatenate((labels, batch[1]), axis=0) 34 | 35 | images = self._preprocess_images(images) 36 | labels = self._preprocess_labels(labels) 37 | 38 | train = Dataset(images, labels) 39 | 40 | images, labels = self._load_batch('test_batch') 41 | images = self._preprocess_images(images) 42 | labels = self._preprocess_labels(labels) 43 | val = Dataset(images, labels) 44 | test = Dataset(images, labels) 45 | 46 | super(Cifar10, self).__init__(train, val, test) 47 | 48 | @property 49 | def classes(self): 50 | return [ 51 | 'airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 52 | 'horse', 'ship', 'truck' 53 | ] 54 | 55 | @property 56 | def width(self): 57 | return 32 58 | 59 | @property 60 | def height(self): 61 | return 32 62 | 63 | @property 64 | def num_channels(self): 65 | return 3 66 | 67 | def _preprocess_images(self, images): 68 | images = np.reshape(images, 69 | (-1, self.num_channels, self.width, self.height)) 70 | images = np.transpose(images, (0, 2, 3, 1)) 71 | return (1 / 255) * images.astype(np.float32) 72 | 73 | def _preprocess_labels(self, labels): 74 | # Convert labels to one hot. 75 | labels = np.array(labels, np.uint8) 76 | size = labels.shape[0] 77 | index_offset = np.arange(size) * self.num_classes 78 | labels_one_hot = np.zeros((size, self.num_classes), np.uint8) 79 | labels_one_hot.flat[index_offset + labels] = 1 80 | return labels_one_hot 81 | 82 | def _load_batch(self, name): 83 | with open(os.path.join(self.data_dir, name), 'rb') as f: 84 | if sys.version_info >= (3, 0): 85 | batch = pickle.load(f, encoding='latin1') 86 | else: # pragma: no cover 87 | batch = pickle.load(f) 88 | return batch['data'], batch['labels'] 89 | -------------------------------------------------------------------------------- /lib/datasets/cifar_10_test.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | import numpy as np 4 | 5 | from .cifar_10 import Cifar10 6 | 7 | data = Cifar10('data/cifar_10', val_size=10000) 8 | 9 | 10 | class Cifar10Test(TestCase): 11 | def test_init(self): 12 | self.assertEqual(data.train.num_examples, 50000) 13 | self.assertEqual(data.val.num_examples, 10000) 14 | self.assertEqual(data.test.num_examples, 10000) 15 | 16 | def test_shapes(self): 17 | images, labels = data.train.next_batch(32, shuffle=False) 18 | self.assertEqual(images.shape, (32, 32, 32, 3)) 19 | self.assertEqual(labels.shape, (32, 10)) 20 | data.train.next_batch(data.train.num_examples - 32, shuffle=False) 21 | 22 | images, labels = data.val.next_batch(32, shuffle=False) 23 | self.assertEqual(images.shape, (32, 32, 32, 3)) 24 | self.assertEqual(labels.shape, (32, 10)) 25 | data.val.next_batch(data.val.num_examples - 32, shuffle=False) 26 | 27 | images, labels = data.test.next_batch(32, shuffle=False) 28 | self.assertEqual(images.shape, (32, 32, 32, 3)) 29 | self.assertEqual(labels.shape, (32, 10)) 30 | data.test.next_batch(data.test.num_examples - 32, shuffle=False) 31 | 32 | def test_images(self): 33 | images, _ = data.train.next_batch( 34 | data.train.num_examples, shuffle=False) 35 | 36 | self.assertEqual(images.dtype, np.float32) 37 | self.assertLessEqual(images.max(), 1) 38 | self.assertGreaterEqual(images.min(), 0) 39 | 40 | images, _ = data.val.next_batch(data.val.num_examples, shuffle=False) 41 | 42 | self.assertEqual(images.dtype, np.float32) 43 | self.assertLessEqual(images.max(), 1) 44 | self.assertGreaterEqual(images.min(), 0) 45 | 46 | images, _ = data.test.next_batch(data.test.num_examples, shuffle=False) 47 | 48 | self.assertEqual(images.dtype, np.float32) 49 | self.assertLessEqual(images.max(), 1) 50 | self.assertGreaterEqual(images.min(), 0) 51 | 52 | def test_labels(self): 53 | _, labels = data.train.next_batch( 54 | data.train.num_examples, shuffle=False) 55 | 56 | self.assertEqual(labels.dtype, np.uint8) 57 | 58 | _, labels = data.val.next_batch(data.val.num_examples, shuffle=False) 59 | 60 | self.assertEqual(labels.dtype, np.uint8) 61 | 62 | _, labels = data.test.next_batch(data.test.num_examples, shuffle=False) 63 | 64 | self.assertEqual(labels.dtype, np.uint8) 65 | 66 | def test_class_functions(self): 67 | self.assertEqual(data.classes, [ 68 | 'airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 69 | 'horse', 'ship', 'truck' 70 | ]) 71 | self.assertEqual(data.num_classes, 10) 72 | 73 | _, labels = data.test.next_batch(5, shuffle=False) 74 | 75 | self.assertEqual(data.classnames(labels[0]), ['cat']) 76 | self.assertEqual(data.classnames(labels[1]), ['ship']) 77 | self.assertEqual(data.classnames(labels[2]), ['ship']) 78 | self.assertEqual(data.classnames(labels[3]), ['airplane']) 79 | self.assertEqual(data.classnames(labels[4]), ['frog']) 80 | 81 | data.test.next_batch(data.test.num_examples - 5, shuffle=False) 82 | -------------------------------------------------------------------------------- /lib/datasets/dataset.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | class Datasets(object): 5 | def __init__(self, train, val, test): 6 | self.train = train 7 | self.val = val 8 | self.test = test 9 | 10 | @property 11 | def classes(self): 12 | raise NotImplementedError 13 | 14 | @property 15 | def width(self): 16 | raise NotImplementedError 17 | 18 | @property 19 | def height(self): 20 | raise NotImplementedError 21 | 22 | @property 23 | def num_channels(self): 24 | raise NotImplementedError 25 | 26 | @property 27 | def num_classes(self): 28 | return len(self.classes) 29 | 30 | def classnames(self, label): 31 | idx = np.where(label == 1)[0] 32 | return [self.classes[i] for i in idx] 33 | 34 | 35 | class Dataset(object): 36 | def __init__(self, images, labels): 37 | self.epochs_completed = 0 38 | 39 | self._images = images 40 | self._labels = labels 41 | self._index_in_epoch = 0 42 | 43 | @property 44 | def num_examples(self): 45 | return self._labels.shape[0] 46 | 47 | def _random_shuffle_examples(self): 48 | perm = np.arange(self.num_examples) 49 | np.random.shuffle(perm) 50 | self._images = self._images[perm] 51 | self._labels = self._labels[perm] 52 | 53 | def next_batch(self, batch_size, shuffle=True): 54 | start = self._index_in_epoch 55 | 56 | # Shuffle for the first epoch. 57 | if self.epochs_completed == 0 and start == 0 and shuffle: 58 | self._random_shuffle_examples() 59 | 60 | if start + batch_size > self.num_examples: 61 | # Finished epoch. 62 | self.epochs_completed += 1 63 | 64 | # Get the rest examples in this epoch. 65 | rest_num_examples = self.num_examples - start 66 | images_rest = self._images[start:self.num_examples] 67 | labels_rest = self._labels[start:self.num_examples] 68 | 69 | # Shuffle the examples. 70 | if shuffle: 71 | self._random_shuffle_examples() 72 | 73 | # Start next epoch. 74 | start = 0 75 | self._index_in_epoch = batch_size - rest_num_examples 76 | end = self._index_in_epoch 77 | images_new = self._images[start:end] 78 | labels_new = self._labels[start:end] 79 | 80 | labels = np.concatenate((labels_rest, labels_new), axis=0) 81 | images = np.concatenate((images_rest, images_new), axis=0) 82 | else: 83 | # Just slice the examples. 84 | self._index_in_epoch += batch_size 85 | end = self._index_in_epoch 86 | images = self._images[start:end] 87 | labels = self._labels[start:end] 88 | 89 | return images, labels 90 | -------------------------------------------------------------------------------- /lib/datasets/dataset_test.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | import numpy as np 4 | from numpy.testing import assert_equal 5 | 6 | from .dataset import Datasets, Dataset 7 | 8 | 9 | class DatasetTest(TestCase): 10 | def test_init_dataset(self): 11 | images = np.array([1, 2, 3, 4, 5]) 12 | labels = np.array([0, 1, 0, 1, 0]) 13 | dataset = Dataset(images, labels) 14 | 15 | self.assertEqual(dataset.epochs_completed, 0) 16 | self.assertEqual(dataset._index_in_epoch, 0) 17 | self.assertEqual(dataset.num_examples, 5) 18 | 19 | def test_next_batch(self): 20 | images = np.array([1, 2, 3, 4, 5]) 21 | labels = np.array([0, 1, 0, 1, 0]) 22 | dataset = Dataset(images, labels) 23 | 24 | batch = dataset.next_batch(4, shuffle=False) 25 | assert_equal(batch[0], [1, 2, 3, 4]) 26 | assert_equal(batch[1], [0, 1, 0, 1]) 27 | 28 | batch = dataset.next_batch(4, shuffle=False) 29 | assert_equal(batch[0], [5, 1, 2, 3]) 30 | assert_equal(batch[1], [0, 0, 1, 0]) 31 | 32 | batch = dataset.next_batch(2, shuffle=False) 33 | assert_equal(batch[0], [4, 5]) 34 | assert_equal(batch[1], [1, 0]) 35 | 36 | batch = dataset.next_batch(5, shuffle=False) 37 | assert_equal(batch[0], [1, 2, 3, 4, 5]) 38 | assert_equal(batch[1], [0, 1, 0, 1, 0]) 39 | 40 | def test_next_batch_shuffle(self): 41 | images = np.array([1, 2, 3, 4, 5]) 42 | labels = np.array([0, 1, 0, 1, 0]) 43 | dataset = Dataset(images, labels) 44 | 45 | batch_1 = dataset.next_batch(4, shuffle=True) 46 | self.assertEqual(batch_1[0].shape, (4, )) 47 | self.assertEqual(batch_1[1].shape, (4, )) 48 | 49 | batch_2 = dataset.next_batch(1, shuffle=True) 50 | self.assertEqual(batch_2[0].shape, (1, )) 51 | self.assertEqual(batch_2[1].shape, (1, )) 52 | 53 | images = np.concatenate((batch_1[0], batch_2[0]), axis=0) 54 | labels = np.concatenate((batch_1[1], batch_2[1]), axis=0) 55 | 56 | assert_equal(np.sort(images), [1, 2, 3, 4, 5]) 57 | assert_equal(np.sort(labels), [0, 0, 0, 1, 1]) 58 | 59 | images, labels = dataset.next_batch(5, shuffle=True) 60 | assert_equal(np.sort(images), [1, 2, 3, 4, 5]) 61 | assert_equal(np.sort(labels), [0, 0, 0, 1, 1]) 62 | 63 | def test_init_datasets(self): 64 | images = [1, 2, 3, 4, 5] 65 | labels = np.array([0, 1, 0, 1, 0]) 66 | dataset = Dataset(images, labels) 67 | datasets = Datasets(dataset, dataset, dataset) 68 | 69 | batch = datasets.train.next_batch(5, shuffle=False) 70 | assert_equal(images, batch[0]) 71 | assert_equal(labels, batch[1]) 72 | batch = datasets.val.next_batch(5, shuffle=False) 73 | assert_equal(images, batch[0]) 74 | assert_equal(labels, batch[1]) 75 | batch = datasets.test.next_batch(5, shuffle=False) 76 | assert_equal(images, batch[0]) 77 | assert_equal(labels, batch[1]) 78 | -------------------------------------------------------------------------------- /lib/datasets/download.py: -------------------------------------------------------------------------------- 1 | from __future__ import division 2 | from __future__ import print_function 3 | 4 | import os 5 | import sys 6 | import tarfile 7 | from six.moves import urllib 8 | 9 | 10 | def _print_status(name, percentage): 11 | sys.stdout.write('\r>> Downloading {} {:.2f}%'.format(name, percentage)) 12 | sys.stdout.flush() 13 | 14 | 15 | def maybe_download_and_extract(url, data_dir): 16 | if not os.path.exists(data_dir): 17 | os.makedirs(data_dir) 18 | 19 | filename = url.split('/')[-1] 20 | filepath = os.path.join(data_dir, filename) 21 | 22 | # Only download if file doesn't exist. 23 | if not os.path.exists(filepath): 24 | 25 | def _progress(count, block_size, total_size): 26 | percentage = 100 * count * block_size / total_size 27 | _print_status(filename, percentage) 28 | 29 | filepath, _ = urllib.request.urlretrieve(url, filepath, _progress) 30 | size = os.stat(filepath).st_size 31 | 32 | print() 33 | print('Successfully downloaded {} ({} bytes).'.format(filename, size)) 34 | 35 | if ".tar" in filename: 36 | return extract_tar(data_dir, filename) 37 | else: # pragma: no cover 38 | return filepath 39 | 40 | 41 | def extract_tar(data_dir, filename): 42 | filepath = os.path.join(data_dir, filename) 43 | mode = 'r:gz' if filename.split('.')[-1] == 'gz' else 'r' 44 | archive = tarfile.open(filepath, mode) 45 | 46 | # Get the top level directory in the tar file. 47 | extracted_dir = os.path.join(data_dir, 48 | os.path.commonprefix(archive.getnames())) 49 | 50 | # Only extract if file doesn't exist. 51 | if not os.path.exists(extracted_dir): 52 | sys.stdout.write( 53 | '>> Extracting {} to {}...'.format(filename, extracted_dir)) 54 | sys.stdout.flush() 55 | 56 | tarfile.open(filepath, mode).extractall(data_dir) 57 | 58 | print(' Done!') 59 | 60 | return extracted_dir 61 | -------------------------------------------------------------------------------- /lib/datasets/download_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rusty1s/embedded_gcnn/06db3799e794d6ebcd9db023ebd8b0937587df94/lib/datasets/download_test.py -------------------------------------------------------------------------------- /lib/datasets/mnist.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | 4 | from .dataset import Datasets, Dataset 5 | 6 | 7 | class MNIST(Datasets): 8 | def __init__(self, data_dir, val_size=5000): 9 | mnist = input_data.read_data_sets( 10 | data_dir, one_hot=True, validation_size=val_size) 11 | 12 | images = self._preprocess_images(mnist.train.images) 13 | labels = self._preprocess_labels(mnist.train.labels) 14 | train = Dataset(images, labels) 15 | 16 | images = self._preprocess_images(mnist.validation.images) 17 | labels = self._preprocess_labels(mnist.validation.labels) 18 | val = Dataset(images, labels) 19 | 20 | images = self._preprocess_images(mnist.test.images) 21 | labels = self._preprocess_labels(mnist.test.labels) 22 | test = Dataset(images, labels) 23 | 24 | super(MNIST, self).__init__(train, val, test) 25 | 26 | @property 27 | def classes(self): 28 | return ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9'] 29 | 30 | @property 31 | def width(self): 32 | return 28 33 | 34 | @property 35 | def height(self): 36 | return 28 37 | 38 | @property 39 | def num_channels(self): 40 | return 1 41 | 42 | def _preprocess_images(self, images): 43 | return np.reshape(images, (-1, self.height, self.width, 44 | self.num_channels)) 45 | 46 | def _preprocess_labels(self, labels): 47 | return labels.astype(np.uint8) 48 | -------------------------------------------------------------------------------- /lib/datasets/mnist_test.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | import numpy as np 4 | 5 | from .mnist import MNIST 6 | 7 | data = MNIST('data/mnist', val_size=10000) 8 | 9 | 10 | class MNISTTest(TestCase): 11 | def test_init(self): 12 | self.assertEqual(data.train.num_examples, 50000) 13 | self.assertEqual(data.val.num_examples, 10000) 14 | self.assertEqual(data.test.num_examples, 10000) 15 | 16 | def test_shapes(self): 17 | images, labels = data.train.next_batch(32, shuffle=False) 18 | self.assertEqual(images.shape, (32, 28, 28, 1)) 19 | self.assertEqual(labels.shape, (32, 10)) 20 | data.train.next_batch(data.train.num_examples - 32, shuffle=False) 21 | 22 | images, labels = data.val.next_batch(32, shuffle=False) 23 | self.assertEqual(images.shape, (32, 28, 28, 1)) 24 | self.assertEqual(labels.shape, (32, 10)) 25 | data.val.next_batch(data.val.num_examples - 32, shuffle=False) 26 | 27 | images, labels = data.test.next_batch(32, shuffle=False) 28 | self.assertEqual(images.shape, (32, 28, 28, 1)) 29 | self.assertEqual(labels.shape, (32, 10)) 30 | data.test.next_batch(data.test.num_examples - 32, shuffle=False) 31 | 32 | def test_images(self): 33 | images, _ = data.train.next_batch( 34 | data.train.num_examples, shuffle=False) 35 | 36 | self.assertEqual(images.dtype, np.float32) 37 | self.assertLessEqual(images.max(), 1) 38 | self.assertGreaterEqual(images.min(), 0) 39 | 40 | images, _ = data.val.next_batch(data.val.num_examples, shuffle=False) 41 | 42 | self.assertEqual(images.dtype, np.float32) 43 | self.assertLessEqual(images.max(), 1) 44 | self.assertGreaterEqual(images.min(), 0) 45 | 46 | images, _ = data.test.next_batch(data.test.num_examples, shuffle=False) 47 | 48 | self.assertEqual(images.dtype, np.float32) 49 | self.assertLessEqual(images.max(), 1) 50 | self.assertGreaterEqual(images.min(), 0) 51 | 52 | def test_labels(self): 53 | _, labels = data.train.next_batch( 54 | data.train.num_examples, shuffle=False) 55 | 56 | self.assertEqual(labels.dtype, np.uint8) 57 | 58 | _, labels = data.val.next_batch( 59 | data.val.num_examples, shuffle=False) 60 | 61 | self.assertEqual(labels.dtype, np.uint8) 62 | 63 | _, labels = data.test.next_batch( 64 | data.test.num_examples, shuffle=False) 65 | 66 | self.assertEqual(labels.dtype, np.uint8) 67 | 68 | def test_class_functions(self): 69 | self.assertEqual(data.classes, 70 | ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']) 71 | self.assertEqual(data.num_classes, 10) 72 | 73 | _, labels = data.test.next_batch(5, shuffle=False) 74 | 75 | self.assertEqual(data.classnames(labels[0]), ['7']) 76 | self.assertEqual(data.classnames(labels[1]), ['2']) 77 | self.assertEqual(data.classnames(labels[2]), ['1']) 78 | self.assertEqual(data.classnames(labels[3]), ['0']) 79 | self.assertEqual(data.classnames(labels[4]), ['4']) 80 | 81 | data.test.next_batch(data.test.num_examples - 5, shuffle=False) 82 | -------------------------------------------------------------------------------- /lib/datasets/pascal_voc.py: -------------------------------------------------------------------------------- 1 | from __future__ import division 2 | 3 | import os 4 | from math import ceil, floor 5 | from xml.dom.minidom import parse 6 | 7 | import numpy as np 8 | from skimage.io import imread 9 | from skimage.transform import rescale 10 | 11 | from .dataset import Datasets 12 | from .download import maybe_download_and_extract 13 | 14 | 15 | URL = 'http://host.robots.ox.ac.uk/pascal/VOC/voc2012/'\ 16 | 'VOCtrainval_11-May-2012.tar' 17 | CLASSES = [ 18 | 'person', 'bird', 'cat', 'cow', 'dog', 'horse', 'sheep', 'aeroplane', 19 | 'bicycle', 'boat', 'bus', 'car', 'motorbike', 'train', 'bottle', 'chair', 20 | 'diningtable', 'pottedplant', 'sofa', 'tvmonitor' 21 | ] 22 | 23 | 24 | class PascalVOC(Datasets): 25 | def __init__(self, data_dir, val_size=1500, fixed_size=None): 26 | self._fixed_size = fixed_size 27 | 28 | maybe_download_and_extract(URL, data_dir) 29 | 30 | data_dir = os.path.join(data_dir, 'VOCdevkit', 'VOC2012') 31 | names = os.listdir(os.path.join(data_dir, 'Annotations')) 32 | names = [name.split('.')[0] for name in names] 33 | names = sorted(names) 34 | 35 | # PascalVOC didn't release the full test annotations yet, use the 36 | # validation set instead :( 37 | train = Dataset(names[val_size:], data_dir, fixed_size) 38 | val = Dataset(names[:val_size], data_dir, fixed_size) 39 | test = Dataset(names[:val_size], data_dir, fixed_size) 40 | 41 | super(PascalVOC, self).__init__(train, val, test) 42 | 43 | @property 44 | def classes(self): 45 | return CLASSES 46 | 47 | @property 48 | def width(self): 49 | return self._fixed_size 50 | 51 | @property 52 | def height(self): 53 | return self._fixed_size 54 | 55 | @property 56 | def num_channels(self): 57 | return 3 58 | 59 | 60 | class Dataset(object): 61 | def __init__(self, names, data_dir, fixed_size=None): 62 | self.epochs_completed = 0 63 | 64 | self._data_dir = data_dir 65 | self._fixed_size = fixed_size 66 | self._names = names 67 | self._index_in_epoch = 0 68 | 69 | @property 70 | def num_examples(self): 71 | return len(self._names) 72 | 73 | def _random_shuffle_examples(self): 74 | perm = np.arange(self.num_examples) 75 | np.random.shuffle(perm) 76 | self._names = [self._names[i] for i in perm] 77 | 78 | def next_batch(self, batch_size, shuffle=True): 79 | # This code is nearly complete identical to the default next_batch 80 | # method of the default dataset class, but instead of shuffling the 81 | # examples in memory, we shuffle just the filenames. 82 | # 83 | # Therefore the duplication check of this file is diabled in 84 | # .codeclimate.yml. 85 | 86 | start = self._index_in_epoch 87 | 88 | # Shuffle for the first epoch. 89 | if self.epochs_completed == 0 and start == 0 and shuffle: 90 | self._random_shuffle_examples() 91 | 92 | if start + batch_size > self.num_examples: 93 | # Finished epoch. 94 | self.epochs_completed += 1 95 | 96 | # Get the rest examples in this epoch. 97 | rest_num_examples = self.num_examples - start 98 | names_rest = self._names[start:self.num_examples] 99 | 100 | # Shuffle the examples. 101 | if shuffle: 102 | self._random_shuffle_examples() 103 | 104 | # Start next epoch. 105 | start = 0 106 | self._index_in_epoch = batch_size - rest_num_examples 107 | end = self._index_in_epoch 108 | names = names_rest + self._names[start:end] 109 | else: 110 | # Just slice the examples. 111 | self._index_in_epoch += batch_size 112 | end = self._index_in_epoch 113 | names = self._names[start:end] 114 | 115 | if self._fixed_size is None: 116 | images = [self._read_image(name) for name in names] 117 | else: 118 | images = np.stack([self._read_image(name) for name in names]) 119 | labels = np.stack([self._read_label(name) for name in names]) 120 | 121 | return images, labels 122 | 123 | def _read_image(self, name): 124 | path = os.path.join(self._data_dir, 'JPEGImages', 125 | '{}.jpg'.format(name)) 126 | image = imread(path) 127 | 128 | if self._fixed_size is None: 129 | image = (1 / 255) * image.astype(np.float32) 130 | return image.astype(np.float32) 131 | else: 132 | height, width, _ = image.shape 133 | 134 | scale_y = self._fixed_size / height 135 | scale_x = self._fixed_size / width 136 | scale = min(scale_y, scale_x) 137 | 138 | image = rescale(image, (scale, scale), mode='constant') 139 | 140 | pad_y = self._fixed_size - image.shape[0] 141 | pad_x = self._fixed_size - image.shape[1] 142 | 143 | image = np.pad(image, 144 | ((int(ceil(pad_y / 2)), int(floor(pad_y / 2))), 145 | (int(ceil(pad_x / 2)), 146 | int(floor(pad_x / 2))), (0, 0)), 'constant') 147 | return image 148 | 149 | def _read_label(self, name): 150 | path = os.path.join(self._data_dir, 'Annotations', 151 | '{}.xml'.format(name)) 152 | annotation = parse(path) 153 | 154 | label = np.zeros((len(CLASSES)), np.uint8) 155 | 156 | # Get the label to the greatest bounding box. 157 | max_area = 0 158 | max_name = '' 159 | for obj in annotation.getElementsByTagName('object'): 160 | name = obj.getElementsByTagName('name')[0].firstChild.nodeValue 161 | bbox = obj.getElementsByTagName('bndbox')[0] 162 | xmin = bbox.getElementsByTagName('xmin')[0].firstChild.nodeValue 163 | xmax = bbox.getElementsByTagName('xmax')[0].firstChild.nodeValue 164 | ymin = bbox.getElementsByTagName('ymin')[0].firstChild.nodeValue 165 | ymax = bbox.getElementsByTagName('ymax')[0].firstChild.nodeValue 166 | area = (float(xmax) - float(xmin)) * (float(ymax) - float(ymin)) 167 | if area > max_area: 168 | max_area = area 169 | max_name = name 170 | 171 | label[CLASSES.index(max_name)] = 1 172 | 173 | return label 174 | -------------------------------------------------------------------------------- /lib/datasets/pascal_voc_test.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | import numpy as np 4 | 5 | from .pascal_voc import PascalVOC 6 | 7 | data = PascalVOC('test_data', val_size=4) 8 | 9 | 10 | class PascalVOCTest(TestCase): 11 | def test_init(self): 12 | self.assertEqual(data.train.num_examples, 4) 13 | self.assertEqual(data.val.num_examples, 4) 14 | self.assertEqual(data.test.num_examples, 4) 15 | 16 | self.assertEqual(data.width, None) 17 | self.assertEqual(data.height, None) 18 | self.assertEqual(data.num_channels, 3) 19 | 20 | def test_shapes(self): 21 | images, labels = data.train.next_batch(4, shuffle=False) 22 | for image in images: 23 | self.assertGreater(image.shape[0], 0) 24 | self.assertGreater(image.shape[0], 0) 25 | self.assertEqual(image.shape[2], 3) 26 | self.assertEqual(labels.shape, (4, 20)) 27 | 28 | images, labels = data.val.next_batch(4, shuffle=False) 29 | for image in images: 30 | self.assertGreater(image.shape[0], 0) 31 | self.assertGreater(image.shape[0], 0) 32 | self.assertEqual(image.shape[2], 3) 33 | self.assertEqual(labels.shape, (4, 20)) 34 | 35 | images, labels = data.test.next_batch(4, shuffle=False) 36 | for image in images: 37 | self.assertGreater(image.shape[0], 0) 38 | self.assertGreater(image.shape[0], 0) 39 | self.assertEqual(image.shape[2], 3) 40 | self.assertEqual(labels.shape, (4, 20)) 41 | 42 | def test_images(self): 43 | images, _ = data.train.next_batch( 44 | data.train.num_examples, shuffle=False) 45 | 46 | for image in images: 47 | self.assertEqual(image.dtype, np.float32) 48 | self.assertLessEqual(image.max(), 1) 49 | self.assertGreaterEqual(image.min(), 0) 50 | 51 | images, _ = data.val.next_batch(data.val.num_examples, shuffle=False) 52 | 53 | for image in images: 54 | self.assertEqual(image.dtype, np.float32) 55 | self.assertLessEqual(image.max(), 1) 56 | self.assertGreaterEqual(image.min(), 0) 57 | 58 | images, _ = data.test.next_batch(data.test.num_examples, shuffle=False) 59 | 60 | for image in images: 61 | self.assertEqual(image.dtype, np.float32) 62 | self.assertLessEqual(image.max(), 1) 63 | self.assertGreaterEqual(image.min(), 0) 64 | 65 | def test_labels(self): 66 | _, labels = data.train.next_batch(4, shuffle=False) 67 | self.assertEqual(labels.dtype, np.uint8) 68 | 69 | _, labels = data.val.next_batch(4, shuffle=False) 70 | self.assertEqual(labels.dtype, np.uint8) 71 | 72 | _, labels = data.test.next_batch(4, shuffle=False) 73 | self.assertEqual(labels.dtype, np.uint8) 74 | 75 | def test_class_functions(self): 76 | self.assertEqual(data.classes, [ 77 | 'person', 'bird', 'cat', 'cow', 'dog', 'horse', 'sheep', 78 | 'aeroplane', 'bicycle', 'boat', 'bus', 'car', 'motorbike', 'train', 79 | 'bottle', 'chair', 'diningtable', 'pottedplant', 'sofa', 80 | 'tvmonitor' 81 | ]) 82 | self.assertEqual(data.num_classes, 20) 83 | 84 | _, labels = data.test.next_batch(4, shuffle=False) 85 | 86 | self.assertEqual(data.classnames(labels[0]), ['person']) 87 | self.assertEqual(data.classnames(labels[1]), ['aeroplane']) 88 | self.assertEqual(data.classnames(labels[2]), ['aeroplane']) 89 | self.assertEqual(data.classnames(labels[3]), ['tvmonitor']) 90 | 91 | def test_next_batch_shuffle(self): 92 | data_new = PascalVOC('test_data', val_size=4) 93 | 94 | _, _ = data_new.train.next_batch(2, shuffle=True) 95 | _, _ = data_new.train.next_batch(2, shuffle=True) 96 | _, _ = data_new.train.next_batch(2, shuffle=True) 97 | _, _ = data_new.train.next_batch(2, shuffle=True) 98 | 99 | def test_fix_size(self): 100 | data_fixed = PascalVOC('test_data', val_size=4, fixed_size=224) 101 | 102 | self.assertEqual(data_fixed.width, 224) 103 | self.assertEqual(data_fixed.height, 224) 104 | self.assertEqual(data_fixed.num_channels, 3) 105 | 106 | images, labels = data_fixed.train.next_batch(2, shuffle=True) 107 | 108 | self.assertEqual(images.shape, (2, 224, 224, 3)) 109 | self.assertEqual(labels.shape, (2, 20)) 110 | -------------------------------------------------------------------------------- /lib/datasets/queue.py: -------------------------------------------------------------------------------- 1 | from threading import Thread, Event 2 | from six.moves import xrange 3 | 4 | from .augment import (random_flip_left_right_image, random_brightness, 5 | random_contrast) 6 | 7 | try: 8 | from queue import Queue 9 | except ImportError: 10 | from Queue import Queue 11 | 12 | 13 | class PreprocessQueue(object): 14 | def __init__(self, 15 | dataset, 16 | preprocess_algorithm, 17 | augment, 18 | batch_size, 19 | capacity, 20 | shuffle=False, 21 | num_threads=1): 22 | 23 | inputs = Queue(capacity) 24 | outputs = Queue(capacity) 25 | stopper = Event() 26 | 27 | class ProducerThread(Thread): 28 | def run(self): 29 | while not stopper.isSet(): 30 | images, labels = dataset.next_batch(batch_size, shuffle) 31 | for i in xrange(batch_size): 32 | inputs.put((images[i], labels[i])) 33 | 34 | class ConsumerThread(Thread): 35 | def run(self): 36 | while not stopper.isSet(): 37 | image, label = inputs.get() 38 | inputs.task_done() 39 | 40 | if augment: 41 | image = random_flip_left_right_image(image) 42 | image = random_brightness(image, max_delta=0.3) 43 | image = random_contrast(image, max_delta=0.3) 44 | 45 | data = preprocess_algorithm(image) 46 | data += (label, ) 47 | 48 | outputs.put(data) 49 | 50 | self._threads = [ProducerThread()] 51 | 52 | for i in xrange(num_threads): 53 | self._threads.append(ConsumerThread()) 54 | for t in self._threads: 55 | t.start() 56 | 57 | self._stopper = stopper 58 | self._inputs = inputs 59 | self._outputs = outputs 60 | self._batch_size = batch_size 61 | 62 | def dequeue(self): 63 | batch = [] 64 | 65 | for i in xrange(self._batch_size): 66 | data = self._outputs.get() 67 | self._outputs.task_done() 68 | batch.append(data) 69 | 70 | return batch 71 | 72 | def close(self): 73 | self._stopper.set() 74 | 75 | # Delete all items in both queues. 76 | while not self._inputs.empty(): 77 | self._inputs.get() 78 | self._inputs.task_done() 79 | self._inputs.join() 80 | 81 | while not self._outputs.empty(): 82 | self._outputs.get() 83 | self._outputs.task_done() 84 | self._outputs.join() 85 | 86 | # Shut down all threads. 87 | for t in self._threads: 88 | t.join() 89 | -------------------------------------------------------------------------------- /lib/datasets/queue_test.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | import time 3 | 4 | from .queue import PreprocessQueue 5 | from .mnist import MNIST 6 | from ..pipeline import preprocess_pipeline_fixed 7 | from ..segmentation import slic_fixed, extract_features_fixed 8 | 9 | data = MNIST('data/mnist') 10 | segmentation_algorithm = slic_fixed( 11 | num_segments=100, compactness=5, max_iterations=10, sigma=0) 12 | feature_extraction_algorithm = extract_features_fixed([0, 1, 2]) 13 | preprocess_algorithm = preprocess_pipeline_fixed( 14 | segmentation_algorithm, feature_extraction_algorithm, 2) 15 | 16 | 17 | class QueueTest(TestCase): 18 | def test_dequeue(self): 19 | queue = PreprocessQueue( 20 | data.train, 21 | preprocess_algorithm, 22 | augment=True, 23 | batch_size=2, 24 | capacity=8, 25 | shuffle=False) 26 | 27 | batch = queue.dequeue() 28 | example = batch[0] 29 | features, adjs_dist, adjs_rad, label = example 30 | 31 | self.assertEqual(len(batch), 2) 32 | self.assertEqual(len(example), 4) 33 | self.assertEqual(features.shape[1], 4) 34 | self.assertEqual(len(adjs_dist), 3) 35 | self.assertEqual(len(adjs_rad), 3) 36 | self.assertEqual(label.shape[0], 10) 37 | 38 | # Ensure items in both queues before closing. 39 | time.sleep(5) 40 | queue.close() 41 | -------------------------------------------------------------------------------- /lib/graph/__init__.py: -------------------------------------------------------------------------------- 1 | from .coarsening import coarsen_adj 2 | from .distortion import (perm_features, filter_adj, filter_features, 3 | gray_color_threshold, degree_threshold, 4 | area_threshold) 5 | from .grid import grid_adj, grid_points, grid_mass 6 | from .spatial import receptive_fields, fill_features 7 | 8 | __all__ = [ 9 | 'coarsen_adj', 10 | 'perm_features', 11 | 'filter_adj', 12 | 'filter_features', 13 | 'gray_color_threshold', 14 | 'degree_threshold', 15 | 'area_threshold', 16 | 'grid_adj', 17 | 'grid_points', 18 | 'grid_mass', 19 | 'receptive_fields', 20 | 'fill_features', 21 | ] 22 | -------------------------------------------------------------------------------- /lib/graph/adjacency.py: -------------------------------------------------------------------------------- 1 | from __future__ import division 2 | 3 | import numpy as np 4 | import scipy.sparse as sp 5 | import numpy_groupies as npg 6 | 7 | 8 | def zero_one_scale_adj(adj, scale_invariance=False): 9 | """Normalize adjacency matrix to interval [0, 1]. Can choose between global 10 | or locale normalization.""" 11 | 12 | data = adj.data 13 | 14 | if data.size == 0: 15 | return adj 16 | 17 | if not scale_invariance: 18 | adj.data = (1 / data.max()) * data 19 | else: 20 | rows = adj.row 21 | multiplicator = 1 / npg.aggregate(rows, data, func='max') 22 | multiplicator = multiplicator[rows] 23 | adj.data = data * multiplicator 24 | return (adj + adj.transpose()) / 2 25 | 26 | return adj 27 | 28 | 29 | def invert_adj(adj, stddev=1): 30 | """Return (inverted) gaussian kernel representation.""" 31 | 32 | denominator = -2 * stddev * stddev 33 | adj.data = np.exp(adj.data / denominator) 34 | return adj 35 | 36 | 37 | def points_to_l2_adj(adj, points): 38 | """Build an embedded adjacency matrix based on points (y, x) of nodes.""" 39 | 40 | ys = points[:, :1].flatten() 41 | xs = points[:, 1:].flatten() 42 | 43 | rows = adj.row 44 | rows_ys = ys[rows] 45 | rows_xs = xs[rows] 46 | 47 | cols = adj.col 48 | cols_ys = ys[cols] 49 | cols_xs = xs[cols] 50 | 51 | vector_y = rows_ys - cols_ys 52 | vector_x = cols_xs - rows_xs 53 | 54 | dists = vector_y * vector_y + vector_x * vector_x 55 | rads = np.arctan2(vector_x, vector_y) 56 | # Adjust radians to lay in ]0, 2 * pi]. 57 | rads = np.where(rads > 0, rads, rads + 2 * np.pi) 58 | 59 | n = adj.shape[0] 60 | adj_dist = sp.coo_matrix((dists, (rows, cols)), (n, n)) 61 | adj_rad = sp.coo_matrix((rads, (rows, cols)), (n, n)) 62 | 63 | return adj_dist, adj_rad 64 | 65 | 66 | def points_to_adj(adj, points, scale_invariance=False, stddev=1): 67 | """Build an embedded adjacency matrix based on points (y, x) of nodes, so 68 | that nearer nodes have a smaller weight.""" 69 | 70 | adj_dist, adj_rad = points_to_l2_adj(adj, points) 71 | adj_dist = zero_one_scale_adj(adj_dist, scale_invariance) 72 | adj_dist = invert_adj(adj_dist, stddev) 73 | return adj_dist, adj_rad 74 | -------------------------------------------------------------------------------- /lib/graph/adjacency_test.py: -------------------------------------------------------------------------------- 1 | from __future__ import division 2 | 3 | from unittest import TestCase 4 | 5 | import numpy as np 6 | from numpy import pi as PI 7 | from numpy.testing import assert_equal 8 | import scipy.sparse as sp 9 | 10 | from .adjacency import (zero_one_scale_adj, invert_adj, points_to_l2_adj, 11 | points_to_adj) 12 | 13 | 14 | class GraphTest(TestCase): 15 | def test_zero_one_scale_adj(self): 16 | adj = [[0, 1, 0], [1, 0, 2], [0, 2, 0]] 17 | adj = sp.coo_matrix(adj) 18 | 19 | expected = [[0, 0.5, 0], [0.5, 0, 1], [0, 1, 0]] 20 | 21 | assert_equal(zero_one_scale_adj(adj).toarray(), expected) 22 | 23 | adj = [[0, 1, 0], [1, 0, 2], [0, 2, 0]] 24 | adj = sp.coo_matrix(adj) 25 | 26 | expected = [[0, 0.75, 0], [0.75, 0, 1], [0, 1, 0]] 27 | 28 | assert_equal( 29 | zero_one_scale_adj(adj, scale_invariance=True).toarray(), expected) 30 | 31 | def test_invert_adj(self): 32 | adj = [[0, 1, 0], [1, 0, 2], [0, 2, 0]] 33 | adj = sp.coo_matrix(adj) 34 | 35 | expected = [[0, np.exp(-1 / 2), 0], 36 | [np.exp(-1 / 2), 0, np.exp(-2 / 2)], 37 | [0, np.exp(-2 / 2), 0]] 38 | 39 | assert_equal(invert_adj(adj, stddev=1).toarray(), expected) 40 | 41 | adj = [[0, 1, 0], [1, 0, 2], [0, 2, 0]] 42 | adj = sp.coo_matrix(adj) 43 | 44 | expected = [[0, np.exp(-1 / 8), 0], 45 | [np.exp(-1 / 8), 0, np.exp(-2 / 8)], 46 | [0, np.exp(-2 / 8), 0]] 47 | 48 | assert_equal(invert_adj(adj, stddev=2).toarray(), expected) 49 | 50 | def test_points_to_l2_adj(self): 51 | points = np.array([[2, 2], [2, 4], [3, 2], [2, 1], [1, 2]]) 52 | adj = sp.coo_matrix([[0, 2, 1, 1, 1], [2, 0, 0, 0, 0], [1, 0, 0, 0, 0], 53 | [1, 0, 0, 0, 0], [1, 0, 0, 0, 0]]) 54 | adj_dist, adj_rad = points_to_l2_adj(adj, points) 55 | 56 | expected_dist = [[0, 4, 1, 1, 1], [4, 0, 0, 0, 0], [1, 0, 0, 0, 0], 57 | [1, 0, 0, 0, 0], [1, 0, 0, 0, 0]] 58 | expected_rad = [[0, 0.5 * PI, PI, 1.5 * PI, 2 * PI], 59 | [1.5 * PI, 0, 0, 0, 0], [2 * PI, 0, 0, 0, 0], 60 | [0.5 * PI, 0, 0, 0, 0], [PI, 0, 0, 0, 0]] 61 | 62 | assert_equal(adj_dist.toarray(), expected_dist) 63 | assert_equal(adj_rad.toarray(), expected_rad) 64 | 65 | def test_points_to_adj(self): 66 | points = np.array([[2, 2], [2, 4], [3, 2], [2, 1], [1, 2]]) 67 | adj = sp.coo_matrix([[0, 2, 1, 1, 1], [2, 0, 0, 0, 0], [1, 0, 0, 0, 0], 68 | [1, 0, 0, 0, 0], [1, 0, 0, 0, 0]]) 69 | adj_dist, adj_rad = points_to_adj(adj, points, stddev=2) 70 | 71 | expected_dist = [[ 72 | 0, np.exp(-1 / 8), np.exp(-0.25 / 8), np.exp(-0.25 / 8), 73 | np.exp(-0.25 / 8) 74 | ], [np.exp(-1 / 8), 0, 0, 0, 0], [np.exp(-0.25 / 8), 0, 0, 0, 0], 75 | [np.exp(-0.25 / 8), 0, 0, 0, 0], 76 | [np.exp(-0.25 / 8), 0, 0, 0, 0]] 77 | expected_rad = [[0, 0.5 * PI, PI, 1.5 * PI, 2 * PI], 78 | [1.5 * PI, 0, 0, 0, 0], [2 * PI, 0, 0, 0, 0], 79 | [0.5 * PI, 0, 0, 0, 0], [PI, 0, 0, 0, 0]] 80 | 81 | assert_equal(adj_dist.toarray(), expected_dist) 82 | assert_equal(adj_rad.toarray(), expected_rad) 83 | -------------------------------------------------------------------------------- /lib/graph/clustering.py: -------------------------------------------------------------------------------- 1 | from __future__ import division 2 | 3 | from six.moves import xrange 4 | 5 | import numpy as np 6 | # import numpy_groupies as npg 7 | import scipy.sparse as sp 8 | 9 | 10 | def normalized_cut(adj, rid=None): 11 | """Perform NormalizedCut on a given adjacency matrix. Return a cluster 12 | map indicating the resulting pairwise cluster.""" 13 | 14 | if rid is None: 15 | rid = np.random.permutation(np.arange(adj.shape[0])) 16 | 17 | n = adj.shape[0] 18 | cluster_map = np.zeros(n, np.int32) - 1 19 | clustercount = 0 20 | 21 | # Sort by row index. 22 | rows, cols, weights = sp.find(adj) 23 | perm = np.argsort(rows) 24 | rows = rows[perm] 25 | cols = cols[perm] 26 | weights = weights[perm] 27 | degree = np.array(adj.sum(axis=0)).flatten() 28 | 29 | # Get the beginning indices and the count of every row. 30 | _, rowstart, rowlength = np.unique( 31 | rows, return_index=True, return_counts=True) 32 | 33 | # Dismiss if adjacency contains just a single node. 34 | if rowstart.size == 0: 35 | return np.ones(1, np.int32) 36 | 37 | for r in xrange(n): 38 | # Iterate randomly. 39 | tid = rid[r] 40 | 41 | if cluster_map[tid] == -1: # Not already marked 42 | cluster_map[tid] = clustercount 43 | wmax = 0.0 44 | rs = rowstart[tid] 45 | bestneighbor = -1 46 | 47 | # Find best neighbor (Normalized Cut). 48 | for c in range(rowlength[tid]): 49 | nid = cols[rs + c] 50 | w = weights[rs + c] * (1.0 / degree[tid] + 1.0 / degree[nid] 51 | ) if cluster_map[nid] == -1 else 0.0 52 | if w > wmax: 53 | wmax = w 54 | bestneighbor = nid 55 | 56 | if bestneighbor > -1: 57 | cluster_map[bestneighbor] = clustercount 58 | clustercount += 1 59 | 60 | return cluster_map 61 | -------------------------------------------------------------------------------- /lib/graph/clustering_test.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | import numpy as np 4 | from numpy.testing import assert_equal 5 | import scipy.sparse as sp 6 | 7 | from .clustering import normalized_cut 8 | 9 | 10 | class ClusteringTest(TestCase): 11 | def test_normalized_cut_without_singletons(self): 12 | adj = [[0, 2, 1, 0], [2, 0, 0, 1], [1, 0, 0, 2], [0, 1, 2, 0]] 13 | adj = sp.coo_matrix(adj) 14 | rid = np.array([0, 1, 2, 3]) 15 | assert_equal(normalized_cut(adj, rid), [0, 0, 1, 1]) 16 | 17 | rid = np.array([3, 2, 1, 0]) 18 | assert_equal(normalized_cut(adj, rid), [1, 1, 0, 0]) 19 | 20 | def test_normalized_cut_with_singletons(self): 21 | adj = [[0, 3, 0, 2, 0], [3, 0, 2, 0, 0], [0, 2, 0, 3, 0], 22 | [2, 0, 3, 0, 1], [0, 0, 0, 1, 0]] 23 | adj = sp.coo_matrix(adj) 24 | rid = np.array([0, 1, 2, 3, 4]) 25 | assert_equal(normalized_cut(adj, rid), [0, 0, 1, 1, 2]) 26 | 27 | rid = np.array([4, 3, 2, 1, 0]) 28 | assert_equal(normalized_cut(adj, rid), [2, 1, 1, 0, 0]) 29 | 30 | rid = np.array([1, 0, 4, 2, 3]) 31 | assert_equal(normalized_cut(adj, rid), [0, 0, 2, 1, 1]) 32 | -------------------------------------------------------------------------------- /lib/graph/coarsening.py: -------------------------------------------------------------------------------- 1 | from __future__ import division 2 | from six.moves import xrange 3 | 4 | import numpy as np 5 | import numpy_groupies as npg 6 | import scipy.sparse as sp 7 | 8 | from .clustering import normalized_cut 9 | from .distortion import perm_adj 10 | from .adjacency import points_to_adj 11 | 12 | 13 | def coarsen_adj(adj, 14 | points, 15 | mass, 16 | levels, 17 | scale_invariance=False, 18 | stddev=1, 19 | rid=None): 20 | """Coarsen and permute a given adjacency matrix a number of levels deep, 21 | so that pairwise nodes in the same cluster are neighbored in the adjacency 22 | matrix. Returns `levels + 1` adjacency matrices (dist and rad) and the 23 | permutation of the first level to permute a given feature matrix.""" 24 | 25 | # Coarse adjacency a defined number of levels deep. 26 | adjs_dist, adjs_rad, cluster_maps = _coarsen_adj( 27 | adj, points, mass, levels, scale_invariance, stddev, rid) 28 | 29 | # Permutate adjacencies to a binary tree for an efficient O(n) pooling. 30 | perms = _compute_perms(cluster_maps) 31 | adjs_dist = [perm_adj(adjs_dist[i], perms[i]) for i in xrange(levels + 1)] 32 | adjs_rad = [perm_adj(adjs_rad[i], perms[i]) for i in xrange(levels + 1)] 33 | 34 | return adjs_dist, adjs_rad, perms[0] 35 | 36 | 37 | def _coarsen_adj(adj, 38 | points, 39 | mass, 40 | levels, 41 | scale_invariance=False, 42 | stddev=1, 43 | rid=None): 44 | 45 | adj_dist, adj_rad = points_to_adj(adj, points, scale_invariance, stddev) 46 | 47 | adjs_dist = [adj_dist] 48 | adjs_rad = [adj_rad] 49 | cluster_maps = [] 50 | 51 | for _ in xrange(levels): 52 | # Calculate normalized cut clustering. 53 | cluster_map = normalized_cut(adj_dist, rid) 54 | cluster_maps.append(cluster_map) 55 | 56 | # Coarsen adjacency. 57 | adj, points, mass = _coarsen_clustered_adj(adj, points, mass, 58 | cluster_map) 59 | # Compute to distance/radian adjacency. 60 | adj_dist, adj_rad = points_to_adj(adj, points, scale_invariance, 61 | stddev) 62 | adjs_dist.append(adj_dist) 63 | adjs_rad.append(adj_rad) 64 | 65 | # Iterate by degree at next iteration. 66 | if adj_dist.data.size > 0: 67 | degree = npg.aggregate(adj_dist.row, adj_dist.data, func='sum') 68 | rid = np.argsort(degree) 69 | else: 70 | rid = None 71 | 72 | return adjs_dist, adjs_rad, cluster_maps 73 | 74 | 75 | def _coarsen_clustered_adj(adj, points, mass, cluster_map): 76 | # Abort if adjacency contains only one or less points. 77 | if points.shape[0] < 2: 78 | return adj, points, mass 79 | 80 | rows = cluster_map[adj.row] 81 | cols = cluster_map[adj.col] 82 | 83 | n = cluster_map.max() + 1 84 | adj = sp.coo_matrix((adj.data, (rows, cols)), shape=(n, n)) 85 | adj.setdiag(0) 86 | adj.eliminate_zeros() 87 | adj = adj.tocsc().tocoo() # Sum up duplicate row/col entries. 88 | 89 | points_y = mass * points[:, :1].flatten() 90 | points_y = npg.aggregate(cluster_map, points_y, func='sum') 91 | 92 | points_x = mass * points[:, 1:].flatten() 93 | points_x = npg.aggregate(cluster_map, points_x, func='sum') 94 | 95 | mass = npg.aggregate(cluster_map, mass, func='sum') 96 | 97 | points_y = points_y / mass 98 | points_x = points_x / mass 99 | 100 | points_y = np.reshape(points_y, (-1, 1)) 101 | points_x = np.reshape(points_x, (-1, 1)) 102 | points = np.concatenate((points_y, points_x), axis=1) 103 | 104 | return adj.tocsc().tocoo(), points, mass 105 | 106 | 107 | def _compute_perms(cluster_maps): 108 | max_cluster = np.max(cluster_maps[-1]) + 1 109 | n = max_cluster 110 | perm = np.arange(n) 111 | perms = [perm] 112 | 113 | for i in xrange(len(cluster_maps) - 1, -1, -1): # Iterate backwards 114 | cluster_map = cluster_maps[i] 115 | 116 | # Add single fake nodes to end of cluster map. 117 | idx, counts = np.unique(cluster_map, return_counts=True) 118 | singles = idx[np.where(counts == 1)] 119 | 120 | # Add double fake nodes to end of cluster map. 121 | max_cluster = (cluster_map.size + singles.size) // 2 122 | doubles = np.arange(max_cluster, n).repeat(2) 123 | 124 | cluster_map = np.concatenate((cluster_map, singles, doubles)) 125 | 126 | # Permutate cluster_map. 127 | cluster_map = np.argsort(perm)[cluster_map] 128 | rid = np.argsort(cluster_map) 129 | n *= 2 130 | perm = np.arange(n)[rid] 131 | perms.append(perm) 132 | 133 | return perms[::-1] 134 | -------------------------------------------------------------------------------- /lib/graph/distortion.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import scipy.sparse as sp 3 | import numpy_groupies as npg 4 | 5 | 6 | def perm_adj(adj, perm): 7 | """Permute an adjacency matrix given a permutation. The permutation must 8 | contain all existing nodes in an arbitrary order and can contain fake 9 | nodes.""" 10 | 11 | n = perm.shape[0] 12 | sorted_perm = np.argsort(perm) 13 | row = sorted_perm[adj.row] 14 | col = sorted_perm[adj.col] 15 | return sp.coo_matrix((adj.data, (row, col)), (n, n)) 16 | 17 | 18 | def perm_features(features, perm): 19 | """Permute an feature matrix given a permutation. The permutation must 20 | contain all existing nodes in an arbitrary order and can contain fake 21 | nodes.""" 22 | 23 | n, k = features.shape 24 | num_fake_nodes = perm.shape[0] - n 25 | zeros = np.zeros((num_fake_nodes, k), features.dtype) 26 | features = np.concatenate((features, zeros), axis=0) 27 | return features[perm] 28 | 29 | 30 | def filter_adj(adj, nodes): 31 | """Filters a given adjacency matrix by its given nodes indices.""" 32 | 33 | # Filter by rows. 34 | in1d = np.in1d(adj.row, nodes) 35 | rows = adj.row[in1d] 36 | cols = adj.col[in1d] 37 | data = adj.data[in1d] 38 | 39 | # Filter by cols. 40 | in1d = np.in1d(cols, nodes) 41 | rows = rows[in1d] 42 | cols = cols[in1d] 43 | data = data[in1d] 44 | 45 | # Remap indices to new range. 46 | rows = np.unique(rows, return_inverse=True)[1] 47 | cols = np.unique(cols, return_inverse=True)[1] 48 | 49 | n = nodes.size 50 | return sp.coo_matrix((data, (rows, cols)), (n, n)) 51 | 52 | 53 | def filter_features(features, nodes): 54 | """Filters a feature matrix by its given nodes indices.""" 55 | return features[nodes] 56 | 57 | 58 | def gray_color_threshold(adj, features, k): 59 | """Node elimination by a color threshold `k`.""" 60 | gray = features[:, :1] 61 | return np.where(gray >= k)[0] 62 | 63 | 64 | def degree_threshold(adj, features, k): 65 | """Node elimination by a degree threshold `k`.""" 66 | # Adjacency must contain one in every entry. 67 | degree = npg.aggregate(adj.row, adj.data, func='sum') 68 | return np.where(degree <= k)[0] 69 | 70 | 71 | def area_threshold(adj, features, k, idx=1): 72 | """Node elimination by an area threshold `k`. `idx` points to the index of 73 | the area feature in the feature matrix.""" 74 | 75 | area = features[:, idx] 76 | return np.where(area <= k)[0] 77 | -------------------------------------------------------------------------------- /lib/graph/distortion_test.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | import numpy as np 4 | from numpy.testing import assert_equal 5 | import scipy.sparse as sp 6 | 7 | from .distortion import (perm_adj, perm_features, filter_adj, filter_features, 8 | gray_color_threshold, degree_threshold, 9 | area_threshold) 10 | 11 | 12 | class DistortionTest(TestCase): 13 | def test_perm_adj(self): 14 | adj = [[0, 2, 1, 0], [2, 0, 0, 1], [1, 0, 0, 2], [0, 1, 2, 0]] 15 | adj = sp.coo_matrix(adj) 16 | 17 | perm = np.array([2, 1, 3, 0]) 18 | expected = [[0, 0, 2, 1], [0, 0, 1, 2], [2, 1, 0, 0], [1, 2, 0, 0]] 19 | assert_equal(perm_adj(adj, perm).toarray(), expected) 20 | 21 | # Add fake nodes. 22 | perm = np.array([3, 2, 0, 4, 1, 5]) 23 | expected = [[0, 2, 0, 0, 1, 0], [2, 0, 1, 0, 0, 0], [0, 1, 0, 0, 2, 0], 24 | [0, 0, 0, 0, 0, 0], [1, 0, 2, 0, 0, 0], [0, 0, 0, 0, 0, 0]] 25 | assert_equal(perm_adj(adj, perm).toarray(), expected) 26 | 27 | def test_perm_features(self): 28 | features = np.array([[1, 2], [3, 4], [5, 6], [7, 8]]) 29 | 30 | perm = np.array([2, 1, 3, 0]) 31 | expected = [[5, 6], [3, 4], [7, 8], [1, 2]] 32 | assert_equal(perm_features(features, perm), expected) 33 | 34 | # Add fake nodes. 35 | perm = np.array([3, 2, 0, 4, 1, 5]) 36 | expected = [[7, 8], [5, 6], [1, 2], [0, 0], [3, 4], [0, 0]] 37 | assert_equal(perm_features(features, perm), expected) 38 | 39 | def test_filter_adj(self): 40 | adj = [[0, 2, 1, 0], [2, 0, 0, 1], [1, 0, 0, 2], [0, 1, 2, 0]] 41 | adj = sp.coo_matrix(adj) 42 | 43 | nodes = np.array([0, 1, 3]) 44 | expected = [[0, 2, 0], [2, 0, 1], [0, 1, 0]] 45 | assert_equal(filter_adj(adj, nodes).toarray(), expected) 46 | 47 | def test_filter_features(self): 48 | features = np.array([[1, 2], [3, 4], [5, 6], [7, 8]]) 49 | 50 | nodes = np.array([1, 3]) 51 | expected = [[3, 4], [7, 8]] 52 | assert_equal(filter_features(features, nodes), expected) 53 | 54 | def test_gray_color_threshold(self): 55 | features = np.array([[1, 2], [3, 4], [5, 6], [7, 8]]) 56 | 57 | nodes = gray_color_threshold(None, features, 3) 58 | expected = [1, 2, 3] 59 | assert_equal(nodes, expected) 60 | 61 | def test_degree_threshold(self): 62 | adj = [[0, 1, 1, 1, 1], [1, 0, 1, 0, 0], [1, 1, 0, 1, 0], 63 | [1, 0, 1, 0, 1], [1, 0, 0, 1, 0]] 64 | adj = sp.coo_matrix(adj) 65 | 66 | nodes = degree_threshold(adj, None, 3) 67 | expected = [1, 2, 3, 4] 68 | assert_equal(nodes, expected) 69 | 70 | def test_area_threshold(self): 71 | features = np.array([[1, 2], [3, 4], [5, 6], [7, 8]]) 72 | 73 | nodes = area_threshold(None, features, 4, idx=1) 74 | expected = [0, 1] 75 | assert_equal(nodes, expected) 76 | -------------------------------------------------------------------------------- /lib/graph/grid.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import scipy.sparse as sp 3 | 4 | from .distortion import filter_adj 5 | 6 | 7 | def grid_adj(shape, connectivity=4, dtype=np.float32): 8 | """Return adjacency matrix of a regular grid.""" 9 | 10 | assert connectivity == 4 or connectivity == 8 11 | 12 | h, w = shape 13 | 14 | if connectivity == 4: 15 | filt = [-w - 2, -1, 1, w + 2] 16 | else: 17 | filt = [-w - 3, -w - 2, -w - 1, -1, 1, w + 1, w + 2, w + 3] 18 | 19 | # Build basic rows and cols with +1 padding on all sides. 20 | n = (h + 1) * (w + 2) - 1 21 | rows = np.arange(w + 3, n).repeat(connectivity) 22 | rows = np.reshape(rows, (-1, connectivity)) 23 | cols = rows + filt 24 | rows = rows.flatten() 25 | cols = cols.flatten() 26 | 27 | data = np.ones_like(rows, dtype=np.uint8) 28 | n = (h + 2) * (w + 2) 29 | adj = sp.coo_matrix((data, (rows, cols)), (n, n)) 30 | 31 | # Compute filter nodes. 32 | rows = np.arange(w + 2, h * (w + 3), w + 2).repeat(w) 33 | rows = np.reshape(rows, (-1, w)) 34 | cols = np.arange(1, w + 1) 35 | nodes = (rows + cols).flatten() 36 | 37 | return filter_adj(adj, nodes) 38 | 39 | 40 | def grid_points(shape, dtype=np.float32): 41 | """Return the grid points of a given shape with distance `1`.""" 42 | 43 | x, y = np.meshgrid(np.arange(shape[1]), np.arange(shape[0])) 44 | y = y.flatten() 45 | x = x.flatten() 46 | 47 | z = np.empty((shape[0] * shape[1], 2), dtype) 48 | z[:, 0] = y 49 | z[:, 1] = x 50 | return z 51 | 52 | 53 | def grid_mass(shape, dtype=np.float32): 54 | """Return the mass of grid points of a given shape with distance `1`.""" 55 | return np.ones(shape[0] * shape[1], dtype) 56 | -------------------------------------------------------------------------------- /lib/graph/grid_test.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | from numpy.testing import assert_equal 4 | 5 | from .grid import grid_adj, grid_points, grid_mass 6 | 7 | 8 | class GridTest(TestCase): 9 | def test_grid_points(self): 10 | points = grid_points((3, 2)) 11 | expected = [[0, 0], [0, 1], [1, 0], [1, 1], [2, 0], [2, 1]] 12 | assert_equal(points, expected) 13 | 14 | def test_grid_mass(self): 15 | mass = grid_mass((3, 2)) 16 | expected = [1, 1, 1, 1, 1, 1] 17 | assert_equal(mass, expected) 18 | 19 | def test_grid_adj_connectivity_4(self): 20 | adj = grid_adj((3, 2), connectivity=4) 21 | expected = [[0, 1, 1, 0, 0, 0], [1, 0, 0, 1, 0, 0], [1, 0, 0, 1, 1, 0], 22 | [0, 1, 1, 0, 0, 1], [0, 0, 1, 0, 0, 1], [0, 0, 0, 1, 1, 0]] 23 | 24 | assert_equal(adj.toarray(), expected) 25 | 26 | def test_grid_adj_connectivity_8(self): 27 | adj = grid_adj((3, 2), connectivity=8) 28 | expected = [[0, 1, 1, 1, 0, 0], [1, 0, 1, 1, 0, 0], [1, 1, 0, 1, 1, 1], 29 | [1, 1, 1, 0, 1, 1], [0, 0, 1, 1, 0, 1], [0, 0, 1, 1, 1, 0]] 30 | 31 | assert_equal(adj.toarray(), expected) 32 | -------------------------------------------------------------------------------- /lib/graph/spatial.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def node_selection(points, size, stride=1, delta=1): 5 | """Select nodes of an adjacency matrix given their coordinates.""" 6 | 7 | # Find coordinate max values. 8 | y_min = points[:, :1].min() 9 | x_min = points[:, -1:].min() 10 | x_max = points[:, -1:].max() 11 | w_max = x_max - x_min 12 | 13 | # Translate points to min zero. 14 | points = points - np.array([y_min, x_min]) 15 | 16 | # Scale y-coordinates to natural numbers. 17 | points[:, :1] = w_max * np.floor(points[:, :1] / delta) 18 | 19 | # Sort points. 20 | points = points.sum(axis=1) 21 | order = np.argsort(points) 22 | 23 | # Stride and slice points. 24 | idx = np.arange(np.min([size * stride, order.shape[0]]), step=stride) 25 | 26 | # Fill the rest of the nodes with id -1 until we reach the given size. 27 | fake = -np.ones(np.max([size - idx.shape[0], 0]), dtype=np.int64) 28 | return np.concatenate([order[idx], fake], axis=0) 29 | 30 | 31 | def neighborhood_selection(idx, points, adj, size): 32 | """Select a neighborhood given an adjacency matrix and their node points. 33 | """ 34 | 35 | if idx == -1: 36 | return np.ones((size), np.int64) * adj.shape[0] 37 | 38 | nodes = np.array([idx]) 39 | current_nodes = np.array([idx]) 40 | 41 | while nodes.shape[0] < size and nodes.shape[0] < adj.shape[0]: 42 | # Calculate all neighbors of current iteration. 43 | neighbor_idx = np.where(np.in1d(adj.row, current_nodes))[0] 44 | neighbor_col = adj.col[neighbor_idx] 45 | neighbor_col = np.unique(neighbor_col) 46 | filter_idx = np.where(np.in1d(neighbor_col, nodes, invert=True))[0] 47 | neighbor_col = neighbor_col[filter_idx] 48 | 49 | # Calculate vectors. 50 | vectors_y = points[idx, 0] - points[neighbor_col, 0] 51 | vectors_x = points[neighbor_col, 1] - points[idx, 1] 52 | rads = np.arctan2(vectors_x, vectors_y) 53 | rads = np.where(rads > 0, rads, rads + 2 * np.pi) 54 | 55 | # Sort by radians. 56 | order = np.argsort(rads) 57 | neighbor_col = neighbor_col[order] 58 | 59 | # Append to nodes and iterate over current neighbors in next step. 60 | nodes = np.concatenate([nodes, neighbor_col], axis=0) 61 | current_nodes = neighbor_col 62 | 63 | # Slice or append fake nodes with value N. 64 | N = adj.shape[0] 65 | nodes = nodes[:size] 66 | fake = N * np.ones(np.max([size - nodes.shape[0], 0]), dtype=np.int64) 67 | return np.concatenate([nodes, fake], axis=0) 68 | 69 | 70 | def receptive_fields(points, 71 | adj, 72 | node_size, 73 | node_stride, 74 | neighborhood_size, 75 | delta=1): 76 | """Create receptive fields for a graph.""" 77 | 78 | # Compute node selection. 79 | nodes = node_selection(points, node_size, node_stride, delta) 80 | 81 | # Stack receptive fields of node selection vertically. 82 | return np.vstack([ 83 | neighborhood_selection(node, points, adj, neighborhood_size) 84 | for node in nodes 85 | ]) 86 | 87 | 88 | def fill_features(receptive_fields, features): 89 | """Fill receptive field with features.""" 90 | 91 | # Append zero features for fake nodes. 92 | zero = np.reshape(np.zeros_like(features[0]), (1, -1)) 93 | features = np.concatenate([features, zero], axis=0) 94 | 95 | # Fill features. 96 | node_size, neighborhood_size = receptive_fields.shape 97 | flat = receptive_fields.flatten() 98 | return np.reshape(features[flat], (node_size, neighborhood_size, -1)) 99 | -------------------------------------------------------------------------------- /lib/graph/spatial_test.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | import numpy as np 4 | from numpy.testing import assert_equal 5 | import scipy.sparse as sp 6 | 7 | from .spatial import (node_selection, neighborhood_selection, receptive_fields, 8 | fill_features) 9 | 10 | 11 | class SpatialTest(TestCase): 12 | def test_node_selection(self): 13 | points = np.array([[2, 4], [0, 1], [2.1, 2.5], [0.2, 3], [0.1, 4]]) 14 | 15 | assert_equal(node_selection(points, size=6), [1, 3, 4, 2, 0, -1]) 16 | assert_equal(node_selection(points, size=2), [1, 3]) 17 | assert_equal(node_selection(points, size=3, stride=3), [1, 2, -1]) 18 | assert_equal(node_selection(points, size=4, stride=2), [1, 4, 0, -1]) 19 | assert_equal( 20 | node_selection(points, size=6, delta=3), [1, 2, 3, 0, 4, -1]) 21 | 22 | def test_neighborhood_selection(self): 23 | points = np.array([[0, 0], [1, 0], [0, 1], [-1, 0], [0, -1]]) 24 | 25 | adj = [[0, 1, 1, 1, 1], [1, 0, 0, 0, 0], [1, 0, 0, 0, 0], 26 | [1, 0, 0, 0, 0], [1, 0, 0, 0, 0]] 27 | adj = sp.coo_matrix(adj) 28 | 29 | assert_equal( 30 | neighborhood_selection(0, points, adj, size=5), [0, 2, 1, 4, 3]) 31 | assert_equal(neighborhood_selection(0, points, adj, size=3), [0, 2, 1]) 32 | assert_equal( 33 | neighborhood_selection(0, points, adj, size=7), 34 | [0, 2, 1, 4, 3, 5, 5]) 35 | 36 | assert_equal( 37 | neighborhood_selection(1, points, adj, size=5), [1, 0, 2, 4, 3]) 38 | 39 | assert_equal(neighborhood_selection(-1, points, adj, size=2), [5, 5]) 40 | 41 | def test_receptive_fields(self): 42 | points = np.array([ 43 | [0, 0], 44 | [1, 0], 45 | [0, 1], 46 | [-1, 0], 47 | [0, -1], 48 | ]) 49 | 50 | adj = sp.coo_matrix([ 51 | [0, 1, 1, 1, 1], 52 | [1, 0, 0, 0, 0], 53 | [1, 0, 0, 0, 0], 54 | [1, 0, 0, 0, 0], 55 | [1, 0, 0, 0, 0], 56 | ]) 57 | 58 | fields = receptive_fields( 59 | points, adj, node_size=3, node_stride=2, neighborhood_size=4) 60 | 61 | excepted = [[3, 0, 2, 1], [0, 2, 1, 4], [1, 0, 2, 4]] 62 | 63 | assert_equal(fields, excepted) 64 | 65 | def test_fill_features(self): 66 | receptive_fields = np.array([ 67 | [3, 0, 2, 1], 68 | [0, 2, 1, 4], 69 | [1, 0, 2, 5], 70 | ]) 71 | 72 | features = np.array([ 73 | [0, 1, 2], 74 | [1, 2, 3], 75 | [2, 3, 4], 76 | [3, 4, 5], 77 | [4, 5, 6], 78 | ]) 79 | 80 | block = fill_features(receptive_fields, features) 81 | 82 | expected = [ 83 | [[3, 4, 5], [0, 1, 2], [2, 3, 4], [1, 2, 3]], 84 | [[0, 1, 2], [2, 3, 4], [1, 2, 3], [4, 5, 6]], 85 | [[1, 2, 3], [0, 1, 2], [2, 3, 4], [0, 0, 0]], 86 | ] 87 | 88 | assert_equal(block, expected) 89 | -------------------------------------------------------------------------------- /lib/layer/__init__.py: -------------------------------------------------------------------------------- 1 | from .conv2d import Conv2d 2 | from .fc import FC 3 | from .chebyshev_gcnn import ChebyshevGCNN 4 | from .gcnn import GCNN 5 | from .embedded_gcnn import EmbeddedGCNN 6 | from .spatial import SpatialCNN 7 | from .max_pool import MaxPool 8 | from .average_pool import AveragePool 9 | from .fire import Fire 10 | from .image_augment import ImageAugment 11 | 12 | __all__ = [ 13 | 'Conv2d', 14 | 'FC', 15 | 'ChebyshevGCNN', 16 | 'GCNN', 17 | 'EmbeddedGCNN', 18 | 'SpatialCNN', 19 | 'MaxPool', 20 | 'AveragePool', 21 | 'Fire', 22 | 'ImageAugment', 23 | ] 24 | -------------------------------------------------------------------------------- /lib/layer/average_pool.py: -------------------------------------------------------------------------------- 1 | from six.moves import xrange 2 | 3 | import tensorflow as tf 4 | 5 | from .layer import Layer 6 | 7 | 8 | class AveragePool(Layer): 9 | def __init__(self, **kwargs): 10 | super(AveragePool, self).__init__(**kwargs) 11 | 12 | def _call(self, inputs): 13 | if isinstance(inputs, list): 14 | batch_size = len(inputs) 15 | outputs = [] 16 | for i in xrange(batch_size): 17 | output = tf.reduce_mean(inputs[i], axis=0) 18 | outputs.append(output) 19 | return tf.stack(outputs) 20 | else: 21 | outputs = tf.reduce_mean(inputs, axis=[1, 2]) 22 | return tf.squeeze(outputs) 23 | -------------------------------------------------------------------------------- /lib/layer/average_pool_test.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | from .average_pool import AveragePool 4 | 5 | 6 | class AveragePoolTest(tf.test.TestCase): 7 | def test_init(self): 8 | layer = AveragePool() 9 | self.assertEqual(layer.name, 'averagepool_1') 10 | 11 | def test_call(self): 12 | layer = AveragePool(name='call') 13 | input_1 = [[1, 2], [3, 4], [5, 6], [7, 8]] 14 | input_2 = [[9, 10], [11, 12], [13, 14], [15, 16]] 15 | input_1 = tf.constant(input_1, dtype=tf.float32) 16 | input_2 = tf.constant(input_2, dtype=tf.float32) 17 | inputs = [input_1, input_2] 18 | outputs = layer(inputs) 19 | 20 | expected = [[4, 5], [12, 13]] 21 | 22 | with self.test_session(): 23 | # Average pooling converts lists to tensors. 24 | self.assertAllEqual(outputs.eval(), expected) 25 | 26 | def test_call_with_tensor(self): 27 | layer = AveragePool(name='call_with_tensor') 28 | inputs = tf.constant([[[ 29 | [1, 2], 30 | [3, 4], 31 | ], [ 32 | [5, 6], 33 | [7, 8], 34 | ]], [[ 35 | [1, 2], 36 | [3, 4], 37 | ], [ 38 | [5, 6], 39 | [7, 8], 40 | ]]]) 41 | 42 | outputs = layer(inputs) 43 | 44 | expected = [[4, 5], [4, 5]] 45 | 46 | with self.test_session(): 47 | self.assertAllEqual(outputs.eval(), expected) 48 | -------------------------------------------------------------------------------- /lib/layer/chebyshev_gcnn.py: -------------------------------------------------------------------------------- 1 | from six.moves import xrange 2 | 3 | import tensorflow as tf 4 | 5 | from .var_layer import VarLayer 6 | from ..tf import rescaled_laplacian 7 | 8 | 9 | def conv(features, adj, weights): 10 | K = weights.get_shape()[0].value - 1 11 | 12 | # Create and rescale normalized laplacian. 13 | lap = rescaled_laplacian(adj) 14 | 15 | Tx_0 = features 16 | output = tf.matmul(Tx_0, weights[0]) 17 | 18 | if K > 0: 19 | Tx_1 = tf.sparse_tensor_dense_matmul(lap, features) 20 | output += tf.matmul(Tx_1, weights[1]) 21 | 22 | for k in xrange(2, K + 1): 23 | Tx_2 = 2 * tf.sparse_tensor_dense_matmul(lap, Tx_1) - Tx_0 24 | output += tf.matmul(Tx_2, weights[k]) 25 | 26 | Tx_0, Tx_1 = Tx_1, Tx_2 27 | 28 | return output 29 | 30 | 31 | class ChebyshevGCNN(VarLayer): 32 | def __init__(self, in_channels, out_channels, adjs, degree, **kwargs): 33 | 34 | self.adjs = adjs 35 | 36 | super(ChebyshevGCNN, self).__init__( 37 | weight_shape=[degree + 1, in_channels, out_channels], 38 | bias_shape=[out_channels], 39 | **kwargs) 40 | 41 | def _call(self, inputs): 42 | batch_size = len(inputs) 43 | outputs = [] 44 | 45 | for i in xrange(batch_size): 46 | output = conv(inputs[i], self.adjs[i], self.vars['weights']) 47 | 48 | if self.bias: 49 | output = tf.nn.bias_add(output, self.vars['bias']) 50 | 51 | output = self.act(output) 52 | outputs.append(output) 53 | 54 | return outputs 55 | -------------------------------------------------------------------------------- /lib/layer/chebyshev_gcnn_test.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import scipy.sparse as sp 3 | import tensorflow as tf 4 | 5 | from .chebyshev_gcnn import conv, ChebyshevGCNN 6 | from ..tf.convert import sparse_to_tensor 7 | from ..tf.laplacian import rescaled_laplacian 8 | 9 | 10 | class ChebyshevGCNNTest(tf.test.TestCase): 11 | def test_conv_K2(self): 12 | adj = [[0, 1, 0], [1, 0, 2], [0, 2, 0]] 13 | adj = sp.coo_matrix(adj, dtype=np.float32) 14 | adj = sparse_to_tensor(adj) 15 | lap = rescaled_laplacian(adj) 16 | 17 | features = [[1, 2], [3, 4], [5, 6]] 18 | features = tf.constant(features, dtype=tf.float32) 19 | 20 | weights = [[[0.3], [0.7]], [[0.4], [0.6]], [[0.8], [0.2]]] 21 | weights = tf.constant(weights, dtype=tf.float32) 22 | 23 | Tx_0 = features 24 | expected = tf.matmul(Tx_0, weights[0]) 25 | Tx_1 = tf.sparse_tensor_dense_matmul(lap, features) 26 | expected = tf.add(tf.matmul(Tx_1, weights[1]), expected) 27 | Tx_2 = 2 * tf.sparse_tensor_dense_matmul(lap, Tx_1) - Tx_0 28 | expected = tf.add(tf.matmul(Tx_2, weights[2]), expected) 29 | 30 | with self.test_session(): 31 | self.assertAllEqual( 32 | conv(features, adj, weights).eval(), expected.eval()) 33 | 34 | def test_init(self): 35 | layer = ChebyshevGCNN(1, 2, adjs=None, degree=3) 36 | self.assertEqual(layer.name, 'chebyshevgcnn_1') 37 | self.assertIsNone(layer.adjs) 38 | self.assertEqual(layer.vars['weights'].get_shape(), [4, 1, 2]) 39 | self.assertEqual(layer.vars['bias'].get_shape(), [2]) 40 | 41 | def test_call(self): 42 | adj = [[0, 1, 0], [1, 0, 2], [0, 2, 0]] 43 | adj = sp.coo_matrix(adj, dtype=np.float32) 44 | adj = sparse_to_tensor(adj) 45 | 46 | layer = ChebyshevGCNN(2, 3, [adj, adj], degree=3, name='call') 47 | 48 | input_1 = [[1, 2], [3, 4], [5, 6]] 49 | input_1 = tf.constant(input_1, dtype=tf.float32) 50 | input_2 = [[7, 8], [9, 10], [11, 12]] 51 | input_2 = tf.constant(input_2, dtype=tf.float32) 52 | inputs = [input_1, input_2] 53 | outputs = layer(inputs) 54 | 55 | expected_1 = conv(input_1, adj, layer.vars['weights']) 56 | expected_1 = tf.nn.bias_add(expected_1, layer.vars['bias']) 57 | expected_1 = tf.nn.relu(expected_1) 58 | 59 | expected_2 = conv(input_2, adj, layer.vars['weights']) 60 | expected_2 = tf.nn.bias_add(expected_2, layer.vars['bias']) 61 | expected_2 = tf.nn.relu(expected_2) 62 | 63 | with self.test_session() as sess: 64 | sess.run(tf.global_variables_initializer()) 65 | 66 | self.assertEqual(len(outputs), 2) 67 | self.assertEqual(outputs[0].eval().shape, (3, 3)) 68 | self.assertEqual(outputs[1].eval().shape, (3, 3)) 69 | self.assertAllEqual(outputs[0].eval(), expected_1.eval()) 70 | 71 | def test_call_without_bias(self): 72 | adj = [[0, 1, 0], [1, 0, 2], [0, 2, 0]] 73 | adj = sp.coo_matrix(adj, dtype=np.float32) 74 | adj = sparse_to_tensor(adj) 75 | 76 | layer = ChebyshevGCNN( 77 | 2, 3, [adj, adj], degree=3, bias=False, name='call_without_bias') 78 | 79 | input_1 = [[1, 2], [3, 4], [5, 6]] 80 | input_1 = tf.constant(input_1, dtype=tf.float32) 81 | input_2 = [[7, 8], [9, 10], [11, 12]] 82 | input_2 = tf.constant(input_2, dtype=tf.float32) 83 | inputs = [input_1, input_2] 84 | outputs = layer(inputs) 85 | 86 | expected_1 = conv(input_1, adj, layer.vars['weights']) 87 | expected_1 = tf.nn.relu(expected_1) 88 | 89 | expected_2 = conv(input_2, adj, layer.vars['weights']) 90 | expected_2 = tf.nn.relu(expected_2) 91 | 92 | with self.test_session() as sess: 93 | sess.run(tf.global_variables_initializer()) 94 | 95 | self.assertEqual(len(outputs), 2) 96 | self.assertEqual(outputs[0].eval().shape, (3, 3)) 97 | self.assertEqual(outputs[1].eval().shape, (3, 3)) 98 | self.assertAllEqual(outputs[0].eval(), expected_1.eval()) 99 | self.assertAllEqual(outputs[1].eval(), expected_2.eval()) 100 | self.assertAllEqual(outputs[1].eval(), expected_2.eval()) 101 | -------------------------------------------------------------------------------- /lib/layer/conv2d.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | from .var_layer import VarLayer 4 | 5 | 6 | def conv(inputs, weights, stride): 7 | return tf.nn.conv2d( 8 | inputs, weights, strides=[1, stride, stride, 1], padding='SAME') 9 | 10 | 11 | class Conv2d(VarLayer): 12 | def __init__(self, 13 | in_channels, 14 | out_channels, 15 | size=3, 16 | stride=1, 17 | dropout=None, 18 | **kwargs): 19 | self.stride = stride 20 | self.dropout = dropout 21 | 22 | super(Conv2d, self).__init__( 23 | weight_shape=[size, size, in_channels, out_channels], 24 | bias_shape=[out_channels], 25 | **kwargs) 26 | 27 | def _call(self, inputs): 28 | if self.dropout is not None: 29 | outputs = tf.nn.dropout(inputs, 1 - self.dropout) 30 | 31 | outputs = conv(inputs, self.vars['weights'], self.stride) 32 | 33 | if self.bias: 34 | outputs = tf.nn.bias_add(outputs, self.vars['bias']) 35 | 36 | return self.act(outputs) 37 | -------------------------------------------------------------------------------- /lib/layer/conv2d_test.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | from .conv2d import conv, Conv2d 4 | 5 | 6 | class Conv2dTest(tf.test.TestCase): 7 | def test_conv(self): 8 | image = [[1, 2, 3], [4, 5, 6], [7, 8, 9]] 9 | inputs = tf.constant(image, tf.float32) 10 | inputs = tf.reshape(inputs, [1, 3, 3, 1]) 11 | 12 | weights = [[1, 2, 3], [4, 5, 6], [7, 8, 9]] 13 | weights = tf.constant(weights, tf.float32) 14 | weights = tf.reshape(weights, [3, 3, 1, 1]) 15 | 16 | outputs = conv(inputs, weights, stride=1) 17 | 18 | expected = [[ 19 | [ 20 | [1 * 5 + 2 * 6 + 4 * 8 + 5 * 9], 21 | [1 * 4 + 2 * 5 + 3 * 6 + 4 * 7 + 5 * 8 + 6 * 9], 22 | [2 * 4 + 3 * 5 + 5 * 7 + 6 * 8], 23 | ], 24 | [ 25 | [1 * 2 + 2 * 3 + 4 * 5 + 5 * 6 + 7 * 8 + 8 * 9], 26 | [1 + 4 + 9 + 16 + 25 + 36 + 49 + 64 + 81], 27 | [2 * 1 + 3 * 2 + 5 * 4 + 6 * 5 + 8 * 7 + 9 * 8], 28 | ], 29 | [ 30 | [4 * 2 + 5 * 3 + 7 * 5 + 8 * 6], 31 | [4 * 1 + 5 * 2 + 6 * 3 + 7 * 4 + 5 * 8 + 9 * 6], 32 | [5 * 1 + 6 * 2 + 8 * 4 + 9 * 5], 33 | ], 34 | ]] 35 | 36 | with self.test_session(): 37 | self.assertEqual(outputs.eval().shape, (1, 3, 3, 1)) 38 | self.assertAllEqual(outputs.eval(), expected) 39 | 40 | def test_init(self): 41 | layer = Conv2d(1, 2) 42 | self.assertEqual(layer.name, 'conv2d_1') 43 | self.assertIsNone(layer.dropout) 44 | self.assertEqual(layer.stride, 1) 45 | self.assertEqual(layer.vars['weights'].get_shape(), [3, 3, 1, 2]) 46 | self.assertEqual(layer.vars['bias'].get_shape(), [2]) 47 | 48 | layer = Conv2d(2, 3, size=5, stride=4, dropout=0.5) 49 | self.assertEqual(layer.name, 'conv2d_2') 50 | self.assertEqual(layer.dropout, 0.5) 51 | self.assertEqual(layer.stride, 4) 52 | self.assertEqual(layer.vars['weights'].get_shape(), [5, 5, 2, 3]) 53 | self.assertEqual(layer.vars['bias'].get_shape(), [3]) 54 | 55 | def test_call(self): 56 | layer = Conv2d(1, 2, name='call') 57 | 58 | image = [[1, 2, 3], [4, 5, 6], [7, 8, 9]] 59 | inputs = tf.constant(image, tf.float32) 60 | inputs = tf.reshape(inputs, [1, 3, 3, 1]) 61 | 62 | outputs = layer(inputs) 63 | 64 | expected = conv(inputs, layer.vars['weights'], stride=1) 65 | expected = tf.nn.bias_add(expected, layer.vars['bias']) 66 | expected = tf.nn.relu(expected) 67 | 68 | with self.test_session() as sess: 69 | sess.run(tf.global_variables_initializer()) 70 | 71 | self.assertEqual(outputs.eval().shape, (1, 3, 3, 2)) 72 | self.assertAllEqual(outputs.eval(), expected.eval()) 73 | 74 | def test_call_without_bias(self): 75 | layer = Conv2d(1, 2, bias=False, name='call_wihtout_bias') 76 | 77 | image = [[1, 2, 3], [4, 5, 6], [7, 8, 9]] 78 | inputs = tf.constant(image, tf.float32) 79 | inputs = tf.reshape(inputs, [1, 3, 3, 1]) 80 | 81 | outputs = layer(inputs) 82 | 83 | expected = conv(inputs, layer.vars['weights'], stride=1) 84 | expected = tf.nn.relu(expected) 85 | 86 | with self.test_session() as sess: 87 | sess.run(tf.global_variables_initializer()) 88 | 89 | self.assertEqual(outputs.eval().shape, (1, 3, 3, 2)) 90 | self.assertAllEqual(outputs.eval(), expected.eval()) 91 | 92 | def test_call_with_dropout(self): 93 | layer = Conv2d(1, 2, dropout=0.5, name='call_with_dropout') 94 | 95 | image = [[1, 2, 3], [4, 5, 6], [7, 8, 9]] 96 | inputs = tf.constant(image, tf.float32) 97 | inputs = tf.reshape(inputs, [1, 3, 3, 1]) 98 | 99 | outputs = layer(inputs) 100 | 101 | with self.test_session() as sess: 102 | sess.run(tf.global_variables_initializer()) 103 | 104 | # Dropout is random and therefore not testable, so we just ran it 105 | # and ensure that the computation succeeds. 106 | self.assertEqual(outputs.eval().shape, (1, 3, 3, 2)) 107 | -------------------------------------------------------------------------------- /lib/layer/embedded_gcnn.py: -------------------------------------------------------------------------------- 1 | from six.moves import xrange 2 | 3 | import tensorflow as tf 4 | 5 | from .var_layer import VarLayer 6 | from ..tf import base, sparse_tensor_diag_matmul 7 | 8 | 9 | def conv(features, adj_dist, adj_rad, weights, K=2): 10 | P = weights.get_shape()[0].value - 1 11 | 12 | degree = tf.sparse_reduce_sum(adj_dist, axis=1) 13 | degree = degree + tf.ones_like(degree) 14 | degree = tf.cast(degree, tf.float32) 15 | 16 | features_rescaled = tf.reshape(tf.pow(degree, -1), [-1, 1]) * features 17 | output = tf.matmul(features_rescaled, weights[P]) 18 | 19 | degree = tf.pow(degree, -0.5) 20 | adj_dist = sparse_tensor_diag_matmul(adj_dist, degree, transpose=True) 21 | adj_dist = sparse_tensor_diag_matmul(adj_dist, degree, transpose=False) 22 | 23 | for p in xrange(P): 24 | partition = base(adj_rad, K, P, p) 25 | 26 | # Note that we can perform element-wise multiplication on the two 27 | # adjacency matrices, although the sparse partition matrix has way less 28 | # elements than adj_dist. `base()` doesn't remove any element from 29 | # adj_rad and instead fills the irrelevant values with zeros. It is 30 | # nevertheless important that adj_dist and adj_rad have the same number 31 | # of elements with equal ordering. 32 | adj_values = tf.multiply(adj_dist.values, partition.values) 33 | adj = tf.SparseTensor(adj_dist.indices, adj_values, 34 | adj_dist.dense_shape) 35 | 36 | output_p = tf.sparse_tensor_dense_matmul(adj, features) 37 | output_p = tf.matmul(output_p, weights[p]) 38 | 39 | output += output_p 40 | 41 | return output 42 | 43 | 44 | class EmbeddedGCNN(VarLayer): 45 | def __init__(self, 46 | in_channels, 47 | out_channels, 48 | adjs_dist, 49 | adjs_rad, 50 | local_controllability=2, 51 | sampling_points=8, 52 | **kwargs): 53 | 54 | self.adjs_dist = adjs_dist 55 | self.adjs_rad = adjs_rad 56 | self.K = local_controllability 57 | self.P = sampling_points 58 | 59 | super(EmbeddedGCNN, self).__init__( 60 | weight_shape=[self.P + 1, in_channels, out_channels], 61 | bias_shape=[out_channels], 62 | **kwargs) 63 | 64 | def _call(self, inputs): 65 | batch_size = len(inputs) 66 | outputs = [] 67 | 68 | for i in xrange(batch_size): 69 | output = conv(inputs[i], self.adjs_dist[i], self.adjs_rad[i], 70 | self.vars['weights'], self.K) 71 | 72 | if self.bias: 73 | output = tf.nn.bias_add(output, self.vars['bias']) 74 | 75 | output = self.act(output) 76 | outputs.append(output) 77 | 78 | return outputs 79 | -------------------------------------------------------------------------------- /lib/layer/fc.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | from .var_layer import VarLayer 4 | 5 | 6 | class FC(VarLayer): 7 | def __init__(self, in_channels, out_channels, dropout=None, **kwargs): 8 | self.dropout = dropout 9 | 10 | super(FC, self).__init__( 11 | weight_shape=[in_channels, out_channels], 12 | bias_shape=[out_channels], 13 | **kwargs) 14 | 15 | def _call(self, inputs): 16 | in_channels = self.vars['weights'].get_shape()[0].value 17 | 18 | outputs = tf.reshape(inputs, [-1, in_channels]) 19 | 20 | if self.dropout is not None: 21 | outputs = tf.nn.dropout(outputs, 1 - self.dropout) 22 | 23 | outputs = tf.matmul(outputs, self.vars['weights']) 24 | 25 | if self.bias: 26 | outputs = tf.nn.bias_add(outputs, self.vars['bias']) 27 | 28 | return self.act(outputs) 29 | -------------------------------------------------------------------------------- /lib/layer/fc_test.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | from .fc import FC 4 | 5 | 6 | class FCTest(tf.test.TestCase): 7 | def test_init(self): 8 | layer = FC(1, 2) 9 | self.assertEqual(layer.name, 'fc_1') 10 | self.assertIsNone(layer.dropout) 11 | self.assertIn('weights', layer.vars) 12 | self.assertEqual(layer.vars['weights'].get_shape(), [1, 2]) 13 | self.assertEqual(layer.vars['bias'].get_shape(), [2]) 14 | 15 | layer = FC(1, 2, dropout=0.5) 16 | self.assertEqual(layer.dropout, 0.5) 17 | 18 | def test_call(self): 19 | layer = FC(3, 5, name='call') 20 | inputs = tf.constant([[1, 2, 3], [4, 5, 6]], dtype=tf.float32) 21 | outputs = layer(inputs) 22 | 23 | expected = tf.matmul(inputs, layer.vars['weights']) 24 | expected = tf.nn.bias_add(expected, layer.vars['bias']) 25 | expected = tf.nn.relu(expected) 26 | 27 | with self.test_session() as sess: 28 | sess.run(tf.global_variables_initializer()) 29 | 30 | self.assertAllEqual(outputs.eval().shape, (2, 5)) 31 | self.assertAllEqual(outputs.eval(), expected.eval()) 32 | 33 | def test_call_without_bias(self): 34 | layer = FC(3, 5, bias=False, name='call_without_bias') 35 | inputs = tf.constant([[1, 2, 3], [4, 5, 6]], dtype=tf.float32) 36 | outputs = layer(inputs) 37 | 38 | expected = tf.matmul(inputs, layer.vars['weights']) 39 | expected = tf.nn.relu(expected) 40 | 41 | with self.test_session() as sess: 42 | sess.run(tf.global_variables_initializer()) 43 | 44 | self.assertAllEqual(outputs.eval().shape, (2, 5)) 45 | self.assertAllEqual(outputs.eval(), expected.eval()) 46 | 47 | def test_call_with_dropout(self): 48 | layer = FC(3, 5, dropout=0.5, name='call_with_dropout') 49 | inputs = tf.constant([[1, 2, 3], [4, 5, 6]], dtype=tf.float32) 50 | outputs = layer(inputs) 51 | 52 | with self.test_session() as sess: 53 | sess.run(tf.global_variables_initializer()) 54 | 55 | # Dropout is random and therefore not testable, so we just ran it 56 | # and ensure that the computation succeeds. 57 | self.assertEqual(outputs.eval().shape, (2, 5)) 58 | -------------------------------------------------------------------------------- /lib/layer/fire.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | from .layer import Layer 4 | from .conv2d import Conv2d 5 | 6 | 7 | class Fire(Layer): 8 | def __init__(self, in_channels, reduce_channels, out_channels, **kwargs): 9 | self._conv1 = Conv2d(in_channels, reduce_channels, size=1, **kwargs) 10 | self._conv2 = Conv2d(reduce_channels, out_channels, size=1, **kwargs) 11 | self._conv3 = Conv2d(reduce_channels, out_channels, size=3, **kwargs) 12 | 13 | super(Fire, self).__init__(**kwargs) 14 | 15 | def _call(self, inputs): 16 | outputs = self._conv1(inputs) 17 | outputs_1 = self._conv2(outputs) 18 | outputs_2 = self._conv3(outputs) 19 | return tf.concat([outputs_1, outputs_2], axis=3) 20 | -------------------------------------------------------------------------------- /lib/layer/fire_test.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | from .fire import Fire 4 | 5 | 6 | class FireTest(tf.test.TestCase): 7 | def test_call(self): 8 | layer = Fire(1, 1, 4) 9 | self.assertEqual(layer.name, 'fire_1') 10 | 11 | image = [[1, 2, 3], [4, 5, 6], [7, 8, 9]] 12 | inputs = tf.constant([image, image], tf.float32) 13 | inputs = tf.reshape(inputs, [2, 3, 3, 1]) 14 | 15 | outputs = layer(inputs) 16 | 17 | with self.test_session() as sess: 18 | sess.run(tf.global_variables_initializer()) 19 | self.assertEqual(outputs.eval().shape, (2, 3, 3, 8)) 20 | -------------------------------------------------------------------------------- /lib/layer/gcnn.py: -------------------------------------------------------------------------------- 1 | from six.moves import xrange 2 | 3 | import tensorflow as tf 4 | 5 | from .var_layer import VarLayer 6 | from ..tf import sparse_tensor_diag_matmul 7 | 8 | 9 | def conv(features, adj, weights): 10 | degree = tf.sparse_reduce_sum(adj, axis=1) + 1 11 | degree = tf.cast(degree, tf.float32) 12 | degree = tf.pow(degree, -0.5) 13 | 14 | adj = sparse_tensor_diag_matmul(adj, degree, transpose=True) 15 | adj = sparse_tensor_diag_matmul(adj, degree, transpose=False) 16 | 17 | output = tf.sparse_tensor_dense_matmul(adj, features) 18 | 19 | features = tf.transpose(features) 20 | features = tf.multiply(tf.multiply(degree, features), degree) 21 | features = tf.transpose(features) 22 | output = output + features 23 | 24 | return tf.matmul(output, weights) 25 | 26 | 27 | class GCNN(VarLayer): 28 | def __init__(self, in_channels, out_channels, adjs, **kwargs): 29 | self.adjs = adjs 30 | 31 | super(GCNN, self).__init__( 32 | weight_shape=[in_channels, out_channels], 33 | bias_shape=[out_channels], 34 | **kwargs) 35 | 36 | def _call(self, inputs): 37 | batch_size = len(inputs) 38 | outputs = [] 39 | 40 | for i in xrange(batch_size): 41 | output = conv(inputs[i], self.adjs[i], self.vars['weights']) 42 | 43 | if self.bias: 44 | output = tf.nn.bias_add(output, self.vars['bias']) 45 | 46 | output = self.act(output) 47 | outputs.append(output) 48 | 49 | return outputs 50 | -------------------------------------------------------------------------------- /lib/layer/gcnn_test.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import scipy.sparse as sp 3 | import tensorflow as tf 4 | 5 | from .gcnn import conv, GCNN 6 | from ..tf.convert import sparse_to_tensor 7 | 8 | 9 | class GCNNTest(tf.test.TestCase): 10 | def test_conv(self): 11 | adj = [[0, 1, 0], [1, 0, 2], [0, 2, 0]] 12 | adj = sp.coo_matrix(adj, dtype=np.float32) 13 | adj_norm = adj + sp.eye(3, dtype=np.float32) 14 | degree = np.array(adj_norm.sum(1)).flatten() 15 | degree = np.power(degree, -0.5) 16 | degree = sp.diags(degree) 17 | adj_norm = degree.dot(adj_norm).dot(degree) 18 | adj = sparse_to_tensor(adj) 19 | 20 | features = [[1, 2], [3, 4], [5, 6]] 21 | features_np = np.array(features, dtype=np.float32) 22 | features_tf = tf.constant(features, dtype=tf.float32) 23 | 24 | weights = [[0.3], [0.7]] 25 | weights_np = np.array(weights, dtype=np.float32) 26 | weights_tf = tf.constant(weights, dtype=tf.float32) 27 | 28 | expected = adj_norm.dot(features_np).dot(weights_np) 29 | 30 | with self.test_session(): 31 | self.assertAllEqual( 32 | conv(features_tf, adj, weights_tf).eval(), expected) 33 | 34 | def test_init(self): 35 | layer = GCNN(1, 2, adjs=None) 36 | self.assertEqual(layer.name, 'gcnn_1') 37 | self.assertIsNone(layer.adjs) 38 | self.assertEqual(layer.vars['weights'].get_shape(), [1, 2]) 39 | self.assertEqual(layer.vars['bias'].get_shape(), [2]) 40 | 41 | def test_call(self): 42 | adj = [[0, 1, 0], [1, 0, 2], [0, 2, 0]] 43 | adj = sp.coo_matrix(adj, dtype=np.float32) 44 | adj = sparse_to_tensor(adj) 45 | 46 | layer = GCNN(2, 3, [adj, adj], name='call') 47 | 48 | input_1 = [[1, 2], [3, 4], [5, 6]] 49 | input_1 = tf.constant(input_1, dtype=tf.float32) 50 | input_2 = [[7, 8], [9, 10], [11, 12]] 51 | input_2 = tf.constant(input_2, dtype=tf.float32) 52 | inputs = [input_1, input_2] 53 | outputs = layer(inputs) 54 | 55 | expected_1 = conv(input_1, adj, layer.vars['weights']) 56 | expected_1 = tf.nn.bias_add(expected_1, layer.vars['bias']) 57 | expected_1 = tf.nn.relu(expected_1) 58 | 59 | expected_2 = conv(input_2, adj, layer.vars['weights']) 60 | expected_2 = tf.nn.bias_add(expected_2, layer.vars['bias']) 61 | expected_2 = tf.nn.relu(expected_2) 62 | 63 | with self.test_session() as sess: 64 | sess.run(tf.global_variables_initializer()) 65 | 66 | self.assertEqual(len(outputs), 2) 67 | self.assertEqual(outputs[0].eval().shape, (3, 3)) 68 | self.assertEqual(outputs[1].eval().shape, (3, 3)) 69 | self.assertAllEqual(outputs[0].eval(), expected_1.eval()) 70 | 71 | def test_call_without_bias(self): 72 | adj = [[0, 1, 0], [1, 0, 2], [0, 2, 0]] 73 | adj = sp.coo_matrix(adj, dtype=np.float32) 74 | adj = sparse_to_tensor(adj) 75 | 76 | layer = GCNN(2, 3, [adj, adj], bias=False, name='call_without_bias') 77 | 78 | input_1 = [[1, 2], [3, 4], [5, 6]] 79 | input_1 = tf.constant(input_1, dtype=tf.float32) 80 | input_2 = [[7, 8], [9, 10], [11, 12]] 81 | input_2 = tf.constant(input_2, dtype=tf.float32) 82 | inputs = [input_1, input_2] 83 | outputs = layer(inputs) 84 | 85 | expected_1 = conv(input_1, adj, layer.vars['weights']) 86 | expected_1 = tf.nn.relu(expected_1) 87 | 88 | expected_2 = conv(input_2, adj, layer.vars['weights']) 89 | expected_2 = tf.nn.relu(expected_2) 90 | 91 | with self.test_session() as sess: 92 | sess.run(tf.global_variables_initializer()) 93 | 94 | self.assertEqual(len(outputs), 2) 95 | self.assertEqual(outputs[0].eval().shape, (3, 3)) 96 | self.assertEqual(outputs[1].eval().shape, (3, 3)) 97 | self.assertAllEqual(outputs[0].eval(), expected_1.eval()) 98 | -------------------------------------------------------------------------------- /lib/layer/image_augment.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | from .layer import Layer 4 | 5 | 6 | class ImageAugment(Layer): 7 | def __init__(self, **kwargs): 8 | super(ImageAugment, self).__init__(**kwargs) 9 | 10 | def _call(self, inputs): 11 | outputs = inputs 12 | 13 | outputs = tf.map_fn( 14 | lambda image: tf.image.random_flip_left_right(image), outputs) 15 | outputs = tf.map_fn( 16 | lambda image: tf.image.random_brightness(image, 0.3), outputs) 17 | outputs = tf.map_fn( 18 | lambda image: tf.image.random_contrast(image, 0.7, 1.3), outputs) 19 | outputs = tf.map_fn( 20 | lambda image: tf.image.per_image_standardization(image), outputs) 21 | 22 | return outputs 23 | -------------------------------------------------------------------------------- /lib/layer/image_augment_test.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | from .image_augment import ImageAugment 4 | 5 | 6 | class ImageAugmentTest(tf.test.TestCase): 7 | def test_init(self): 8 | layer = ImageAugment() 9 | self.assertEqual(layer.name, 'imageaugment_1') 10 | 11 | def test_call(self): 12 | layer = ImageAugment(name='call') 13 | inputs = tf.constant([[[ 14 | [0.1, 0.2, 0.3], 15 | [0.3, 0.4, 0.5], 16 | ], [ 17 | [0.5, 0.6, 0.7], 18 | [0.7, 0.8, 0.9], 19 | ]], [[ 20 | [0.1, 0.2, 0.3], 21 | [0.3, 0.4, 0.5], 22 | ], [ 23 | [0.5, 0.6, 0.7], 24 | [0.7, 0.8, 0.9], 25 | ]]]) 26 | 27 | outputs = layer(inputs) 28 | 29 | with self.test_session(): 30 | # Augmentation is random and therefore not testable, so we just ran 31 | # it and ensure that the computation succeeds. 32 | self.assertEqual(outputs.eval().shape, (2, 2, 2, 3)) 33 | -------------------------------------------------------------------------------- /lib/layer/inits.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | 4 | def weight_variable(shape, name, stddev=0.1, decay=0.0, dtype=tf.float32): 5 | initializer = tf.truncated_normal_initializer(stddev=stddev, dtype=dtype) 6 | var = tf.get_variable(name, shape, dtype, initializer) 7 | 8 | if decay > 0: 9 | decay = tf.multiply(tf.nn.l2_loss(var), decay, name='weight_loss') 10 | tf.add_to_collection('losses', decay) 11 | 12 | return var 13 | 14 | 15 | def bias_variable(shape, name, constant=0.1, decay=0.0, dtype=tf.float32): 16 | initializer = tf.constant_initializer(constant, dtype) 17 | var = tf.get_variable(name, shape, dtype, initializer) 18 | 19 | if decay > 0: 20 | decay = tf.multiply(tf.nn.l2_loss(var), decay, name='bias_loss') 21 | tf.add_to_collection('losses', decay) 22 | 23 | return var 24 | -------------------------------------------------------------------------------- /lib/layer/inits_test.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from numpy.testing import assert_almost_equal 3 | 4 | from .inits import weight_variable, bias_variable 5 | 6 | 7 | class InitsTest(tf.test.TestCase): 8 | def test_weight_variable(self): 9 | weights = weight_variable([2, 3], name='weights_1') 10 | 11 | with self.test_session() as sess: 12 | sess.run(tf.global_variables_initializer()) 13 | 14 | self.assertEqual(weights.name, 'weights_1:0') 15 | assert_almost_equal(weights.eval().shape, (2, 3)) 16 | 17 | weights = weight_variable( 18 | [2, 3], name='weights_2', stddev=0, dtype=tf.float64) 19 | 20 | expected = [[0, 0, 0], [0, 0, 0]] 21 | 22 | with self.test_session() as sess: 23 | sess.run(tf.global_variables_initializer()) 24 | 25 | self.assertEqual(weights.name, 'weights_2:0') 26 | self.assertAllEqual(weights.eval(), expected) 27 | self.assertEqual(tf.get_collection('losses'), []) 28 | 29 | def test_weight_variable_with_decay(self): 30 | weights = weight_variable([2, 3], name='weights', decay=0.01) 31 | losses = tf.get_collection('losses') 32 | 33 | expected = tf.nn.l2_loss(weights) 34 | expected = expected * 0.01 35 | 36 | with self.test_session() as sess: 37 | sess.run(tf.global_variables_initializer()) 38 | 39 | self.assertEqual(len(losses), 1) 40 | self.assertEqual(losses[0].name, 'weight_loss:0') 41 | self.assertEqual(losses[0].eval(), expected.eval()) 42 | 43 | def test_bias_variable(self): 44 | bias = bias_variable([2, 3], 'bias_1') 45 | 46 | expected = [[0.1, 0.1, 0.1], [0.1, 0.1, 0.1]] 47 | 48 | with self.test_session() as sess: 49 | sess.run(tf.global_variables_initializer()) 50 | 51 | self.assertEqual(bias.name, 'bias_1:0') 52 | assert_almost_equal(bias.eval(), expected) 53 | 54 | bias = bias_variable([1, 4], name='bias_2', constant=1, dtype=tf.uint8) 55 | 56 | expected = [[1, 1, 1, 1]] 57 | 58 | with self.test_session() as sess: 59 | sess.run(tf.global_variables_initializer()) 60 | 61 | self.assertEqual(bias.name, 'bias_2:0') 62 | self.assertAllEqual(bias.eval(), expected) 63 | 64 | def test_bias_variable_with_decay(self): 65 | bias = bias_variable([2, 3], name='biases', decay=0.01) 66 | losses = tf.get_collection('losses') 67 | 68 | expected = tf.nn.l2_loss(bias) 69 | expected = expected * 0.01 70 | 71 | with self.test_session() as sess: 72 | sess.run(tf.global_variables_initializer()) 73 | 74 | self.assertEqual(len(losses), 1) 75 | self.assertEqual(losses[0].name, 'bias_loss:0') 76 | self.assertEqual(losses[0].eval(), expected.eval()) 77 | -------------------------------------------------------------------------------- /lib/layer/layer.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | _LAYER_UIDS = {} 4 | 5 | 6 | def _layer_uid(name): 7 | if name not in _LAYER_UIDS: 8 | _LAYER_UIDS[name] = 0 9 | 10 | _LAYER_UIDS[name] += 1 11 | return _LAYER_UIDS[name] 12 | 13 | 14 | class Layer(object): 15 | def __init__(self, name=None, logging=False): 16 | 17 | if not name: 18 | layer = self.__class__.__name__.lower() 19 | name = '{}_{}'.format(layer, _layer_uid(layer)) 20 | 21 | self.name = name 22 | self.logging = logging 23 | 24 | def __call__(self, inputs): 25 | with tf.name_scope(self.name): 26 | outputs = self._call(inputs) 27 | 28 | return outputs 29 | 30 | def _call(self, inputs): 31 | return inputs 32 | -------------------------------------------------------------------------------- /lib/layer/layer_test.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | from .layer import Layer 4 | 5 | 6 | class LayerTest(tf.test.TestCase): 7 | def test_init(self): 8 | layer = Layer(name='layer') 9 | self.assertEqual(layer.name, 'layer') 10 | self.assertEqual(layer.logging, False) 11 | 12 | layer = Layer(logging=True) 13 | self.assertEqual(layer.name, 'layer_1') 14 | self.assertEqual(layer.logging, True) 15 | 16 | layer = Layer() 17 | self.assertEqual(layer.name, 'layer_2') 18 | 19 | def test_call(self): 20 | layer = Layer(name='call') 21 | inputs = tf.constant([1, 2, 3]) 22 | 23 | with self.test_session(): 24 | self.assertAllEqual(layer(inputs).eval(), inputs.eval()) 25 | -------------------------------------------------------------------------------- /lib/layer/max_pool.py: -------------------------------------------------------------------------------- 1 | from six.moves import xrange 2 | 3 | import tensorflow as tf 4 | 5 | from .layer import Layer 6 | 7 | 8 | class MaxPool(Layer): 9 | def __init__(self, size, stride=None, **kwargs): 10 | self.size = size 11 | self.stride = size if stride is None else stride 12 | 13 | super(MaxPool, self).__init__(**kwargs) 14 | 15 | def _call(self, inputs): 16 | # Different treatment of the inputs, whether we learn on arbitary 17 | # graphs or images. Graph features are passed as a normal python list 18 | # and need a special adjustment, whereas image tensors use the normal 19 | # tensorflow `max_pool` operation. 20 | 21 | if isinstance(inputs, list): 22 | batch_size = len(inputs) 23 | 24 | outputs = [] 25 | for i in xrange(batch_size): 26 | output = tf.expand_dims(inputs[i], axis=0) 27 | output = tf.expand_dims(output, axis=3) 28 | output = tf.nn.max_pool( 29 | output, 30 | ksize=[1, self.size, 1, 1], 31 | strides=[1, self.stride, 1, 1], 32 | padding='SAME') 33 | output = tf.squeeze(output, axis=[0, 3]) 34 | outputs.append(output) 35 | 36 | return outputs 37 | else: 38 | return tf.nn.max_pool( 39 | inputs, 40 | ksize=[1, self.size, self.size, 1], 41 | strides=[1, self.stride, self.stride, 1], 42 | padding='SAME') 43 | -------------------------------------------------------------------------------- /lib/layer/max_pool_test.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | from .max_pool import MaxPool 4 | 5 | 6 | class MaxPool2DTest(tf.test.TestCase): 7 | def test_init(self): 8 | layer = MaxPool(size=2, stride=1) 9 | self.assertEqual(layer.name, 'maxpool_1') 10 | self.assertEqual(layer.size, 2) 11 | self.assertEqual(layer.stride, 1) 12 | 13 | layer = MaxPool(size=2) 14 | self.assertEqual(layer.size, 2) 15 | self.assertEqual(layer.stride, 2) 16 | 17 | def test_call(self): 18 | layer = MaxPool(size=2, name='call') 19 | input_1 = [[1, 2], [3, 4], [5, 6]] 20 | input_2 = [[7, 8], [9, 10], [11, 12]] 21 | input_1 = tf.constant(input_1, dtype=tf.float32) 22 | input_2 = tf.constant(input_2, dtype=tf.float32) 23 | inputs = [input_1, input_2] 24 | outputs = layer(inputs) 25 | 26 | expected = [[[3, 4], [5, 6]], [[9, 10], [11, 12]]] 27 | 28 | with self.test_session(): 29 | self.assertEqual(len(outputs), 2) 30 | self.assertAllEqual(outputs[0].eval(), expected[0]) 31 | self.assertAllEqual(outputs[1].eval(), expected[1]) 32 | 33 | def test_call_with_tensor(self): 34 | layer = MaxPool(size=2, name='call_with_tensor') 35 | 36 | image = [[1, 2, 3], [4, 5, 6], [7, 8, 9]] 37 | inputs = tf.constant(image, tf.float32) 38 | inputs = tf.reshape(inputs, [1, 3, 3, 1]) 39 | 40 | outputs = layer(inputs) 41 | 42 | expected = [[[[5], [6]], [[8], [9]]]] 43 | 44 | with self.test_session(): 45 | self.assertEqual(outputs.eval().shape, (1, 2, 2, 1)) 46 | self.assertAllEqual(outputs.eval(), expected) 47 | -------------------------------------------------------------------------------- /lib/layer/spatial.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | from .var_layer import VarLayer 4 | 5 | 6 | def conv(inputs, weights): 7 | outputs = tf.nn.conv2d( 8 | inputs, 9 | weights, 10 | strides=[1, 1, weights.get_shape()[1], 1], 11 | padding='VALID') 12 | return tf.squeeze(outputs, axis=2) 13 | 14 | 15 | class SpatialCNN(VarLayer): 16 | def __init__(self, in_channels, out_channels, neighborhood_size, **kwargs): 17 | 18 | super(SpatialCNN, self).__init__( 19 | weight_shape=[1, neighborhood_size, in_channels, out_channels], 20 | bias_shape=[out_channels], 21 | **kwargs) 22 | 23 | def _call(self, inputs): 24 | outputs = conv(inputs, self.vars['weights']) 25 | 26 | if self.bias: 27 | outputs = tf.nn.bias_add(outputs, self.vars['bias']) 28 | 29 | return self.act(outputs) 30 | -------------------------------------------------------------------------------- /lib/layer/spatial_test.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | from .spatial import conv, SpatialCNN 4 | 5 | 6 | class SpatialCNNTest(tf.test.TestCase): 7 | def test_conv(self): 8 | cube = [ 9 | [[1, 2], [3, 4], [5, 6]], 10 | [[6, 5], [4, 3], [2, 1]], 11 | ] 12 | 13 | inputs = tf.constant([cube, cube], tf.float32) 14 | 15 | weights = tf.constant(1, tf.float32, shape=[1, 3, 2, 8]) 16 | self.assertAllEqual(conv(inputs, weights).get_shape(), [2, 2, 8]) 17 | 18 | weights = [1, 2, 3, 4, 5, 6] 19 | weights = tf.constant(weights, tf.float32) 20 | weights = tf.reshape(weights, [1, 3, 2, 1]) 21 | 22 | outputs = conv(inputs, weights) 23 | 24 | expected = [[1 * 1 + 2 * 2 + 3 * 3 + 4 * 4 + 5 * 5 + 6 * 6], 25 | [1 * 6 + 2 * 5 + 3 * 4 + 4 * 3 + 5 * 2 + 6 * 1]] 26 | 27 | with self.test_session(): 28 | self.assertAllEqual(outputs.eval()[0], expected) 29 | self.assertAllEqual(outputs.eval()[1], expected) 30 | 31 | def test_init(self): 32 | layer = SpatialCNN(3, 64, 9) 33 | self.assertEqual(layer.name, 'spatialcnn_1') 34 | self.assertEqual(layer.vars['weights'].get_shape(), [1, 9, 3, 64]) 35 | self.assertEqual(layer.vars['bias'].get_shape(), [64]) 36 | 37 | def test_call(self): 38 | layer = SpatialCNN(3, 64, 9, name='call') 39 | 40 | inputs = tf.constant(1, tf.float32, shape=[4, 25, 9, 3]) 41 | 42 | outputs = layer(inputs) 43 | 44 | with self.test_session() as sess: 45 | sess.run(tf.global_variables_initializer()) 46 | self.assertEqual(outputs.eval().shape, (4, 25, 64)) 47 | -------------------------------------------------------------------------------- /lib/layer/var_layer.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | from .layer import Layer 4 | from .inits import weight_variable, bias_variable 5 | 6 | 7 | class VarLayer(Layer): 8 | def __init__(self, 9 | weight_shape, 10 | bias_shape, 11 | weight_stddev=0.1, 12 | weight_decay=0.0, 13 | bias=True, 14 | bias_constant=0.1, 15 | bias_decay=0.0, 16 | act=tf.nn.relu, 17 | **kwargs): 18 | 19 | super(VarLayer, self).__init__(**kwargs) 20 | 21 | self.bias = bias 22 | self.act = act 23 | self.vars = {} 24 | 25 | with tf.variable_scope('{}_vars'.format(self.name)): 26 | self.vars['weights'] = weight_variable( 27 | weight_shape, '{}_weights'.format(self.name), weight_stddev, 28 | weight_decay) 29 | 30 | if self.bias: 31 | self.vars['bias'] = bias_variable(bias_shape, 32 | '{}_bias'.format(self.name), 33 | bias_constant, bias_decay) 34 | 35 | if self.logging: 36 | for var in self.vars: 37 | tf.summary.histogram('{}/vars/{}'.format(self.name, var), 38 | self.vars[var]) 39 | -------------------------------------------------------------------------------- /lib/layer/var_layer_test.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | from .var_layer import VarLayer 4 | 5 | 6 | class VarLayerTest(tf.test.TestCase): 7 | def test_init(self): 8 | layer = VarLayer(weight_shape=[1, 2], bias_shape=[2]) 9 | self.assertEqual(layer.name, 'varlayer_1') 10 | self.assertTrue(layer.bias) 11 | self.assertEqual(layer.act, tf.nn.relu) 12 | self.assertEqual(layer.vars['weights'].get_shape(), [1, 2]) 13 | self.assertEqual(tf.get_collection('losses'), []) 14 | self.assertEqual(layer.vars['bias'].get_shape(), [2]) 15 | 16 | layer = VarLayer( 17 | weight_shape=[1, 2], 18 | bias_shape=[2], 19 | weight_stddev=0.0, 20 | weight_decay=0.01, 21 | bias=True, 22 | bias_constant=1, 23 | logging=True) 24 | losses = tf.get_collection('losses') 25 | 26 | expected_weights = [[0, 0]] 27 | expected_bias = [1, 1] 28 | 29 | with self.test_session() as sess: 30 | sess.run(tf.global_variables_initializer()) 31 | 32 | self.assertEqual(layer.name, 'varlayer_2') 33 | self.assertTrue(layer.bias) 34 | self.assertAllEqual(layer.vars['weights'].eval(), expected_weights) 35 | self.assertAllEqual(layer.vars['bias'].eval(), expected_bias) 36 | self.assertEqual(losses[0].name, 'varlayer_2_vars/weight_loss:0') 37 | self.assertEqual(losses[0].eval(), 0) 38 | 39 | layer = VarLayer(weight_shape=[1, 2], bias_shape=[2], bias=False) 40 | self.assertEqual(layer.name, 'varlayer_3') 41 | self.assertFalse(layer.bias) 42 | self.assertIsNone(layer.vars.get('bias', None)) 43 | -------------------------------------------------------------------------------- /lib/model/__init__.py: -------------------------------------------------------------------------------- 1 | from .model import Model 2 | from .metrics import softmax_cross_entropy 3 | from .placeholder import generate_placeholders 4 | from .train import train 5 | 6 | __all__ = [ 7 | 'Model', 8 | 'softmax_cross_entropy', 9 | 'generate_placeholders', 10 | 'train', 11 | ] 12 | -------------------------------------------------------------------------------- /lib/model/metrics.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | 4 | def softmax_cross_entropy(outputs, labels): 5 | """Calculate softmax cross-entropy loss.""" 6 | 7 | loss_per_example = tf.nn.softmax_cross_entropy_with_logits( 8 | logits=outputs, labels=labels, name='loss_per_example') 9 | return tf.reduce_mean(loss_per_example, name='loss') 10 | 11 | 12 | def total_loss(loss): 13 | tf.add_to_collection('losses', loss) 14 | losses = tf.get_collection('losses') 15 | return tf.add_n(losses, name='total_loss') 16 | 17 | 18 | def accuracy(outputs, labels): 19 | """Calculate accuracy.""" 20 | 21 | num_labels = labels.get_shape()[1] 22 | 23 | with tf.name_scope('accuracy'): 24 | labels = tf.cast(labels, tf.bool) 25 | 26 | predicted_labels = tf.argmax(outputs, axis=1) 27 | predicted_labels_one_hot = tf.one_hot(predicted_labels, num_labels) 28 | predicted_labels_one_hot = tf.cast(predicted_labels_one_hot, tf.bool) 29 | 30 | correct_prediction = tf.logical_and(predicted_labels_one_hot, labels) 31 | correct_prediction = tf.cast(correct_prediction, tf.float32) 32 | correct_prediction = tf.reduce_max(correct_prediction, axis=1) 33 | 34 | accuracy = tf.reduce_mean(correct_prediction) 35 | 36 | return accuracy 37 | 38 | 39 | def precision(outputs, labels, k=0.5): 40 | with tf.name_scope('precision'): 41 | predicted_labels = _threshold_outputs(outputs, k) 42 | true_positives = _true_positives(labels, predicted_labels) 43 | true_and_false_positives = tf.reduce_sum( 44 | tf.cast(predicted_labels, tf.float32)) 45 | 46 | res = true_positives / true_and_false_positives 47 | zero = tf.constant(0, tf.float32) 48 | return tf.cond(tf.is_nan(res), lambda: zero, lambda: res) 49 | 50 | 51 | def recall(outputs, labels, k=0.5): 52 | with tf.name_scope('recall'): 53 | predicted_labels = _threshold_outputs(outputs, k) 54 | true_positives = _true_positives(labels, predicted_labels) 55 | relevant_elements = tf.reduce_sum(tf.cast(labels, tf.float32)) 56 | 57 | res = true_positives / relevant_elements 58 | zero = tf.constant(0, tf.float32) 59 | return tf.cond(tf.is_nan(res), lambda: zero, lambda: res) 60 | 61 | 62 | def _threshold_outputs(outputs, k=0.5): 63 | outputs = tf.nn.sigmoid(outputs) 64 | k = tf.zeros_like(outputs) + k 65 | outputs = tf.greater(outputs, k) 66 | return tf.cast(outputs, tf.uint8) 67 | 68 | 69 | def _true_positives(labels, predicted_labels): 70 | labels = tf.cast(labels, tf.bool) 71 | predicted_labels = tf.cast(predicted_labels, tf.bool) 72 | 73 | true_positives = tf.logical_and(labels, predicted_labels) 74 | true_positives = tf.reduce_sum(tf.cast(true_positives, tf.float32)) 75 | 76 | return true_positives 77 | -------------------------------------------------------------------------------- /lib/model/metrics_test.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | from .metrics import (softmax_cross_entropy, total_loss, accuracy, precision, 4 | recall) 5 | 6 | 7 | class MetricsTest(tf.test.TestCase): 8 | def test_softmax(self): 9 | outputs = [[0, 0, 0, 1], [0, 0, 1, 0]] 10 | outputs = tf.constant(outputs, tf.float32) 11 | 12 | labels = [[0, 0, 0, 1], [0, 0, 1, 0]] 13 | labels = tf.constant(labels, tf.uint8) 14 | 15 | with self.test_session(): 16 | self.assertAlmostEqual( 17 | softmax_cross_entropy(outputs, labels).eval(), 0.74366844) 18 | 19 | def test_total_loss(self): 20 | with self.test_session(): 21 | self.assertEqual(total_loss(2).eval(), 2) 22 | 23 | def test_accuracy(self): 24 | outputs = [[8, 5, 3, 9], [3, 4, 6, 4]] 25 | # [[0, 0, 0, 1], [0, 0, 1, 0]] 26 | outputs = tf.constant(outputs, tf.float32) 27 | 28 | # None right. 29 | labels = [[0, 1, 0, 0], [0, 1, 0, 0]] 30 | labels = tf.constant(labels, tf.int32) 31 | 32 | with self.test_session(): 33 | self.assertEqual(accuracy(outputs, labels).eval(), 0) 34 | 35 | # All right. 36 | labels = [[0, 0, 0, 1], [0, 0, 1, 0]] 37 | labels = tf.constant(labels, tf.int32) 38 | 39 | with self.test_session(): 40 | self.assertEqual(accuracy(outputs, labels).eval(), 1) 41 | 42 | # 1 out of 2 right. 43 | labels = [[1, 0, 0, 0], [0, 0, 1, 0]] 44 | labels = tf.constant(labels, tf.int32) 45 | 46 | with self.test_session(): 47 | self.assertEqual(accuracy(outputs, labels).eval(), 0.5) 48 | 49 | def test_accuracy_multilabel(self): 50 | outputs = [[8, 5, 3, 9], [3, 4, 6, 4]] 51 | # [[0, 0, 0, 1], [0, 0, 1, 0]] 52 | outputs = tf.constant(outputs, tf.float32) 53 | 54 | # None right. 55 | labels = [[0, 1, 1, 0], [0, 1, 0, 1]] 56 | labels = tf.constant(labels, tf.int32) 57 | with self.test_session(): 58 | self.assertEqual(accuracy(outputs, labels).eval(), 0) 59 | 60 | # All right. 61 | labels = [[0, 1, 0, 1], [1, 0, 1, 0]] 62 | labels = tf.constant(labels, tf.int32) 63 | 64 | with self.test_session(): 65 | self.assertEqual(accuracy(outputs, labels).eval(), 1) 66 | 67 | # 1 out of 2 right. 68 | labels = [[1, 1, 0, 0], [0, 1, 1, 1]] 69 | labels = tf.constant(labels, tf.int32) 70 | 71 | with self.test_session(): 72 | self.assertEqual(accuracy(outputs, labels).eval(), 0.5) 73 | 74 | def test_precision(self): 75 | outputs = [[-2, -1, 0, 1, 2], [1, 2, 0, -1, -2]] 76 | # [[0, 0, 0, 1, 1], [1, 1, 0, 0, 0]] 77 | outputs = tf.constant(outputs, tf.float32) 78 | 79 | # None right. 80 | labels = [[1, 1, 1, 0, 0], [0, 0, 1, 1, 1]] 81 | labels = tf.constant(labels, tf.int32) 82 | 83 | with self.test_session(): 84 | self.assertEqual(precision(outputs, labels).eval(), 0.0) 85 | 86 | # All right. 87 | labels = [[0, 0, 0, 1, 1], [1, 1, 0, 0, 0]] 88 | labels = tf.constant(labels, tf.int32) 89 | 90 | with self.test_session(): 91 | self.assertEqual(precision(outputs, labels).eval(), 1.0) 92 | 93 | labels = [[1, 1, 0, 1, 0], [1, 1, 1, 0, 0]] 94 | labels = tf.constant(labels, tf.int32) 95 | 96 | with self.test_session(): 97 | self.assertAlmostEqual(precision(outputs, labels).eval(), 0.75) 98 | 99 | # Test NaN. 100 | outputs = [[-4, -3, -2, -1, 0], [-4, -3, -2, -1, 0]] 101 | outputs = tf.constant(outputs, tf.float32) 102 | 103 | with self.test_session(): 104 | self.assertAlmostEqual(precision(outputs, labels).eval(), 0) 105 | 106 | def test_recall(self): 107 | outputs = [[-2, -1, 0, 1, 2], [1, 2, 0, -1, -2]] 108 | # [[0, 0, 0, 1, 1], [1, 1, 0, 0, 0]] 109 | outputs = tf.constant(outputs, tf.float32) 110 | 111 | # None right. 112 | labels = [[1, 1, 1, 0, 0], [0, 0, 1, 1, 1]] 113 | labels = tf.constant(labels, tf.int32) 114 | 115 | with self.test_session(): 116 | self.assertEqual(recall(outputs, labels).eval(), 0.0) 117 | 118 | # All right. 119 | labels = [[0, 0, 0, 1, 1], [1, 1, 0, 0, 0]] 120 | labels = tf.constant(labels, tf.int32) 121 | 122 | with self.test_session(): 123 | self.assertEqual(recall(outputs, labels).eval(), 1.0) 124 | 125 | labels = [[1, 1, 0, 1, 0], [1, 1, 1, 0, 0]] 126 | labels = tf.constant(labels, tf.int32) 127 | 128 | with self.test_session(): 129 | self.assertAlmostEqual(recall(outputs, labels).eval(), 0.5) 130 | 131 | # Test NaN. 132 | labels = [[0, 0, 0, 0, 0], [0, 0, 0, 0, 0]] 133 | labels = tf.constant(labels, tf.int32) 134 | 135 | with self.test_session(): 136 | self.assertAlmostEqual(precision(outputs, labels).eval(), 0) 137 | -------------------------------------------------------------------------------- /lib/model/model.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | import tensorflow as tf 4 | 5 | from .metrics import (softmax_cross_entropy, total_loss, accuracy) 6 | 7 | 8 | class Model(object): 9 | def __init__(self, 10 | placeholders, 11 | name=None, 12 | learning_rate=0.001, 13 | num_steps_per_decay=None, 14 | epsilon=1e-08, 15 | train_dir=None, 16 | log_dir=None): 17 | 18 | if not name: 19 | name = self.__class__.__name__.lower() 20 | 21 | self.placeholders = placeholders 22 | self.name = name 23 | self._loss_algorithm = softmax_cross_entropy 24 | 25 | self.train_dir = train_dir 26 | self.log_dir = log_dir 27 | self.logging = False if log_dir is None else True 28 | self.sess = None 29 | 30 | self.inputs = placeholders['features'] 31 | self.labels = placeholders['labels'] 32 | self.outputs = None 33 | 34 | self.layers = [] 35 | self.vars = {} 36 | 37 | self._loss = None 38 | self._accuracy = None 39 | self._precision = None 40 | self._recall = None 41 | self._train = None 42 | self._summary = None 43 | self._writer = None 44 | 45 | # Create global step variable. 46 | self._global_step = tf.get_variable( 47 | '{}/global_step'.format(self.name), 48 | shape=[], 49 | dtype=tf.int32, 50 | initializer=tf.constant_initializer(0, dtype=tf.int32), 51 | trainable=False) 52 | 53 | # Create learning rate and optimizer. 54 | if num_steps_per_decay is not None: 55 | learning_rate = tf.train.exponential_decay( 56 | learning_rate, 57 | self._global_step, 58 | num_steps_per_decay, 59 | decay_rate=0.96, 60 | staircase=True) 61 | 62 | tf.summary.scalar('learning_rate', learning_rate) 63 | 64 | self.optimizer = tf.train.AdamOptimizer(learning_rate, epsilon=epsilon) 65 | 66 | def build(self): 67 | with tf.variable_scope(self.name): 68 | self._build() 69 | 70 | # Store model variables for saving and loading. 71 | variables = tf.get_collection( 72 | tf.GraphKeys.GLOBAL_VARIABLES, scope=self.name) 73 | self.vars = {var.name: var for var in variables} 74 | 75 | # Call each layer with the previous outputs. 76 | self.outputs = self.inputs 77 | for layer in self.layers: 78 | self.outputs = layer(self.outputs) 79 | 80 | # Build metrics. 81 | self._loss = self._loss_algorithm(self.outputs, self.labels) 82 | self._loss = total_loss(self._loss) 83 | self._accuracy = accuracy(self.outputs, self.labels) 84 | 85 | # Build train op. 86 | self._train = self.optimizer.minimize( 87 | self._loss, global_step=self._global_step) 88 | 89 | # Create session. 90 | self.sess = tf.Session() 91 | if self.logging: 92 | if tf.gfile.Exists(self.log_dir): 93 | tf.gfile.DeleteRecursively(self.log_dir) 94 | tf.gfile.MakeDirs(self.log_dir) 95 | 96 | self._summary = tf.summary.merge_all() 97 | self._writer = tf.summary.FileWriter(self.log_dir, self.sess.graph) 98 | 99 | def _build(self): 100 | raise NotImplementedError 101 | 102 | def initialize(self): 103 | self.sess.run(tf.global_variables_initializer()) 104 | 105 | if self.train_dir is None: 106 | return self.sess.run(self._global_step) 107 | 108 | if tf.gfile.Exists(self.train_dir): 109 | saver = tf.train.Saver(self.vars) 110 | save_path = '{}/checkpoint.ckpt'.format(self.train_dir) 111 | saver.restore(self.sess, save_path) 112 | print('Model restored from file {}.'.format(save_path)) 113 | else: 114 | tf.gfile.MakeDirs(self.train_dir) 115 | 116 | return self.sess.run(self._global_step) 117 | 118 | def save(self): 119 | if self.train_dir is None: 120 | return 121 | 122 | saver = tf.train.Saver(self.vars) 123 | save_path = '{}/checkpoint.ckpt'.format(self.train_dir) 124 | saver.save(self.sess, save_path) 125 | print('Model saved in file {}.'.format(save_path)) 126 | 127 | def train(self, feed_dict, step=None): 128 | t = time.time() 129 | 130 | if self.logging: 131 | _, summary = self.sess.run([self._train, self._summary], feed_dict) 132 | self._writer.add_summary(summary, step) 133 | else: 134 | self.sess.run(self._train, feed_dict) 135 | 136 | return time.time() - t 137 | 138 | def evaluate(self, feed_dict, step=None, name=None): 139 | loss, acc = self.sess.run([self._loss, self._accuracy], feed_dict) 140 | if self.logging and step is not None and name is not None: 141 | self._add_summary('{}_loss'.format(name), loss, step) 142 | self._add_summary('{}_accuracy'.format(name), acc, step) 143 | return loss, acc 144 | 145 | def _add_summary(self, name, value, step): 146 | summary = tf.Summary( 147 | value=[tf.Summary.Value(tag=name, simple_value=value)]) 148 | self._writer.add_summary(summary, step) 149 | -------------------------------------------------------------------------------- /lib/model/placeholder.py: -------------------------------------------------------------------------------- 1 | from six.moves import xrange 2 | 3 | import numpy as np 4 | import tensorflow as tf 5 | 6 | from ..tf.convert import sparse_to_tensor 7 | 8 | 9 | def generate_placeholders(batch_size, levels, num_features, num_labels): 10 | placeholders = { 11 | 'features': [ 12 | tf.placeholder(tf.float32, [None, num_features], 13 | 'features_{}'.format(i + 1)) 14 | for i in xrange(batch_size) 15 | ], 16 | 'labels': 17 | tf.placeholder(tf.uint8, [batch_size, num_labels], 'labels'), 18 | 'dropout': 19 | tf.placeholder(tf.float32, [], 'dropout'), 20 | } 21 | 22 | for j in xrange(1, levels + 1): 23 | placeholders.update({ 24 | 'adj_dist_{}'.format(j): [ 25 | tf.sparse_placeholder( 26 | tf.float32, name='adj_dist_{}_{}'.format(j, i + 1)) 27 | for i in xrange(batch_size) 28 | ], 29 | }) 30 | 31 | placeholders.update({ 32 | 'adj_rad_{}'.format(j): [ 33 | tf.sparse_placeholder( 34 | tf.float32, name='adj_rad_{}_{}'.format(j, i + 1)) 35 | for i in xrange(batch_size) 36 | ], 37 | }) 38 | 39 | return placeholders 40 | 41 | 42 | def feed_dict_with_batch(placeholders, batch, dropout=0.0): 43 | batch_size = len(batch) 44 | levels = len(batch[0][1]) - 1 45 | labels = np.array([batch[i][-1] for i in xrange(batch_size)], np.int32) 46 | 47 | feed_dict = { 48 | placeholders['labels']: labels, 49 | placeholders['dropout']: dropout, 50 | } 51 | 52 | feed_dict.update( 53 | {placeholders['features'][i]: batch[i][0] 54 | for i in xrange(batch_size)}) 55 | 56 | for j in xrange(levels): 57 | feed_dict.update({ 58 | placeholders['adj_dist_{}'.format(j + 1)][i]: 59 | sparse_to_tensor(batch[i][1][j]) 60 | for i in xrange(batch_size) 61 | }) 62 | 63 | feed_dict.update({ 64 | placeholders['adj_rad_{}'.format(j + 1)][i]: 65 | sparse_to_tensor(batch[i][2][j]) 66 | for i in xrange(batch_size) 67 | }) 68 | 69 | return feed_dict 70 | -------------------------------------------------------------------------------- /lib/model/train.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from __future__ import division 3 | 4 | import os 5 | import time 6 | from six.moves import xrange 7 | from sklearn.preprocessing import StandardScaler 8 | 9 | from ..datasets import PreprocessQueue 10 | from .placeholder import feed_dict_with_batch 11 | from ..pipeline import PreprocessedDataset, FileQueue, augment_batch 12 | 13 | 14 | def train(model, 15 | data, 16 | preprocess_algorithm, 17 | batch_size, 18 | dropout, 19 | augment, 20 | max_steps, 21 | preprocess_first=None, 22 | display_step=10, 23 | save_step=250): 24 | 25 | train_queue, val_queue, test_queue = _generate_queues( 26 | data, preprocess_first, preprocess_algorithm, augment, batch_size) 27 | 28 | model.build() 29 | global_step = model.initialize() 30 | 31 | try: 32 | for step in xrange(global_step, max_steps): 33 | t_pre = time.process_time() 34 | batch = train_queue.dequeue() 35 | 36 | # Augment the preprocessed data. 37 | if augment and preprocess_first is not None: 38 | batch = augment_batch(batch) if augment else batch 39 | 40 | batch = _standard_scale_batch(batch) 41 | feed_dict = feed_dict_with_batch(model.placeholders, batch, 42 | dropout) 43 | t_pre = time.process_time() - t_pre 44 | 45 | t_train = model.train(feed_dict, step) 46 | 47 | if step % display_step == 0: 48 | # Evaluate on training and validation set with zero dropout. 49 | feed_dict.update({model.placeholders['dropout']: 0}) 50 | batch = val_queue.dequeue() 51 | batch = _standard_scale_batch(batch) 52 | val_feed_dict = feed_dict_with_batch(model.placeholders, batch) 53 | 54 | train_info = model.evaluate(feed_dict, step, 'train') 55 | val_info = model.evaluate(val_feed_dict, step, 'val') 56 | 57 | log = 'step={}, '.format(step) 58 | log += 'time={:.2f}s + {:.2f}s, '.format(t_pre, t_train) 59 | log += 'train_loss={:.5f}, '.format(train_info[0]) 60 | log += 'train_acc={:.5f}, '.format(train_info[1]) 61 | log += 'val_loss={:.5f}, '.format(val_info[0]) 62 | log += 'val_acc={:.5f}'.format(val_info[1]) 63 | 64 | print(log) 65 | 66 | if step % save_step == 0: 67 | model.save() 68 | 69 | except KeyboardInterrupt: 70 | print() 71 | 72 | print('Optimization finished!') 73 | print('Evaluate on test set. This can take a few minutes.') 74 | 75 | try: 76 | num_steps = data.test.num_examples // batch_size 77 | test_info = [0, 0] 78 | 79 | for i in xrange(num_steps): 80 | batch = test_queue.dequeue() 81 | batch = _standard_scale_batch(batch) 82 | feed_dict = feed_dict_with_batch(model.placeholders, batch) 83 | 84 | batch_info = model.evaluate(feed_dict) 85 | test_info = [a + b for a, b in zip(test_info, batch_info)] 86 | 87 | log = 'Test results: ' 88 | log += 'loss={:.5f}, '.format(test_info[0] / num_steps) 89 | log += 'acc={:.5f}'.format(test_info[1] / num_steps) 90 | 91 | print(log) 92 | 93 | except KeyboardInterrupt: 94 | print() 95 | print('Test evaluation aborted.') 96 | 97 | finally: 98 | train_queue.close() 99 | val_queue.close() 100 | test_queue.close() 101 | 102 | 103 | def _preprocess_data(data, data_dir, preprocess_algorithm): 104 | data.train = PreprocessedDataset( 105 | os.path.join(data_dir, 'train'), data.train, preprocess_algorithm) 106 | data.val = PreprocessedDataset( 107 | os.path.join(data_dir, 'val'), data.val, preprocess_algorithm) 108 | data.test = PreprocessedDataset( 109 | os.path.join(data_dir, 'test'), data.test, preprocess_algorithm) 110 | return data 111 | 112 | 113 | def _generate_queues(data, preprocess_first, preprocess_algorithm, augment, 114 | batch_size): 115 | capacity = 10 * batch_size 116 | 117 | if preprocess_first is not None: 118 | data = _preprocess_data(data, preprocess_first, preprocess_algorithm) 119 | 120 | train_queue = FileQueue(data.train, batch_size, capacity, shuffle=True) 121 | val_queue = FileQueue(data.val, batch_size, capacity, shuffle=True) 122 | test_queue = FileQueue(data.test, batch_size, capacity, shuffle=False) 123 | else: 124 | train_queue = PreprocessQueue( 125 | data.train, 126 | preprocess_algorithm, 127 | augment, 128 | batch_size, 129 | capacity, 130 | shuffle=True) 131 | 132 | val_queue = PreprocessQueue( 133 | data.val, 134 | preprocess_algorithm, 135 | augment, 136 | batch_size, 137 | capacity, 138 | shuffle=True) 139 | 140 | test_queue = PreprocessQueue( 141 | data.test, 142 | preprocess_algorithm, 143 | augment, 144 | batch_size, 145 | capacity, 146 | shuffle=False) 147 | 148 | return train_queue, val_queue, test_queue 149 | 150 | 151 | def _standard_scale_batch(batch): 152 | for example in batch: 153 | StandardScaler(copy=False).fit_transform(example[0]) 154 | return batch 155 | -------------------------------------------------------------------------------- /lib/pipeline/__init__.py: -------------------------------------------------------------------------------- 1 | from .preprocess import preprocess_pipeline, preprocess_pipeline_fixed 2 | from .dataset import PreprocessedDataset 3 | from .file_queue import FileQueue 4 | from .augment import augment_batch 5 | 6 | __all__ = [ 7 | 'preprocess_pipeline', 8 | 'preprocess_pipeline_fixed', 9 | 'PreprocessedDataset', 10 | 'FileQueue', 11 | 'augment_batch', 12 | ] 13 | -------------------------------------------------------------------------------- /lib/pipeline/augment.py: -------------------------------------------------------------------------------- 1 | import random 2 | import numpy as np 3 | from numpy import pi as PI 4 | 5 | 6 | def augment_batch(batch): 7 | batch_augmented = [] 8 | for example in batch: 9 | features, adjs_dist, adjs_rad, label = example 10 | 11 | features = random_brightness(features, 0, 3, max_delta=0.3) 12 | features = random_contrast(features, 0, 3, max_delta=0.3) 13 | 14 | adjs_rad = random_flip_left_right_adjs(adjs_rad) 15 | 16 | batch_augmented.append((features, adjs_dist, adjs_rad, label)) 17 | 18 | return batch_augmented 19 | 20 | 21 | def flip_left_right_adj(adj_rad): 22 | adj_rad.data = 2 * PI - adj_rad.data 23 | return adj_rad 24 | 25 | 26 | def random_flip_left_right_adjs(adjs_rad, rand=None): 27 | rand = bool(random.getrandbits(1)) if rand is None else rand 28 | return [flip_left_right_adj(adj) for adj in adjs_rad] if rand else adjs_rad 29 | 30 | 31 | def adjust_brightness(features, start_idx, end_idx, delta): 32 | colors = features[:, start_idx:end_idx] 33 | colors = colors + delta 34 | colors = np.clip(colors, 0, 1) 35 | 36 | return np.concatenate( 37 | (features[:, :start_idx], colors, features[:, end_idx:]), axis=1) 38 | 39 | 40 | def random_brightness(features, start_idx, end_idx, max_delta): 41 | rand = random.uniform(-max_delta, max_delta) 42 | return adjust_brightness(features, start_idx, end_idx, rand) 43 | 44 | 45 | def adjust_contrast(features, start_idx, end_idx, delta): 46 | colors = features[:, start_idx:end_idx] 47 | mean = colors.mean(axis=0) 48 | colors = (colors - mean) * (1 + delta) + mean 49 | colors = np.clip(colors, 0, 1) 50 | 51 | return np.concatenate( 52 | (features[:, :start_idx], colors, features[:, end_idx:]), axis=1) 53 | 54 | 55 | def random_contrast(features, start_idx, end_idx, max_delta): 56 | rand = random.uniform(-max_delta, max_delta) 57 | return adjust_contrast(features, start_idx, end_idx, rand) 58 | -------------------------------------------------------------------------------- /lib/pipeline/augment_test.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | import numpy as np 4 | from numpy import pi as PI 5 | from numpy.testing import assert_equal, assert_almost_equal 6 | import scipy.sparse as sp 7 | 8 | from .augment import (flip_left_right_adj, random_flip_left_right_adjs, 9 | adjust_brightness, random_brightness, adjust_contrast, 10 | random_contrast, augment_batch) 11 | 12 | 13 | class AugmentTest(TestCase): 14 | def test_flip_left_right_adj(self): 15 | adj = [[0, 0.25 * PI, 0.75 * PI], [1.25 * PI, 0, 0], [1.75 * PI, 0, 0]] 16 | adj = sp.coo_matrix(adj) 17 | 18 | output = flip_left_right_adj(adj) 19 | 20 | expected = [[0, 1.75 * PI, 1.25 * PI], [0.75 * PI, 0, 0], 21 | [0.25 * PI, 0, 0]] 22 | assert_equal(output.toarray(), expected) 23 | 24 | # Flip left right is mutable, so we create a new adjacency matrix. 25 | adj = [[0, 0.25 * PI, 0.75 * PI], [1.25 * PI, 0, 0], [1.75 * PI, 0, 0]] 26 | adj = sp.coo_matrix(adj) 27 | 28 | assert_equal( 29 | random_flip_left_right_adjs([adj], True)[0].toarray(), expected) 30 | assert_equal( 31 | random_flip_left_right_adjs([adj], False)[0].toarray(), 32 | adj.toarray()) 33 | 34 | random = random_flip_left_right_adjs([adj])[0].toarray() 35 | adj = [[0, 0.25 * PI, 0.75 * PI], [1.25 * PI, 0, 0], [1.75 * PI, 0, 0]] 36 | 37 | self.assertTrue( 38 | np.array_equal(random, adj) or np.array_equal(random, expected)) 39 | 40 | def test_adjust_brightness(self): 41 | features = np.array([[0.2, 0.4, 0.6, 1], [0.3, 0.2, 0.5, 2]]) 42 | output = adjust_brightness(features, 0, 3, delta=0.5) 43 | 44 | assert_almost_equal(output, [[0.7, 0.9, 1, 1], [0.8, 0.7, 1, 2]]) 45 | 46 | self.assertGreaterEqual( 47 | random_brightness(features, 0, 3, 0.5).min(), 0) 48 | self.assertLessEqual(random_brightness(features, 0, 3, 0.5).min(), 1) 49 | 50 | def test_adjust_contrast(self): 51 | features = np.array([[0.2, 0.4, 0.6, 1], [0.3, 0.2, 0.5, 2]]) 52 | output = adjust_contrast(features, 0, 3, delta=-0.5) 53 | 54 | assert_almost_equal(output, [[0.225, 0.35, 0.575, 1], 55 | [0.275, 0.25, 0.525, 2]]) 56 | 57 | self.assertGreaterEqual(random_contrast(features, 0, 3, 0.5).min(), 0) 58 | self.assertLessEqual(random_contrast(features, 0, 3, 0.5).min(), 1) 59 | 60 | def test_augment_batch(self): 61 | features = np.array([[0.2, 0.4], [0.3, 0.2], [0.8, 0.9]]) 62 | adj_dist = [[0, 1, 2], [1, 0, 0], [2, 0, 0]] 63 | adj_dist = sp.coo_matrix(adj_dist) 64 | adj_rad = [[0, 0.25 * PI, 0.75 * PI], [1.25 * PI, 0, 0], 65 | [1.75 * PI, 0, 0]] 66 | adj_rad = sp.coo_matrix(adj_rad) 67 | label = np.array([0, 0, 1]) 68 | 69 | batch = augment_batch([(features, [adj_dist], [adj_rad], label)]) 70 | 71 | expected = [[0, 1.75 * PI, 1.25 * PI], [0.75 * PI, 0, 0], 72 | [0.25 * PI, 0, 0]] 73 | adj_rad = [[0, 0.25 * PI, 0.75 * PI], [1.25 * PI, 0, 0], 74 | [1.75 * PI, 0, 0]] 75 | 76 | self.assertGreaterEqual(batch[0][0].min(), 0) 77 | self.assertLessEqual(batch[0][0].max(), 1) 78 | assert_equal(batch[0][1][0].toarray(), adj_dist.toarray()) 79 | 80 | self.assertTrue( 81 | np.array_equal(batch[0][2][0].toarray(), expected) or 82 | np.array_equal(batch[0][2][0].toarray(), adj_rad)) 83 | 84 | assert_equal(batch[0][3], label) 85 | -------------------------------------------------------------------------------- /lib/pipeline/dataset.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from __future__ import division 3 | 4 | import os 5 | import sys 6 | from six.moves import xrange 7 | 8 | import numpy as np 9 | 10 | 11 | def _print_status(data_dir, percentage): 12 | sys.stdout.write( 13 | '\r>> Preprocessing to {} {:.2f}%'.format(data_dir, percentage)) 14 | sys.stdout.flush() 15 | 16 | 17 | def _save(data_dir, name, data): 18 | np.save(os.path.join(data_dir, name), data) 19 | 20 | 21 | def _load(data_dir, names): 22 | batch = [] 23 | for name in names: 24 | path = os.path.join(data_dir, name) 25 | batch.append(np.load(path)) 26 | return batch 27 | 28 | 29 | class PreprocessedDataset(object): 30 | def __init__(self, data_dir, dataset, preprocess_algorithm): 31 | self._data_dir = data_dir 32 | self.epochs_completed = 0 33 | self._index_in_epoch = 0 34 | 35 | if os.path.exists(data_dir): 36 | self._names = os.listdir(data_dir) 37 | else: 38 | os.makedirs(data_dir) 39 | num_count = len(str(dataset.num_examples)) 40 | self._names = [ 41 | '{}.npy'.format(str(i).zfill(num_count)) 42 | for i in xrange(dataset.num_examples) 43 | ] 44 | 45 | num_left = dataset.num_examples 46 | batch_size = 25 47 | 48 | j = 0 49 | while num_left > 0: 50 | min_batch = min(batch_size, num_left) 51 | images, labels = dataset.next_batch(min_batch, shuffle=False) 52 | num_left -= min_batch 53 | 54 | for i in xrange(labels.shape[0]): 55 | data = preprocess_algorithm(images[i]) 56 | if isinstance(data, np.ndarray): 57 | data = (data, labels[i]) 58 | else: 59 | data = data + (labels[i], ) 60 | _save(data_dir, self._names[j], data) 61 | j += 1 62 | _print_status(data_dir, 63 | 100 * (1 - num_left / dataset.num_examples)) 64 | 65 | _print_status(data_dir, 100) 66 | print() 67 | 68 | @property 69 | def num_examples(self): 70 | return len(self._names) 71 | 72 | def _random_shuffle_examples(self): 73 | perm = np.arange(self.num_examples) 74 | np.random.shuffle(perm) 75 | self._names = [self._names[i] for i in perm] 76 | 77 | def next_batch(self, batch_size, shuffle=True): 78 | start = self._index_in_epoch 79 | 80 | # Shuffle for the first epoch. 81 | if self.epochs_completed == 0 and start == 0 and shuffle: 82 | self._random_shuffle_examples() 83 | 84 | if start + batch_size > self.num_examples: 85 | # Finished epoch. 86 | self.epochs_completed += 1 87 | 88 | # Get the rest examples in this epoch. 89 | rest_num_examples = self.num_examples - start 90 | names_rest = self._names[start:self.num_examples] 91 | 92 | # Shuffle the examples. 93 | if shuffle: 94 | self._random_shuffle_examples() 95 | 96 | # Start next epoch. 97 | start = 0 98 | self._index_in_epoch = batch_size - rest_num_examples 99 | end = self._index_in_epoch 100 | names = names_rest + self._names[start:end] 101 | else: 102 | # Just slice the examples. 103 | self._index_in_epoch += batch_size 104 | end = self._index_in_epoch 105 | names = self._names[start:end] 106 | 107 | return _load(self._data_dir, names) 108 | -------------------------------------------------------------------------------- /lib/pipeline/dataset_test.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | from unittest import TestCase 3 | 4 | from numpy.testing import assert_equal 5 | 6 | from .dataset import PreprocessedDataset 7 | from ..datasets.mnist import MNIST 8 | from ..segmentation.algorithm import slic_fixed 9 | from ..segmentation.feature_extraction import extract_features_fixed 10 | from ..pipeline import preprocess_pipeline_fixed 11 | 12 | data_dir = '/tmp/preprocessed_dataset_test' 13 | 14 | # Load MNIST dataset and reduce size to 50 examples. 15 | mnist = MNIST('data/mnist').train 16 | mnist._images = mnist._images[:50] 17 | mnist._labels = mnist._labels[:50] 18 | 19 | segmentation_algorithm = slic_fixed( 20 | num_segments=100, compactness=5, max_iterations=10, sigma=0) 21 | feature_extraction_algorithm = extract_features_fixed([0, 1, 2]) 22 | preprocess_algorithm = preprocess_pipeline_fixed( 23 | segmentation_algorithm, feature_extraction_algorithm, levels=4) 24 | 25 | 26 | def preprocess_algorithm_numpy(image): 27 | return image 28 | 29 | 30 | class PreprocessedDatasetTest(TestCase): 31 | def test_init(self): 32 | # Test creating and loading. 33 | PreprocessedDataset(data_dir, mnist, preprocess_algorithm) 34 | dataset = PreprocessedDataset(data_dir, mnist, preprocess_algorithm) 35 | 36 | self.assertEqual(dataset.num_examples, 50) 37 | 38 | shutil.rmtree(data_dir) 39 | 40 | def test_next_batch(self): 41 | dataset = PreprocessedDataset(data_dir, mnist, preprocess_algorithm) 42 | batch = dataset.next_batch(10, shuffle=False) 43 | self.assertEqual(len(batch), 10) 44 | self.assertEqual(len(batch[0]), 4) 45 | self.assertEqual(batch[0][0].shape[1], 4) 46 | self.assertEqual(len(batch[0][1]), 5) 47 | self.assertEqual(len(batch[0][2]), 5) 48 | assert_equal(batch[0][3], [0, 0, 0, 0, 0, 0, 0, 1, 0, 0]) 49 | 50 | dataset.next_batch(dataset.num_examples - 10, shuffle=False) 51 | 52 | batch = dataset.next_batch(dataset.num_examples, shuffle=False) 53 | self.assertEqual(len(batch), 50) 54 | self.assertEqual(len(batch[0]), 4) 55 | self.assertEqual(batch[0][0].shape[1], 4) 56 | self.assertEqual(len(batch[0][1]), 5) 57 | self.assertEqual(len(batch[0][2]), 5) 58 | assert_equal(batch[0][3], [0, 0, 0, 0, 0, 0, 0, 1, 0, 0]) 59 | 60 | shutil.rmtree(data_dir) 61 | 62 | def test_next_batch_shuffle(self): 63 | dataset = PreprocessedDataset(data_dir, mnist, preprocess_algorithm) 64 | 65 | batch = dataset.next_batch(10, shuffle=True) 66 | self.assertEqual(len(batch), 10) 67 | self.assertEqual(len(batch[0]), 4) 68 | 69 | dataset.next_batch(dataset.num_examples - 10, shuffle=True) 70 | 71 | batch = dataset.next_batch(dataset.num_examples, shuffle=True) 72 | self.assertEqual(len(batch), 50) 73 | self.assertEqual(len(batch[0]), 4) 74 | 75 | shutil.rmtree(data_dir) 76 | 77 | def test_with_numpy_preprocess(self): 78 | dataset = PreprocessedDataset('/tmp/preprocessed_dataset_test_numpy', 79 | mnist, preprocess_algorithm_numpy) 80 | 81 | batch = dataset.next_batch(10, shuffle=False) 82 | self.assertEqual(len(batch), 10) 83 | self.assertEqual(batch[0][0].shape, (28, 28, 1)) 84 | self.assertEqual(batch[0][1].shape, (10, )) 85 | -------------------------------------------------------------------------------- /lib/pipeline/file_queue.py: -------------------------------------------------------------------------------- 1 | from __future__ import division 2 | 3 | from threading import Thread, Event 4 | 5 | try: 6 | from queue import Queue 7 | except ImportError: 8 | from Queue import Queue 9 | 10 | 11 | class FileQueue(object): 12 | def __init__(self, dataset, batch_size, capacity, shuffle=False): 13 | 14 | inputs = Queue(capacity // batch_size) 15 | stopper = Event() 16 | 17 | class ProducerThread(Thread): 18 | def run(self): 19 | while not stopper.isSet(): 20 | batch = dataset.next_batch(batch_size, shuffle) 21 | inputs.put(batch) 22 | 23 | self._producer = ProducerThread() 24 | self._producer.start() 25 | self._inputs = inputs 26 | self._stopper = stopper 27 | 28 | def dequeue(self): 29 | batch = self._inputs.get() 30 | self._inputs.task_done() 31 | return batch 32 | 33 | def close(self): 34 | self._stopper.set() 35 | 36 | # Delete all batches in queue. 37 | while not self._inputs.empty(): 38 | self._inputs.get() 39 | self._inputs.task_done() 40 | self._inputs.join() 41 | 42 | # Shut down producer thread. 43 | self._producer.join() 44 | -------------------------------------------------------------------------------- /lib/pipeline/file_queue_test.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | import time 3 | 4 | from .file_queue import FileQueue 5 | from ..datasets import MNIST 6 | 7 | data = MNIST('data/mnist') 8 | 9 | 10 | class QueueTest(TestCase): 11 | def test_dequeue(self): 12 | queue = FileQueue(data.train, batch_size=2, capacity=8, shuffle=False) 13 | 14 | batch = queue.dequeue() 15 | 16 | self.assertEqual(len(batch), 2) 17 | self.assertEqual(batch[0].shape, (2, 28, 28, 1)) 18 | self.assertEqual(batch[1].shape, (2, 10)) 19 | 20 | # Ensure items in inputs queues before closing. 21 | time.sleep(5) 22 | queue.close() 23 | -------------------------------------------------------------------------------- /lib/pipeline/preprocess.py: -------------------------------------------------------------------------------- 1 | from ..segmentation import segmentation_adjacency 2 | from ..graph import coarsen_adj, perm_features 3 | 4 | 5 | def preprocess_pipeline(image, 6 | segmentation_algorithm, 7 | feature_extraction_algorithm, 8 | levels, 9 | connectivity=4, 10 | scale_invariance=False, 11 | stddev=1): 12 | 13 | segmentation = segmentation_algorithm(image) 14 | adj, points, mass = segmentation_adjacency(segmentation, connectivity) 15 | features = feature_extraction_algorithm(segmentation, image) 16 | 17 | adjs_dist, adjs_rad, perm = coarsen_adj(adj, points, mass, levels, 18 | scale_invariance, stddev) 19 | 20 | features = perm_features(features, perm) 21 | 22 | return features, adjs_dist, adjs_rad 23 | 24 | 25 | def preprocess_pipeline_fixed(segmentation_algorithm, 26 | feature_extraction_algorithm, 27 | levels, 28 | connectivity=4, 29 | scale_invariance=False, 30 | stddev=1): 31 | def _preprocess(image): 32 | return preprocess_pipeline(image, segmentation_algorithm, 33 | feature_extraction_algorithm, levels, 34 | connectivity, scale_invariance, stddev) 35 | 36 | return _preprocess 37 | -------------------------------------------------------------------------------- /lib/segmentation/__init__.py: -------------------------------------------------------------------------------- 1 | from .adjacency import segmentation_adjacency 2 | from .form_feature_extraction import FormFeatureExtraction 3 | from .feature_extraction import extract_features, extract_features_fixed 4 | from .form_feature_selection import FormFeatureSelection 5 | from .algorithm import (slic, slic_fixed, quickshift, quickshift_fixed, 6 | felzenszwalb, felzenszwalb_fixed) 7 | 8 | __all__ = [ 9 | 'segmentation_adjacency', 10 | 'FormFeatureExtraction', 11 | 'FormFeatureSelection', 12 | 'extract_features', 13 | 'extract_features_fixed', 14 | 'slic', 15 | 'slic_fixed', 16 | 'quickshift', 17 | 'quickshift_fixed', 18 | 'felzenszwalb', 19 | 'felzenszwalb_fixed', 20 | ] 21 | -------------------------------------------------------------------------------- /lib/segmentation/adjacency.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import numpy_groupies as npg 3 | import scipy.sparse as sp 4 | 5 | 6 | def segmentation_adjacency(segmentation, connectivity=4): 7 | """Generate an adjacency matrix out of a given segmentation.""" 8 | 9 | assert connectivity == 4 or connectivity == 8 10 | 11 | # Get centroids. 12 | idx = np.indices(segmentation.shape) 13 | ys = npg.aggregate(segmentation.flatten(), idx[0].flatten(), func='mean') 14 | xs = npg.aggregate(segmentation.flatten(), idx[1].flatten(), func='mean') 15 | ys = np.reshape(ys, (-1, 1)) 16 | xs = np.reshape(xs, (-1, 1)) 17 | points = np.concatenate((ys, xs), axis=1) 18 | 19 | # Get mass. 20 | nums, mass = np.unique(segmentation, return_counts=True) 21 | n = nums.shape[0] 22 | 23 | # Get adjacency (https://goo.gl/y1xFMq). 24 | tmp = np.zeros((n, n), np.bool) 25 | 26 | # Get vertically adjacency. 27 | a, b = segmentation[:-1, :], segmentation[1:, :] 28 | tmp[a[a != b], b[a != b]] = True 29 | 30 | # Get horizontally adjacency. 31 | a, b = segmentation[:, :-1], segmentation[:, 1:] 32 | tmp[a[a != b], b[a != b]] = True 33 | 34 | # Get diagonal adjacency. 35 | if connectivity == 8: 36 | a, b = segmentation[:-1, :-1], segmentation[1:, 1:] 37 | tmp[a[a != b], b[a != b]] = True 38 | 39 | a, b = segmentation[:-1, 1:], segmentation[1:, :-1] 40 | tmp[a[a != b], b[a != b]] = True 41 | 42 | result = tmp | tmp.T 43 | result = result.astype(np.uint8) 44 | adj = sp.coo_matrix(result) 45 | 46 | return adj, points, mass 47 | -------------------------------------------------------------------------------- /lib/segmentation/adjacency_test.py: -------------------------------------------------------------------------------- 1 | from __future__ import division 2 | 3 | from unittest import TestCase 4 | 5 | import numpy as np 6 | from numpy.testing import assert_equal 7 | 8 | from .adjacency import segmentation_adjacency 9 | 10 | 11 | class AdjacencyTest(TestCase): 12 | def test_segmentation_adjacency_simple_4(self): 13 | segmentation = np.array([ 14 | [0, 0, 1, 1], 15 | [2, 2, 3, 3], 16 | ]) 17 | adj, points, mass = segmentation_adjacency(segmentation) 18 | 19 | assert_equal(adj.toarray(), [ 20 | [0, 1, 1, 0], 21 | [1, 0, 0, 1], 22 | [1, 0, 0, 1], 23 | [0, 1, 1, 0], 24 | ]) 25 | assert_equal(points, [[0, 0.5], [0, 2.5], [1, 0.5], [1, 2.5]]) 26 | assert_equal(mass, [2, 2, 2, 2]) 27 | 28 | def test_segmentation_adjacency_complex_4(self): 29 | segmentation = np.array([ 30 | [0, 1, 1, 4, 6, 6], 31 | [0, 0, 1, 4, 6, 7], 32 | [0, 3, 1, 5, 5, 7], 33 | [0, 2, 2, 2, 5, 7], 34 | [8, 8, 2, 5, 5, 9], 35 | [8, 8, 8, 9, 9, 9], 36 | ]) 37 | adj, points, mass = segmentation_adjacency(segmentation) 38 | 39 | expected_adj = [ 40 | [0, 1, 1, 1, 0, 0, 0, 0, 1, 0], 41 | [1, 0, 1, 1, 1, 1, 0, 0, 0, 0], 42 | [1, 1, 0, 1, 0, 1, 0, 0, 1, 0], 43 | [1, 1, 1, 0, 0, 0, 0, 0, 0, 0], 44 | [0, 1, 0, 0, 0, 1, 1, 0, 0, 0], 45 | [0, 1, 1, 0, 1, 0, 1, 1, 0, 1], 46 | [0, 0, 0, 0, 1, 1, 0, 1, 0, 0], 47 | [0, 0, 0, 0, 0, 1, 1, 0, 0, 1], 48 | [1, 0, 1, 0, 0, 0, 0, 0, 0, 1], 49 | [0, 0, 0, 0, 0, 1, 0, 1, 1, 0], 50 | ] 51 | 52 | expected_points = [[7 / 5, 1 / 5], [3 / 4, 7 / 4], [13 / 4, 8 / 4], 53 | [2, 1], [1 / 2, 6 / 2], [15 / 5, 54 | 18 / 5], [1 / 3, 13 / 3], 55 | [6 / 3, 15 / 3], [23 / 5, 4 / 5], [19 / 4, 17 / 4]] 56 | 57 | assert_equal(adj.toarray(), expected_adj) 58 | assert_equal(points, expected_points) 59 | assert_equal(mass, [5, 4, 4, 1, 2, 5, 3, 3, 5, 4]) 60 | 61 | def test_segmentation_adjacency_simple_8(self): 62 | segmentation = np.array([ 63 | [0, 0, 1, 1], 64 | [2, 2, 3, 3], 65 | ]) 66 | adj, points, mass = segmentation_adjacency(segmentation, 8) 67 | 68 | assert_equal(adj.toarray(), [ 69 | [0, 1, 1, 1], 70 | [1, 0, 1, 1], 71 | [1, 1, 0, 1], 72 | [1, 1, 1, 0], 73 | ]) 74 | 75 | def test_segmentation_adjacency_complex_8(self): 76 | segmentation = np.array([ 77 | [0, 1, 1, 4, 6, 6], 78 | [0, 0, 1, 4, 6, 7], 79 | [0, 3, 1, 5, 5, 7], 80 | [0, 2, 2, 2, 5, 7], 81 | [8, 8, 2, 5, 5, 9], 82 | [8, 8, 8, 9, 9, 9], 83 | ]) 84 | adj, points, mass = segmentation_adjacency(segmentation, 8) 85 | 86 | expected_adj = [ 87 | [0, 1, 1, 1, 0, 0, 0, 0, 1, 0], 88 | [1, 0, 1, 1, 1, 1, 0, 0, 0, 0], 89 | [1, 1, 0, 1, 0, 1, 0, 0, 1, 1], 90 | [1, 1, 1, 0, 0, 0, 0, 0, 0, 0], 91 | [0, 1, 0, 0, 0, 1, 1, 0, 0, 0], 92 | [0, 1, 1, 0, 1, 0, 1, 1, 1, 1], 93 | [0, 0, 0, 0, 1, 1, 0, 1, 0, 0], 94 | [0, 0, 0, 0, 0, 1, 1, 0, 0, 1], 95 | [1, 0, 1, 0, 0, 1, 0, 0, 0, 1], 96 | [0, 0, 1, 0, 0, 1, 0, 1, 1, 0], 97 | ] 98 | 99 | assert_equal(adj.toarray(), expected_adj) 100 | -------------------------------------------------------------------------------- /lib/segmentation/algorithm.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from skimage import color, segmentation 3 | 4 | 5 | def slic(image, num_segments, compactness=10, max_iterations=10, sigma=0): 6 | image = _preprocess(image) 7 | return segmentation.slic(image, num_segments, compactness, max_iterations, 8 | sigma) 9 | 10 | 11 | def slic_fixed(num_segments, compactness=10, max_iterations=10, sigma=0): 12 | def slic_image(image): 13 | return slic(image, num_segments, compactness, max_iterations, sigma) 14 | 15 | return slic_image 16 | 17 | 18 | def quickshift(image, ratio=1, kernel_size=5, max_dist=1, sigma=0): 19 | image = _preprocess(image) 20 | return segmentation.quickshift( 21 | image, ratio, kernel_size, max_dist, sigma=sigma) 22 | 23 | 24 | def quickshift_fixed(ratio=1, kernel_size=5, max_dist=1, sigma=0): 25 | def quickshift_image(image): 26 | return quickshift(image, ratio, kernel_size, max_dist, sigma) 27 | 28 | return quickshift_image 29 | 30 | 31 | def felzenszwalb(image, scale=1, min_size=1, sigma=0): 32 | image = _preprocess(image) 33 | return segmentation.felzenszwalb(image, scale, sigma, min_size) 34 | 35 | 36 | def felzenszwalb_fixed(scale=1, min_size=1, sigma=0): 37 | def felzenszwalb_image(image): 38 | return felzenszwalb(image, scale, min_size, sigma) 39 | 40 | return felzenszwalb_image 41 | 42 | 43 | def _preprocess(image): 44 | if len(image.shape) == 2 or image.shape[2] == 1: 45 | return color.gray2rgb( 46 | np.reshape(image, (image.shape[0], image.shape[1]))) 47 | else: 48 | return image 49 | -------------------------------------------------------------------------------- /lib/segmentation/algorithm_test.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | import numpy as np 4 | from numpy.testing import assert_equal 5 | 6 | from .algorithm import (slic, slic_fixed, quickshift, quickshift_fixed, 7 | felzenszwalb, felzenszwalb_fixed) 8 | from ..datasets import MNIST, Cifar10 9 | 10 | mnist = MNIST('data/mnist') 11 | cifar_10 = Cifar10('data/cifar_10') 12 | 13 | 14 | class AlgorithmTest(TestCase): 15 | def test_slic(self): 16 | # Test grayscaled image. 17 | image = mnist.test.next_batch(1, shuffle=False)[0][0] 18 | segmentation = slic( 19 | image, num_segments=100, compactness=5, max_iterations=10, sigma=0) 20 | 21 | idx = np.unique(segmentation) 22 | assert_equal(idx, np.arange(segmentation.max() + 1)) 23 | 24 | alg = slic_fixed(num_segments=100, compactness=5, max_iterations=10) 25 | assert_equal(alg(image), segmentation) 26 | 27 | # Test colorized image. 28 | image = cifar_10.test.next_batch(1, shuffle=False)[0][0] 29 | segmentation = alg(image) 30 | idx = np.unique(segmentation) 31 | assert_equal(idx, np.arange(segmentation.max() + 1)) 32 | 33 | def test_quickshift(self): 34 | # Test grayscaled image. 35 | image = mnist.test.next_batch(1, shuffle=False)[0][0] 36 | segmentation = quickshift( 37 | image, ratio=1, kernel_size=2, max_dist=2, sigma=0) 38 | 39 | idx = np.unique(segmentation) 40 | assert_equal(idx, np.arange(segmentation.max() + 1)) 41 | 42 | alg = quickshift_fixed(ratio=1, kernel_size=2, max_dist=2) 43 | assert_equal(alg(image), segmentation) 44 | 45 | # Test colorized image. 46 | image = cifar_10.test.next_batch(1, shuffle=False)[0][0] 47 | segmentation = alg(image) 48 | idx = np.unique(segmentation) 49 | assert_equal(idx, np.arange(segmentation.max() + 1)) 50 | 51 | def test_felzenszwalb(self): 52 | # Test grayscaled image. 53 | image = mnist.test.next_batch(1, shuffle=False)[0][0] 54 | segmentation = felzenszwalb(image, scale=1, min_size=1) 55 | 56 | idx = np.unique(segmentation) 57 | assert_equal(idx, np.arange(segmentation.max() + 1)) 58 | 59 | alg = felzenszwalb_fixed(scale=1, min_size=1) 60 | assert_equal(alg(image), segmentation) 61 | 62 | # Test colorized image. 63 | image = cifar_10.test.next_batch(1, shuffle=False)[0][0] 64 | segmentation = alg(image) 65 | idx = np.unique(segmentation) 66 | assert_equal(idx, np.arange(segmentation.max() + 1)) 67 | -------------------------------------------------------------------------------- /lib/segmentation/feature_extraction.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import numpy_groupies as npg 3 | 4 | from .form_feature_extraction import FormFeatureExtraction 5 | 6 | 7 | def extract_features(segmentation, image, form_features=None): 8 | features = FormFeatureExtraction(segmentation).get_features(form_features) 9 | 10 | group_idx = segmentation.flatten() 11 | 12 | # Prepend mean color to form features. 13 | if image.shape[2] == 1: 14 | mean = npg.aggregate(group_idx, image.flatten(), func='mean') 15 | mean = np.reshape(mean, (-1, 1)) 16 | features = np.concatenate((mean, features), axis=1) 17 | elif image.shape[2] == 3: 18 | r = npg.aggregate(group_idx, image[:, :, 0:1].flatten(), func='mean') 19 | r = np.reshape(r, (-1, 1)) 20 | g = npg.aggregate(group_idx, image[:, :, 1:2].flatten(), func='mean') 21 | g = np.reshape(g, (-1, 1)) 22 | b = npg.aggregate(group_idx, image[:, :, 2:3].flatten(), func='mean') 23 | b = np.reshape(b, (-1, 1)) 24 | features = np.concatenate((r, g, b, features), axis=1) 25 | else: 26 | raise ValueError 27 | 28 | return features.astype(np.float32) 29 | 30 | 31 | def extract_features_fixed(form_features=None): 32 | def _extract(segmentation, image): 33 | return extract_features(segmentation, image, form_features) 34 | 35 | return _extract 36 | -------------------------------------------------------------------------------- /lib/segmentation/feature_extraction_test.py: -------------------------------------------------------------------------------- 1 | from __future__ import division 2 | 3 | from unittest import TestCase 4 | 5 | import numpy as np 6 | from numpy.testing import assert_almost_equal 7 | 8 | from .feature_extraction import extract_features, extract_features_fixed 9 | 10 | 11 | class FeatureExtractionTest(TestCase): 12 | def test_extract_features(self): 13 | segmentation = np.array([[0, 0, 1, 1], [0, 0, 1, 1]]) 14 | 15 | gray = np.array([[[0.1], [0.5], [0.1], [0.4]], [[0.3], [0.8], [0.1], 16 | [0.0]]]) 17 | 18 | features = extract_features(segmentation, gray, [0, 2, 3]) 19 | expected = np.array([[1.7 / 4, 4, 2, 2], [0.6 / 4, 4, 2, 2]]) 20 | assert_almost_equal(features, expected) 21 | 22 | rgb = np.array([[[0.1, 0.2, 0.4], [0.5, 0.3, 0.6], [0.1, 0.2, 1.0], 23 | [0.4, 0.9, 1.0]], [[0.3, 0.4, 0.2], [0.8, 0.4, 0.2], 24 | [0.1, 0.4, 0.6], [0.0, 0.0, 1.0]]]) 25 | 26 | features = extract_features(segmentation, rgb, [0, 2, 3]) 27 | expected = [[1.7 / 4, 1.3 / 4, 1.4 / 4, 4, 2, 2], 28 | [0.6 / 4, 1.5 / 4, 3.6 / 4, 4, 2, 2]] 29 | assert_almost_equal(features, expected) 30 | 31 | def test_extract_features_fixed(self): 32 | segmentation = np.array([[0, 0, 1, 1], [0, 0, 1, 1]]) 33 | 34 | gray = np.array([[[0.1], [0.5], [0.1], [0.4]], [[0.3], [0.8], [0.1], 35 | [0.0]]]) 36 | 37 | extract = extract_features_fixed([0, 2, 3]) 38 | features = extract(segmentation, gray) 39 | expected = [[1.7 / 4, 4, 2, 2], [0.6 / 4, 4, 2, 2]] 40 | assert_almost_equal(features, expected) 41 | -------------------------------------------------------------------------------- /lib/segmentation/form_feature_extraction_test.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | import numpy as np 4 | from skimage.measure import regionprops 5 | from numpy.testing import assert_equal, assert_almost_equal 6 | 7 | from .form_feature_extraction import FormFeatureExtraction 8 | 9 | 10 | def _convert(props, key): 11 | p = np.array([prop[key] for prop in props]) 12 | if (p.ndim > 0): 13 | p = p.T 14 | return p 15 | 16 | 17 | class FormFeatureExtractionTest(TestCase): 18 | def test_feature_extraction_helper(self): 19 | segmentation = np.array([[0, 0, 1, 1], [0, 0, 1, 1]]) 20 | features = FormFeatureExtraction(segmentation) 21 | props = regionprops(segmentation + 1) 22 | 23 | # Test private helper 24 | assert_equal(features._group_idx, [0, 1]) 25 | assert_equal(features._min_y, [0, 0]) 26 | assert_equal(features._max_y, [2, 2]) 27 | assert_equal(features._min_x, [0, 2]) 28 | assert_equal(features._max_x, [2, 4]) 29 | 30 | assert_equal(features._M_00, [4, 4]) 31 | assert_equal(features._M_01, [2, 2]) 32 | assert_equal(features._M_10, [2, 10]) 33 | assert_equal(features._M_11, [1, 5]) 34 | assert_equal(features._M_02, [2, 2]) 35 | assert_equal(features._M_20, [2, 26]) 36 | assert_equal(features._M_12, [1, 5]) 37 | assert_equal(features._M_21, [1, 13]) 38 | assert_equal(features._M_03, [2, 2]) 39 | assert_equal(features._M_30, [2, 70]) 40 | 41 | assert_equal(features._centroid_y, _convert(props, 'centroid')[0]) 42 | assert_equal(features._centroid_x, _convert(props, 'centroid')[1]) 43 | 44 | def test_feature_extraction(self): 45 | segmentation = np.array([[0, 1, 1, 4, 6, 6], [0, 0, 1, 4, 6, 7], 46 | [0, 3, 1, 5, 5, 7], [0, 2, 2, 2, 5, 7], 47 | [8, 8, 2, 5, 5, 9], [8, 8, 8, 9, 9, 9]]) 48 | 49 | features = FormFeatureExtraction(segmentation) 50 | props = regionprops(segmentation + 1) 51 | 52 | moments_central = _convert(props, 'moments_central') 53 | assert_almost_equal(features.mu_11, moments_central[1, 1]) 54 | assert_almost_equal(features.mu_02, moments_central[2, 0]) 55 | assert_almost_equal(features.mu_20, moments_central[0, 2]) 56 | assert_almost_equal(features.mu_12, moments_central[2, 1]) 57 | assert_almost_equal(features.mu_21, moments_central[1, 2]) 58 | assert_almost_equal(features.mu_03, moments_central[3, 0]) 59 | assert_almost_equal(features.mu_30, moments_central[0, 3]) 60 | 61 | inertia_tensor = _convert(props, 'inertia_tensor') 62 | assert_almost_equal(features.inertia_tensor_02, inertia_tensor[1, 1]) 63 | assert_almost_equal(features.inertia_tensor_20, inertia_tensor[0, 0]) 64 | nu_11 = -1 * inertia_tensor[0, 1] 65 | assert_almost_equal(features.inertia_tensor_11, nu_11) 66 | nu_11 = -1 * inertia_tensor[1, 0] 67 | assert_almost_equal(features.inertia_tensor_11, nu_11) 68 | 69 | eigvals = _convert(props, 'inertia_tensor_eigvals') 70 | assert_almost_equal(features.inertia_tensor_eigvals_1, eigvals[0]) 71 | assert_almost_equal(features.inertia_tensor_eigvals_2, eigvals[1]) 72 | 73 | moments_normalized = _convert(props, 'moments_normalized') 74 | assert_almost_equal(features.nu_11, moments_normalized[1, 1]) 75 | assert_almost_equal(features.nu_02, moments_normalized[2, 0]) 76 | assert_almost_equal(features.nu_20, moments_normalized[0, 2]) 77 | assert_almost_equal(features.nu_12, moments_normalized[2, 1]) 78 | assert_almost_equal(features.nu_21, moments_normalized[1, 2]) 79 | assert_almost_equal(features.nu_03, moments_normalized[3, 0]) 80 | assert_almost_equal(features.nu_30, moments_normalized[0, 3]) 81 | 82 | moments_hu = _convert(props, 'moments_hu') 83 | assert_almost_equal(features.hu_1, moments_hu[0]) 84 | assert_almost_equal(features.hu_2, moments_hu[1]) 85 | assert_almost_equal(features.hu_3, moments_hu[2]) 86 | assert_almost_equal(features.hu_4, moments_hu[3]) 87 | assert_almost_equal(features.hu_5, moments_hu[4]) 88 | assert_almost_equal(features.hu_6, moments_hu[5]) 89 | assert_almost_equal(features.hu_7, moments_hu[6]) 90 | 91 | assert_equal(features.area, _convert(props, 'area')) 92 | bbox = _convert(props, 'bbox') 93 | assert_equal(features.bbox_height, bbox[2] - bbox[0]) 94 | assert_equal(features.bbox_width, bbox[3] - bbox[1]) 95 | assert_equal(features.bbox_area, 96 | (bbox[2] - bbox[0]) * (bbox[3] - bbox[1])) 97 | assert_almost_equal(features.centroid_y, 98 | _convert(props, 'local_centroid')[0]) 99 | assert_almost_equal(features.centroid_x, 100 | _convert(props, 'local_centroid')[1]) 101 | assert_almost_equal(features.eccentricity, 102 | _convert(props, 'eccentricity')) 103 | assert_equal(features.equivalent_diameter, 104 | _convert(props, 'equivalent_diameter')) 105 | assert_equal(features.extent, _convert(props, 'extent')) 106 | assert_almost_equal(features.major_axis_length, 107 | _convert(props, 'major_axis_length')) 108 | assert_almost_equal(features.minor_axis_length, 109 | _convert(props, 'minor_axis_length')) 110 | assert_almost_equal(features.orientation, 111 | _convert(props, 'orientation')) 112 | 113 | def test_methods(self): 114 | self.assertEqual(len(FormFeatureExtraction.methods), 38) 115 | 116 | expected = [ 117 | 'area', 'bbox_area', 'bbox_height', 'bbox_width', 'centroid_x', 118 | 'centroid_y', 'eccentricity', 'equivalent_diameter', 'extent', 119 | 'hu_1', 'hu_2', 'hu_3', 'hu_4', 'hu_5', 'hu_6', 'hu_7', 120 | 'inertia_tensor_02', 'inertia_tensor_11', 'inertia_tensor_20', 121 | 'inertia_tensor_eigvals_1', 'inertia_tensor_eigvals_2', 122 | 'major_axis_length', 'minor_axis_length', 'mu_02', 'mu_03', 123 | 'mu_11', 'mu_12', 'mu_20', 'mu_21', 'mu_30', 'nu_02', 'nu_03', 124 | 'nu_11', 'nu_12', 'nu_20', 'nu_21', 'nu_30', 'orientation' 125 | ] 126 | 127 | self.assertEqual(FormFeatureExtraction.methods, expected) 128 | 129 | def test_get_features(self): 130 | segmentation = np.array([[0, 0, 1, 1], [0, 0, 1, 1]]) 131 | features = FormFeatureExtraction(segmentation) 132 | 133 | f = features.get_features([0, 1, 2]) 134 | assert_equal(f, [[4, 4, 2], [4, 4, 2]]) 135 | 136 | f = features.get_features() 137 | self.assertEqual(f.shape, (2, 38)) 138 | -------------------------------------------------------------------------------- /lib/segmentation/form_feature_selection.py: -------------------------------------------------------------------------------- 1 | from __future__ import division 2 | from __future__ import print_function 3 | 4 | import sys 5 | from six.moves import xrange 6 | 7 | import numpy as np 8 | from sklearn.preprocessing import StandardScaler 9 | from sklearn.feature_selection import VarianceThreshold 10 | from sklearn.feature_selection import SelectKBest, RFE, f_classif 11 | from sklearn.svm import LinearSVC 12 | 13 | from .form_feature_extraction import FormFeatureExtraction 14 | 15 | 16 | def _print_status(percentage): 17 | sys.stdout.write('\r>> Collecting features {:.2f}%'.format(percentage)) 18 | sys.stdout.flush() 19 | 20 | 21 | class FormFeatureSelection(object): 22 | def __init__(self, 23 | dataset, 24 | segmentation_algorithm, 25 | num_examples=None, 26 | scaler=StandardScaler()): 27 | 28 | if num_examples is None: # pragma: no cover 29 | num_examples = dataset.num_examples 30 | 31 | images, labels = dataset.next_batch(num_examples, shuffle=False) 32 | 33 | self._features = [] 34 | self._labels = [] 35 | 36 | for i in xrange(num_examples): 37 | segmentation = segmentation_algorithm(images[i]) 38 | 39 | features = FormFeatureExtraction(segmentation).get_features() 40 | features = scaler.fit_transform(features) 41 | label = np.where(labels[i] == 1)[0][0] 42 | label = label.repeat(features.shape[0]) 43 | 44 | self._features.append(features) 45 | self._labels.append(label) 46 | 47 | _print_status(100 * i / num_examples) 48 | 49 | _print_status(100) 50 | print() 51 | 52 | self._features = np.concatenate(self._features) 53 | self._idx = np.arange(self._features.shape[1]) 54 | self._labels = np.concatenate(self._labels) 55 | 56 | @property 57 | def features(self): 58 | return self._features 59 | 60 | @property 61 | def num_features(self): 62 | return self._features.shape[1] 63 | 64 | @property 65 | def selected_feature_indices(self): 66 | return self._idx 67 | 68 | @property 69 | def selected_features(self): 70 | return [FormFeatureExtraction.methods[i] for i in self._idx] 71 | 72 | def remove_low_variance(self, threshold): 73 | sel = VarianceThreshold(threshold) 74 | self._features = sel.fit_transform(self._features) 75 | self._idx = self._idx[np.where(sel.get_support())[0]] 76 | 77 | def select_univariate(self, k): 78 | sel = SelectKBest(f_classif, k) 79 | self._features = sel.fit_transform(self._features, self._labels) 80 | self._idx = self._idx[np.where(sel.get_support())[0]] 81 | 82 | def eliminate_recursive(self, k): 83 | sel = RFE(LinearSVC(), k) 84 | self._features = sel.fit_transform(self._features, self._labels) 85 | self._idx = self._idx[np.where(sel.get_support())[0]] 86 | -------------------------------------------------------------------------------- /lib/segmentation/form_feature_selection_test.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | import numpy as np 4 | from numpy.testing import assert_equal 5 | 6 | from .form_feature_selection import FormFeatureSelection 7 | from ..datasets import MNIST 8 | from .algorithm import slic_fixed 9 | 10 | data = MNIST('data/mnist') 11 | slic = slic_fixed(num_segments=100, compactness=5, max_iterations=10) 12 | 13 | 14 | class FormFeatureSelectionTest(TestCase): 15 | def test_form_feature_selection(self): 16 | selector = FormFeatureSelection(data.train, slic, num_examples=10) 17 | 18 | self.assertEqual(selector.features.shape[1], 38) 19 | self.assertEqual(selector.num_features, 38) 20 | self.assertEqual(len(selector.selected_features), 38) 21 | assert_equal(selector.selected_feature_indices, np.arange(38)) 22 | 23 | # Remove low variance: no features removed, because we use standard 24 | # scaler as default. 25 | selector.remove_low_variance(0.1) 26 | self.assertEqual(selector.selected_feature_indices.shape[0], 38) 27 | 28 | # Univariate feature selection. 29 | selector.select_univariate(12) 30 | self.assertEqual(selector.selected_feature_indices.shape[0], 12) 31 | 32 | # Recursive feature eliminiation. 33 | selector.eliminate_recursive(9) 34 | self.assertEqual(selector.selected_feature_indices.shape[0], 9) 35 | -------------------------------------------------------------------------------- /lib/tf/__init__.py: -------------------------------------------------------------------------------- 1 | from .bspline import base 2 | from .convert import sparse_to_tensor 3 | from .laplacian import rescaled_laplacian 4 | from .math import sparse_tensor_diag_matmul 5 | 6 | __all__ = [ 7 | 'base', 8 | 'sparse_to_tensor', 9 | 'rescaled_laplacian', 10 | 'sparse_tensor_diag_matmul', 11 | ] 12 | -------------------------------------------------------------------------------- /lib/tf/bspline.py: -------------------------------------------------------------------------------- 1 | from __future__ import division 2 | 3 | import tensorflow as tf 4 | from numpy import pi as PI 5 | 6 | 7 | def base(adj_rad, K, P, p): 8 | assert p < P 9 | 10 | shape = adj_rad.dense_shape 11 | dtype = tf.as_dtype(adj_rad.values.dtype) 12 | 13 | c = P / (2 * PI) # Don't recalculate coefficient every time. 14 | values = adj_rad.values 15 | indices = adj_rad.indices 16 | 17 | if K == 1: 18 | values = c * values - p 19 | 20 | greater_zero = tf.greater(values, tf.zeros_like(values)) 21 | less_equal_one = tf.less_equal(values, tf.ones_like(values)) 22 | 23 | values = tf.logical_and(greater_zero, less_equal_one) 24 | values = tf.cast(values, dtype) 25 | 26 | return tf.SparseTensorValue(indices, values, shape) 27 | 28 | elif K == 2: 29 | zero = tf.SparseTensorValue( 30 | tf.constant(0, dtype=tf.int64, shape=[0, 2]), 31 | tf.constant(0, dtype, shape=[0]), shape) 32 | 33 | left_values = c * values - p 34 | left = tf.SparseTensorValue(indices, left_values, shape) 35 | left = tf.sparse_maximum(left, zero) 36 | 37 | right_values = -c * values + p + 2 38 | right = tf.SparseTensorValue(indices, right_values, shape) 39 | right = tf.sparse_maximum(right, zero) 40 | 41 | offset_right_values = right_values - P 42 | offset_right = tf.SparseTensorValue(indices, offset_right_values, 43 | shape) 44 | offset_right = tf.sparse_maximum(offset_right, zero) 45 | 46 | return tf.sparse_maximum(tf.sparse_minimum(left, right), offset_right) 47 | 48 | else: 49 | raise NotImplementedError 50 | -------------------------------------------------------------------------------- /lib/tf/convert.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | 4 | 5 | def sparse_to_tensor(value): 6 | """Convert a scipy sparse matrix to a tensorflow SparseTensorValue.""" 7 | 8 | row = np.reshape(value.row, (-1, 1)) 9 | col = np.reshape(value.col, (-1, 1)) 10 | indices = np.concatenate((row, col), axis=1) 11 | return tf.SparseTensorValue(indices, value.data, value.shape) 12 | -------------------------------------------------------------------------------- /lib/tf/convert_test.py: -------------------------------------------------------------------------------- 1 | import scipy.sparse as sp 2 | import tensorflow as tf 3 | 4 | from .convert import sparse_to_tensor 5 | 6 | 7 | class SparseTest(tf.test.TestCase): 8 | def test_sparse_to_tensor(self): 9 | value = [[0, 1, 0], [1, 0, 2], [0, 2, 0]] 10 | value = sp.coo_matrix(value) 11 | 12 | with self.test_session(): 13 | self.assertAllEqual( 14 | tf.sparse_tensor_to_dense(sparse_to_tensor(value)).eval(), 15 | value.toarray()) 16 | 17 | def test_sparse_feed_dict(self): 18 | value = [[0, 1, 0], [1, 0, 2], [0, 2, 0]] 19 | value = sp.coo_matrix(value) 20 | value = sparse_to_tensor(value) 21 | 22 | # Sparse placeholder is buggy and can't convert shape. 23 | # => Need to pass empty shape. 24 | placeholder = tf.sparse_placeholder(tf.float32) 25 | output = tf.sparse_tensor_to_dense(placeholder) 26 | 27 | expected = [[0, 1, 0], [1, 0, 2], [0, 2, 0]] 28 | 29 | with self.test_session() as sess: 30 | result = sess.run(output, feed_dict={placeholder: value}) 31 | self.assertAllEqual(result, expected) 32 | -------------------------------------------------------------------------------- /lib/tf/laplacian.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | from .math import sparse_tensor_diag_matmul, sparse_scalar_multiply 4 | 5 | 6 | def rescaled_laplacian(adj): 7 | """Creates a tensorflow (rescale) laplacian matrix out of a 8 | SparseTensorValue adjacency matrix.""" 9 | 10 | degree = tf.sparse_reduce_sum(adj, axis=1) 11 | degree = tf.cast(degree, tf.float32) 12 | degree = tf.pow(degree, -0.5) 13 | lap = sparse_tensor_diag_matmul(adj, degree, transpose=True) 14 | lap = sparse_tensor_diag_matmul(lap, degree, transpose=False) 15 | 16 | return sparse_scalar_multiply(lap, -1) 17 | -------------------------------------------------------------------------------- /lib/tf/laplacian_test.py: -------------------------------------------------------------------------------- 1 | from numpy.testing import assert_almost_equal 2 | import scipy.sparse as sp 3 | from scipy.sparse.csgraph import laplacian 4 | import tensorflow as tf 5 | 6 | from .laplacian import rescaled_laplacian 7 | from .convert import sparse_to_tensor 8 | 9 | 10 | class LaplacianTest(tf.test.TestCase): 11 | def test_rescaled_laplacian(self): 12 | adj = [[0, 1, 0], [1, 0, 2], [0, 2, 0]] 13 | adj = sp.coo_matrix(adj) 14 | adj_tf = sparse_to_tensor(adj) 15 | lap = rescaled_laplacian(adj_tf) 16 | lap = tf.sparse_tensor_to_dense(lap) 17 | 18 | expected = laplacian(adj, normed=True) - sp.eye(3) 19 | expected = expected.toarray() 20 | 21 | with self.test_session(): 22 | assert_almost_equal(lap.eval(), expected) 23 | -------------------------------------------------------------------------------- /lib/tf/math.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | 4 | 5 | def sparse_scalar_multiply(value, scalar): 6 | """Multiply a SparseTensorValue by a scalar.""" 7 | 8 | return tf.SparseTensorValue(value.indices, scalar * value.values, 9 | value.dense_shape) 10 | 11 | 12 | def sparse_tensor_diag_matmul(a, diag, transpose=False): 13 | """Multiply a SparseTensorValue with a diagonal matrix given its diagonal 14 | values.""" 15 | 16 | _py_func = _diag_matmul_py if not transpose else _diag_matmul_transpose_py 17 | values = tf.py_func( 18 | _py_func, [a.indices, a.values, diag], Tout=diag.dtype, stateful=False) 19 | 20 | return tf.SparseTensorValue(a.indices, values, a.dense_shape) 21 | 22 | 23 | def _diag_matmul_py(indices, values, diag): 24 | diag = diag[indices[:, 0:1]] 25 | diag = np.reshape(diag, (-1)) 26 | return np.multiply(values, diag).astype(diag.dtype) 27 | 28 | 29 | def _diag_matmul_transpose_py(indices, values, diag): 30 | diag = diag[indices[:, 1:2]] 31 | diag = np.reshape(diag, (-1)) 32 | return np.multiply(values, diag).astype(diag.dtype) 33 | -------------------------------------------------------------------------------- /lib/tf/math_test.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from numpy.testing import assert_equal 3 | import scipy.sparse as sp 4 | import tensorflow as tf 5 | 6 | from .math import (sparse_scalar_multiply, sparse_tensor_diag_matmul, 7 | _diag_matmul_py, _diag_matmul_transpose_py) 8 | from .convert import sparse_to_tensor 9 | 10 | 11 | class MathTest(tf.test.TestCase): 12 | def test_sparse_scalar_multiply(self): 13 | a = [[0, 2, 3], [0, 1, 0]] 14 | a = sp.coo_matrix(a) 15 | a = sparse_to_tensor(a) 16 | a = sparse_scalar_multiply(a, 2) 17 | a = tf.sparse_tensor_to_dense(a) 18 | expected = [[0, 4, 6], [0, 2, 0]] 19 | 20 | with self.test_session(): 21 | self.assertAllEqual(a.eval(), expected) 22 | 23 | def test_sparse_tensor_diag_matmul(self): 24 | a = [[2, 3, 0], [1, 0, 2], [0, 3, 0]] 25 | a = sp.coo_matrix(a) 26 | a = sparse_to_tensor(a) 27 | 28 | diag = [2, 0.5, 3] 29 | diag = tf.constant(diag) 30 | 31 | b = sparse_tensor_diag_matmul(a, diag) 32 | b = tf.sparse_tensor_to_dense(b) 33 | expected = [[4, 6, 0], [0.5, 0, 1], [0, 9, 0]] 34 | 35 | with self.test_session(): 36 | self.assertAllEqual(b.eval(), expected) 37 | 38 | b = sparse_tensor_diag_matmul(a, diag, transpose=True) 39 | b = tf.sparse_tensor_to_dense(b) 40 | expected = [[4, 1.5, 0], [2, 0, 6], [0, 1.5, 0]] 41 | 42 | with self.test_session(): 43 | self.assertAllEqual(b.eval(), expected) 44 | 45 | def test_diag_matmul_py(self): 46 | indices = np.array([[0, 0], [0, 1], [1, 0], [1, 2], [2, 1]]) 47 | values = np.array([2, 3, 1, 2, 3]) 48 | diag = np.array([2, 0.5, 3]) 49 | 50 | result = _diag_matmul_py(indices, values, diag) 51 | expected = [4, 6, 0.5, 1, 9] 52 | assert_equal(result, expected) 53 | 54 | def test_diag_matmul_transpose_py(self): 55 | indices = np.array([[1, 0], [0, 0], [0, 1], [1, 2], [2, 1]]) 56 | values = np.array([1, 2, 3, 2, 3]) 57 | diag = np.array([2, 0.5, 3]) 58 | 59 | result = _diag_matmul_transpose_py(indices, values, diag) 60 | expected = [2, 4, 1.5, 6, 1.5] 61 | assert_equal(result, expected) 62 | -------------------------------------------------------------------------------- /mnist_graph.py: -------------------------------------------------------------------------------- 1 | from lib.datasets import MNIST as Data 2 | from lib.model import Model as BaseModel, generate_placeholders, train 3 | from lib.segmentation import extract_features_fixed 4 | # from lib.segmentation import slic_fixed 5 | from lib.segmentation import quickshift_fixed 6 | from lib.pipeline import preprocess_pipeline_fixed 7 | from lib.layer import EmbeddedGCNN as Conv, MaxPool, AveragePool, FC 8 | 9 | # SLIC_FEATURES = [4, 5, 6, 7, 8, 18, 20, 21, 22] 10 | QUICKSHIFT_FEATURES = [4, 6, 7, 8, 24, 28, 29, 31, 37] 11 | 12 | DATA_DIR = 'data/mnist' 13 | 14 | # PREPROCESS_FIRST = 'data/mnist/slic' 15 | PREPROCESS_FIRST = 'data/mnist/quickshift' 16 | 17 | LEVELS = 4 18 | CONNECTIVITY = 8 19 | SCALE_INVARIANCE = False 20 | STDDEV = 1 21 | 22 | LEARNING_RATE = 0.001 23 | TRAIN_DIR = None 24 | # LOG_DIR = 'data/summaries/mnist_slic_graph' 25 | LOG_DIR = 'data/summaries/mnist_quickshift_graph' 26 | 27 | AUGMENT_TRAIN_EXAMPLES = False 28 | DROPOUT = 0.5 29 | BATCH_SIZE = 64 30 | MAX_STEPS = 15000 31 | DISPLAY_STEP = 10 32 | # FORM_FEATURES = SLIC_FEATURES 33 | FORM_FEATURES = QUICKSHIFT_FEATURES 34 | NUM_FEATURES = len(FORM_FEATURES) + 1 35 | 36 | data = Data(DATA_DIR) 37 | 38 | # segmentation_algorithm = slic_fixed( 39 | # num_segments=100, compactness=5, max_iterations=10, sigma=0) 40 | segmentation_algorithm = quickshift_fixed( 41 | ratio=1, kernel_size=2, max_dist=2, sigma=0) 42 | 43 | feature_extraction_algorithm = extract_features_fixed(FORM_FEATURES) 44 | 45 | preprocess_algorithm = preprocess_pipeline_fixed( 46 | segmentation_algorithm, feature_extraction_algorithm, LEVELS, CONNECTIVITY, 47 | SCALE_INVARIANCE, STDDEV) 48 | 49 | 50 | class Model(BaseModel): 51 | def _build(self): 52 | conv_1_1 = Conv( 53 | NUM_FEATURES, 54 | 64, 55 | adjs_dist=self.placeholders['adj_dist_1'], 56 | adjs_rad=self.placeholders['adj_rad_1'], 57 | logging=self.logging) 58 | conv_1_2 = Conv( 59 | 64, 60 | 64, 61 | adjs_dist=self.placeholders['adj_dist_1'], 62 | adjs_rad=self.placeholders['adj_rad_1'], 63 | logging=self.logging) 64 | max_pool_1 = MaxPool(size=4) 65 | conv_2_1 = Conv( 66 | 64, 67 | 128, 68 | adjs_dist=self.placeholders['adj_dist_3'], 69 | adjs_rad=self.placeholders['adj_rad_3'], 70 | logging=self.logging) 71 | conv_2_2 = Conv( 72 | 128, 73 | 128, 74 | adjs_dist=self.placeholders['adj_dist_3'], 75 | adjs_rad=self.placeholders['adj_rad_3'], 76 | logging=self.logging) 77 | max_pool_2 = MaxPool(size=4) 78 | average_pool = AveragePool() 79 | fc_1 = FC( 80 | 128, 81 | data.num_classes, 82 | act=lambda x: x, 83 | bias=False, 84 | dropout=self.placeholders['dropout'], 85 | logging=self.logging) 86 | 87 | self.layers = [ 88 | conv_1_1, conv_1_2, max_pool_1, conv_2_1, conv_2_2, max_pool_2, 89 | average_pool, fc_1 90 | ] 91 | 92 | 93 | placeholders = generate_placeholders(BATCH_SIZE, LEVELS, NUM_FEATURES, 94 | data.num_classes) 95 | 96 | model = Model( 97 | placeholders=placeholders, 98 | learning_rate=LEARNING_RATE, 99 | train_dir=TRAIN_DIR, 100 | log_dir=LOG_DIR) 101 | 102 | train(model, data, preprocess_algorithm, BATCH_SIZE, DROPOUT, 103 | AUGMENT_TRAIN_EXAMPLES, MAX_STEPS, PREPROCESS_FIRST, DISPLAY_STEP) 104 | -------------------------------------------------------------------------------- /mnist_spatial.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from __future__ import division 3 | 4 | from six.moves import xrange 5 | import os 6 | import time 7 | 8 | import tensorflow as tf 9 | import numpy as np 10 | from sklearn.preprocessing import StandardScaler 11 | 12 | from lib.datasets import MNIST as Data 13 | from lib.model import Model as BaseModel 14 | from lib.segmentation import segmentation_adjacency, extract_features_fixed 15 | # from lib.segmentation import slic_fixed 16 | from lib.segmentation import quickshift_fixed 17 | from lib.layer import SpatialCNN as Conv, FC 18 | from lib.graph import receptive_fields, fill_features 19 | from lib.pipeline import PreprocessedDataset, FileQueue 20 | 21 | # SLIC_FEATURES = [4, 5, 6, 7, 8, 18, 20, 21, 22] 22 | QUICKSHIFT_FEATURES = [4, 6, 7, 8, 24, 28, 29, 31, 37] 23 | 24 | DATA_DIR = 'data/mnist' 25 | 26 | # PREPROCESS_FIRST = 'data/mnist/slic_spatial' 27 | PREPROCESS_FIRST = 'data/mnist/quickshift_spatial' 28 | 29 | NODE_SIZE = 25 30 | NODE_STRIDE = 4 31 | DELTA = 3 32 | NEIGHBORHOOD_SIZE = 25 33 | CONNECTIVITY = 8 34 | 35 | LEARNING_RATE = 0.001 36 | TRAIN_DIR = None 37 | # LOG_DIR = 'data/summaries/mnist_slic_spatial' 38 | LOG_DIR = 'data/summaries/mnist_quickshift_spatial' 39 | SAVE_STEP = 250 40 | 41 | AUGMENT_TRAIN_EXAMPLES = False 42 | DROPOUT = 0.5 43 | BATCH_SIZE = 64 44 | MAX_STEPS = 15000 45 | DISPLAY_STEP = 10 46 | # FORM_FEATURES = SLIC_FEATURES 47 | FORM_FEATURES = QUICKSHIFT_FEATURES 48 | NUM_FEATURES = len(FORM_FEATURES) + 1 49 | 50 | data = Data(DATA_DIR) 51 | 52 | # segmentation_algorithm = slic_fixed( 53 | # num_segments=100, compactness=5, max_iterations=10, sigma=0) 54 | segmentation_algorithm = quickshift_fixed( 55 | ratio=1, kernel_size=2, max_dist=2, sigma=0) 56 | 57 | feature_extraction_algorithm = extract_features_fixed(FORM_FEATURES) 58 | 59 | 60 | def preprocess_spatial_fixed( 61 | segmentation_algorithm, feature_extraction_algorithm, node_size, 62 | node_stride, delta, neighborhood_size, connectivity): 63 | def _preprocess(image): 64 | segmentation = segmentation_algorithm(image) 65 | adj, points, _ = segmentation_adjacency(segmentation, connectivity) 66 | features = feature_extraction_algorithm(segmentation, image) 67 | StandardScaler(copy=False).fit_transform(features) 68 | 69 | fields = receptive_fields(points, adj, node_size, node_stride, 70 | neighborhood_size, delta) 71 | return fill_features(fields, features) 72 | 73 | return _preprocess 74 | 75 | 76 | preprocess_algorithm = preprocess_spatial_fixed( 77 | segmentation_algorithm, feature_extraction_algorithm, NODE_SIZE, 78 | NODE_STRIDE, DELTA, NEIGHBORHOOD_SIZE, CONNECTIVITY) 79 | 80 | # Generate preprocessed dataset. 81 | data.train = PreprocessedDataset( 82 | os.path.join(PREPROCESS_FIRST, 'train'), data.train, preprocess_algorithm) 83 | data.val = PreprocessedDataset( 84 | os.path.join(PREPROCESS_FIRST, 'val'), data.val, preprocess_algorithm) 85 | data.test = PreprocessedDataset( 86 | os.path.join(PREPROCESS_FIRST, 'test'), data.test, preprocess_algorithm) 87 | 88 | capacity = 10 * BATCH_SIZE 89 | train_queue = FileQueue(data.train, BATCH_SIZE, capacity, shuffle=True) 90 | val_queue = FileQueue(data.val, BATCH_SIZE, capacity, shuffle=True) 91 | test_queue = FileQueue(data.test, BATCH_SIZE, capacity, shuffle=False) 92 | 93 | placeholders = { 94 | 'features': 95 | tf.placeholder(tf.float32, 96 | [None, NODE_SIZE, NEIGHBORHOOD_SIZE, 97 | NUM_FEATURES], 'features'), 98 | 'labels': 99 | tf.placeholder(tf.uint8, [None, data.num_classes], 'labels'), 100 | 'dropout': 101 | tf.placeholder(tf.float32, [], 'dropout'), 102 | } 103 | 104 | 105 | class Model(BaseModel): 106 | def _build(self): 107 | conv_1 = Conv( 108 | NUM_FEATURES, 64, NEIGHBORHOOD_SIZE, logging=self.logging) 109 | fc_1 = FC(NODE_SIZE * 64, 1024, logging=self.logging) 110 | fc_2 = FC( 111 | 1024, 112 | data.num_classes, 113 | act=lambda x: x, 114 | bias=False, 115 | dropout=self.placeholders['dropout'], 116 | logging=self.logging) 117 | 118 | self.layers = [conv_1, fc_1, fc_2] 119 | 120 | 121 | model = Model( 122 | placeholders=placeholders, 123 | learning_rate=LEARNING_RATE, 124 | train_dir=TRAIN_DIR, 125 | log_dir=LOG_DIR) 126 | 127 | model.build() 128 | global_step = model.initialize() 129 | 130 | 131 | def feed_dict_with_batch(batch, dropout=0): 132 | features = np.array([data[0] for data in batch], np.float32) 133 | labels = np.array([data[1] for data in batch], np.uint8) 134 | return { 135 | placeholders['features']: features, 136 | placeholders['labels']: labels, 137 | placeholders['dropout']: DROPOUT, 138 | } 139 | 140 | 141 | try: 142 | for step in xrange(global_step, MAX_STEPS): 143 | t_pre = time.process_time() 144 | batch = train_queue.dequeue() 145 | feed_dict = feed_dict_with_batch(batch, DROPOUT) 146 | t_pre = time.process_time() - t_pre 147 | 148 | t_train = model.train(feed_dict, step) 149 | 150 | if step % DISPLAY_STEP == 0: 151 | # Evaluate on training and validation set with zero dropout. 152 | feed_dict.update({model.placeholders['dropout']: 0}) 153 | train_info = model.evaluate(feed_dict, step, 'train') 154 | batch = val_queue.dequeue() 155 | feed_dict = feed_dict_with_batch(batch, DROPOUT) 156 | val_info = model.evaluate(feed_dict, step, 'val') 157 | 158 | log = 'step={}, '.format(step) 159 | log += 'time={:.2f}s + {:.2f}s, '.format(t_pre, t_train) 160 | log += 'train_loss={:.5f}, '.format(train_info[0]) 161 | log += 'train_acc={:.5f}, '.format(train_info[1]) 162 | log += 'val_loss={:.5f}, '.format(val_info[0]) 163 | log += 'val_acc={:.5f}'.format(val_info[1]) 164 | print(log) 165 | 166 | if step % SAVE_STEP == 0: 167 | model.save() 168 | 169 | except KeyboardInterrupt: 170 | print() 171 | 172 | train_queue.close() 173 | val_queue.close() 174 | 175 | print('Optimization finished!') 176 | print('Evaluate on test set. This can take a few minutes.') 177 | 178 | try: 179 | num_steps = data.test.num_examples // BATCH_SIZE 180 | test_info = [0, 0] 181 | 182 | for i in xrange(num_steps): 183 | batch = test_queue.dequeue() 184 | feed_dict = feed_dict_with_batch(batch, DROPOUT) 185 | 186 | batch_info = model.evaluate(feed_dict) 187 | test_info = [a + b for a, b in zip(test_info, batch_info)] 188 | 189 | log = 'Test results: ' 190 | log += 'loss={:.5f}, '.format(test_info[0] / num_steps) 191 | log += 'acc={:.5f}, '.format(test_info[1] / num_steps) 192 | 193 | print(log) 194 | 195 | except KeyboardInterrupt: 196 | print() 197 | print('Test evaluation aborted.') 198 | 199 | test_queue.close() 200 | -------------------------------------------------------------------------------- /pascal_conv2d.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from __future__ import division 3 | 4 | from six.moves import xrange 5 | import time 6 | 7 | import tensorflow as tf 8 | 9 | from lib.datasets import PascalVOC as Data 10 | from lib.model import Model as BaseModel 11 | from lib.layer import ImageAugment, Conv2d, Fire, MaxPool, AveragePool 12 | 13 | DATA_DIR = 'data/pascal_voc' 14 | 15 | LEARNING_RATE = 0.0001 16 | TRAIN_DIR = None 17 | LOG_DIR = 'data/summaries/pascal_conv2d' 18 | SAVE_STEP = 250 19 | 20 | DROPOUT = 0.5 21 | BATCH_SIZE = 64 22 | MAX_STEPS = 50000 23 | DISPLAY_STEP = 10 24 | 25 | data = Data(DATA_DIR, fixed_size=224) 26 | 27 | placeholders = { 28 | 'features': 29 | tf.placeholder(tf.float32, 30 | [None, data.width, data.height, 31 | data.num_channels], 'features'), 32 | 'labels': 33 | tf.placeholder(tf.uint8, [None, data.num_classes], 'labels'), 34 | 'dropout': 35 | tf.placeholder(tf.float32, [], 'dropout'), 36 | } 37 | 38 | 39 | class Model(BaseModel): 40 | def _build(self): 41 | augment = ImageAugment() 42 | conv_1 = Conv2d( 43 | data.num_channels, 64, size=3, stride=2, logging=self.logging) 44 | max_pool_1 = MaxPool(3, 2) 45 | 46 | fire_1_1 = Fire(64, 16, 64, logging=self.logging) 47 | fire_1_2 = Fire(128, 16, 64, logging=self.logging) 48 | 49 | max_pool_2 = MaxPool(3, 2) 50 | 51 | fire_2_1 = Fire(128, 32, 128, logging=self.logging) 52 | fire_2_2 = Fire(256, 32, 128, logging=self.logging) 53 | 54 | max_pool_3 = MaxPool(3, 2) 55 | 56 | fire_3_1 = Fire(256, 48, 192, logging=self.logging) 57 | fire_3_2 = Fire(384, 48, 192, logging=self.logging) 58 | fire_3_3 = Fire(384, 64, 256, logging=self.logging) 59 | fire_3_4 = Fire(512, 64, 256, logging=self.logging) 60 | 61 | conv_2 = Conv2d( 62 | 512, 63 | 20, 64 | size=1, 65 | stride=1, 66 | bias=False, 67 | dropout=DROPOUT, 68 | logging=self.logging) 69 | 70 | avg = AveragePool() 71 | 72 | self.layers = [ 73 | augment, conv_1, max_pool_1, fire_1_1, fire_1_2, max_pool_2, 74 | fire_2_1, fire_2_2, max_pool_3, fire_3_1, fire_3_2, fire_3_3, 75 | fire_3_4, conv_2, avg 76 | ] 77 | 78 | 79 | model = Model( 80 | placeholders=placeholders, 81 | learning_rate=LEARNING_RATE, 82 | train_dir=TRAIN_DIR, 83 | log_dir=LOG_DIR) 84 | 85 | model.build() 86 | global_step = model.initialize() 87 | 88 | 89 | def feed_dict_with_batch(images, labels, dropout=0): 90 | return { 91 | placeholders['features']: images, 92 | placeholders['labels']: labels, 93 | placeholders['dropout']: DROPOUT, 94 | } 95 | 96 | 97 | try: 98 | for step in xrange(global_step, MAX_STEPS): 99 | t_pre = time.process_time() 100 | images, labels = data.train.next_batch(BATCH_SIZE, shuffle=True) 101 | feed_dict = feed_dict_with_batch(images, labels, DROPOUT) 102 | t_pre = time.process_time() - t_pre 103 | 104 | t_train = model.train(feed_dict, step) 105 | 106 | if step % DISPLAY_STEP == 0: 107 | # Evaluate on training and validation set with zero dropout. 108 | feed_dict.update({model.placeholders['dropout']: 0}) 109 | train_info = model.evaluate(feed_dict, step, 'train') 110 | images, labels = data.val.next_batch(BATCH_SIZE, shuffle=True) 111 | feed_dict = feed_dict_with_batch(images, labels) 112 | val_info = model.evaluate(feed_dict, step, 'val') 113 | 114 | log = 'step={}, '.format(step) 115 | log += 'time={:.2f}s + {:.2f}s, '.format(t_pre, t_train) 116 | log += 'train_loss={:.5f}, '.format(train_info[0]) 117 | log += 'train_acc={:.5f}, '.format(train_info[1]) 118 | log += 'val_loss={:.5f}, '.format(val_info[0]) 119 | log += 'val_acc={:.5f}'.format(val_info[1]) 120 | print(log) 121 | 122 | if step % SAVE_STEP == 0: 123 | model.save() 124 | 125 | except KeyboardInterrupt: 126 | print() 127 | 128 | print('Optimization finished!') 129 | print('Evaluate on test set. This can take a few minutes.') 130 | 131 | try: 132 | num_steps = data.test.num_examples // BATCH_SIZE 133 | test_info = [0, 0] 134 | 135 | for i in xrange(num_steps): 136 | images, labels = data.test.next_batch(BATCH_SIZE, shuffle=False) 137 | feed_dict = feed_dict_with_batch(images, labels) 138 | 139 | batch_info = model.evaluate(feed_dict) 140 | test_info = [a + b for a, b in zip(test_info, batch_info)] 141 | 142 | log = 'Test results: ' 143 | log += 'loss={:.5f}, '.format(test_info[0] / num_steps) 144 | log += 'acc={:.5f}, '.format(test_info[1] / num_steps) 145 | 146 | print(log) 147 | 148 | except KeyboardInterrupt: 149 | print() 150 | print('Test evaluation aborted.') 151 | -------------------------------------------------------------------------------- /pascal_graph.py: -------------------------------------------------------------------------------- 1 | from lib.datasets import PascalVOC as Data 2 | from lib.model import (Model as BaseModel, generate_placeholders, train) 3 | from lib.segmentation import extract_features_fixed 4 | # from lib.segmentation import slic_fixed 5 | from lib.segmentation import quickshift_fixed 6 | from lib.pipeline import preprocess_pipeline_fixed 7 | from lib.layer import EmbeddedGCNN as Conv, MaxPool, AveragePool, FC 8 | 9 | # SLIC_FORM_FEATURES = [1, 2, 3, 4, 5, 6, 8, 9, 21] 10 | QUICKSHIFT_FORM_FEATURES = [0, 2, 3, 4, 7, 19, 20, 21, 22] 11 | 12 | DATA_DIR = 'data/pascal_voc' 13 | # PREPROCESS_FIRST = 'data/pascal_voc/slic' 14 | PREPROCESS_FIRST = 'data/pascal_voc/quickshift' 15 | 16 | LEVELS = 4 17 | CONNECTIVITY = 8 18 | SCALE_INVARIANCE = False 19 | STDDEV = 1 20 | 21 | LEARNING_RATE = 0.001 22 | TRAIN_DIR = None 23 | # LOG_DIR = 'data/summaries/pascal_slic_graph' 24 | LOG_DIR = 'data/summaries/pascal_quickshift_graph' 25 | 26 | AUGMENT_TRAIN_EXAMPLES = True 27 | DROPOUT = 0.5 28 | BATCH_SIZE = 64 29 | MAX_STEPS = 50000 30 | DISPLAY_STEP = 10 31 | FORM_FEATURES = QUICKSHIFT_FORM_FEATURES 32 | NUM_FEATURES = len(FORM_FEATURES) + 3 33 | 34 | data = Data(DATA_DIR) 35 | 36 | # segmentation_algorithm = slic_fixed( 37 | # num_segments=1600, compactness=30, max_iterations=10, sigma=0) 38 | segmentation_algorithm = quickshift_fixed( 39 | ratio=0.75, kernel_size=2, max_dist=8, sigma=0) 40 | 41 | feature_extraction_algorithm = extract_features_fixed(FORM_FEATURES) 42 | 43 | preprocess_algorithm = preprocess_pipeline_fixed( 44 | segmentation_algorithm, feature_extraction_algorithm, LEVELS, CONNECTIVITY, 45 | SCALE_INVARIANCE, STDDEV) 46 | 47 | 48 | class Model(BaseModel): 49 | def _build(self): 50 | conv_1 = Conv( 51 | NUM_FEATURES, 52 | 64, 53 | adjs_dist=self.placeholders['adj_dist_1'], 54 | adjs_rad=self.placeholders['adj_rad_1'], 55 | bias=False, 56 | logging=self.logging) 57 | max_pool_1 = MaxPool(size=2) 58 | conv_2 = Conv( 59 | 64, 60 | 128, 61 | adjs_dist=self.placeholders['adj_dist_2'], 62 | adjs_rad=self.placeholders['adj_rad_2'], 63 | bias=False, 64 | logging=self.logging) 65 | max_pool_2 = MaxPool(size=2) 66 | conv_3 = Conv( 67 | 128, 68 | 256, 69 | adjs_dist=self.placeholders['adj_dist_3'], 70 | adjs_rad=self.placeholders['adj_rad_3'], 71 | bias=False, 72 | logging=self.logging) 73 | max_pool_3 = MaxPool(size=2) 74 | conv_4 = Conv( 75 | 256, 76 | 512, 77 | adjs_dist=self.placeholders['adj_dist_4'], 78 | adjs_rad=self.placeholders['adj_rad_4'], 79 | bias=False, 80 | logging=self.logging) 81 | average_pool = AveragePool() 82 | fc_1 = FC(512, 256, logging=self.logging) 83 | fc_2 = FC(256, 128, logging=self.logging) 84 | fc_3 = FC( 85 | 128, 86 | data.num_classes, 87 | act=lambda x: x, 88 | bias=False, 89 | dropout=self.placeholders['dropout'], 90 | logging=self.logging) 91 | 92 | self.layers = [ 93 | conv_1, max_pool_1, conv_2, max_pool_2, conv_3, max_pool_3, conv_4, 94 | average_pool, fc_1, fc_2, fc_3 95 | ] 96 | 97 | 98 | placeholders = generate_placeholders(BATCH_SIZE, LEVELS, NUM_FEATURES, 99 | data.num_classes) 100 | 101 | model = Model( 102 | placeholders=placeholders, 103 | learning_rate=LEARNING_RATE, 104 | train_dir=TRAIN_DIR, 105 | log_dir=LOG_DIR) 106 | 107 | train(model, data, preprocess_algorithm, BATCH_SIZE, DROPOUT, 108 | AUGMENT_TRAIN_EXAMPLES, MAX_STEPS, PREPROCESS_FIRST, DISPLAY_STEP) 109 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | numpy 2 | scipy 3 | tensorflow 4 | scikit-image 5 | scikit-learn 6 | numpy_groupies 7 | cached-property 8 | -------------------------------------------------------------------------------- /requirements_test.txt: -------------------------------------------------------------------------------- 1 | # Pull in the packages required to run the project. 2 | -r requirements.txt 3 | 4 | # Testing-specific package requirements 5 | pep8>=1.7.0 6 | flake8>=3.3.0 7 | nose>=1.3.7 8 | coverage>=4.4.1 9 | -------------------------------------------------------------------------------- /test_data/VOCtrainval_11-May-2012.tar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rusty1s/embedded_gcnn/06db3799e794d6ebcd9db023ebd8b0937587df94/test_data/VOCtrainval_11-May-2012.tar --------------------------------------------------------------------------------