├── .gitignore ├── LICENSE ├── README.md ├── figures └── affinity_coding.png ├── goggles ├── __init__.py ├── affinity_matrix_construction │ ├── __init__.py │ ├── construct.py │ └── image_AF │ │ ├── __init__.py │ │ ├── neural_network_AFs.py │ │ └── pretrained_models │ │ ├── __init__.py │ │ └── vgg.py ├── data │ └── cub_dataset │ │ ├── images │ │ ├── Green_Jay_0003_65767.jpg │ │ ├── Green_Jay_0006_65788.jpg │ │ ├── Green_Jay_0011_65947.jpg │ │ ├── Green_Jay_0014_65825.jpg │ │ ├── Green_Jay_0015_65796.jpg │ │ ├── Green_Jay_0016_65864.jpg │ │ ├── Green_Jay_0020_66168.jpg │ │ ├── Green_Jay_0023_65898.jpg │ │ ├── Green_Jay_0027_65783.jpg │ │ ├── Green_Jay_0028_65719.jpg │ │ ├── Green_Jay_0029_65637.jpg │ │ ├── Green_Jay_0030_65824.jpg │ │ ├── Green_Jay_0032_65851.jpg │ │ ├── Green_Jay_0036_65660.jpg │ │ ├── Green_Jay_0038_65702.jpg │ │ ├── Green_Jay_0039_66118.jpg │ │ ├── Green_Jay_0040_65863.jpg │ │ ├── Green_Jay_0041_65548.jpg │ │ ├── Green_Jay_0042_65740.jpg │ │ ├── Green_Jay_0043_65805.jpg │ │ ├── Green_Jay_0044_66228.jpg │ │ ├── Green_Jay_0047_65757.jpg │ │ ├── Green_Jay_0049_65600.jpg │ │ ├── Green_Jay_0051_65662.jpg │ │ ├── Green_Jay_0052_66174.jpg │ │ ├── Green_Jay_0055_65807.jpg │ │ ├── Green_Jay_0059_65586.jpg │ │ ├── Green_Jay_0063_65925.jpg │ │ ├── Green_Jay_0064_65552.jpg │ │ ├── Green_Jay_0066_65902.jpg │ │ ├── Green_Jay_0071_65799.jpg │ │ ├── Green_Jay_0074_65889.jpg │ │ ├── Green_Jay_0075_65701.jpg │ │ ├── Green_Jay_0077_65736.jpg │ │ ├── Green_Jay_0085_66077.jpg │ │ ├── Green_Jay_0086_65847.jpg │ │ ├── Green_Jay_0089_66075.jpg │ │ ├── Green_Jay_0090_65895.jpg │ │ ├── Green_Jay_0092_65884.jpg │ │ ├── Green_Jay_0094_65775.jpg │ │ ├── Green_Jay_0095_65881.jpg │ │ ├── Green_Jay_0098_65857.jpg │ │ ├── Green_Jay_0100_65786.jpg │ │ ├── Green_Jay_0102_65813.jpg │ │ ├── Green_Jay_0104_65908.jpg │ │ ├── Green_Jay_0106_65811.jpg │ │ ├── Green_Jay_0111_65869.jpg │ │ ├── Green_Jay_0114_65841.jpg │ │ ├── Green_Jay_0115_65745.jpg │ │ ├── Green_Jay_0118_66251.jpg │ │ ├── Green_Jay_0121_65564.jpg │ │ ├── Green_Jay_0124_65848.jpg │ │ ├── Green_Jay_0126_65716.jpg │ │ ├── Green_Jay_0128_65629.jpg │ │ ├── Green_Jay_0129_65680.jpg │ │ ├── Green_Jay_0130_65885.jpg │ │ ├── Green_Jay_0132_65886.jpg │ │ ├── Whip_Poor_Will_0001_796411.jpg │ │ ├── Whip_Poor_Will_0002_796427.jpg │ │ ├── Whip_Poor_Will_0003_796409.jpg │ │ ├── Whip_Poor_Will_0004_100479.jpg │ │ ├── Whip_Poor_Will_0005_796425.jpg │ │ ├── Whip_Poor_Will_0006_22800.jpg │ │ ├── Whip_Poor_Will_0007_796431.jpg │ │ ├── Whip_Poor_Will_0008_796420.jpg │ │ ├── Whip_Poor_Will_0009_796423.jpg │ │ ├── Whip_Poor_Will_0010_100464.jpg │ │ ├── Whip_Poor_Will_0011_796416.jpg │ │ ├── Whip_Poor_Will_0012_796414.jpg │ │ ├── Whip_Poor_Will_0013_796439.jpg │ │ ├── Whip_Poor_Will_0015_796435.jpg │ │ ├── Whip_Poor_Will_0016_796424.jpg │ │ ├── Whip_Poor_Will_0017_796429.jpg │ │ ├── Whip_Poor_Will_0018_796403.jpg │ │ ├── Whip_Poor_Will_0019_796433.jpg │ │ ├── Whip_Poor_Will_0020_100396.jpg │ │ ├── Whip_Poor_Will_0021_796426.jpg │ │ ├── Whip_Poor_Will_0022_796438.jpg │ │ ├── Whip_Poor_Will_0023_796410.jpg │ │ ├── Whip_Poor_Will_0024_100444.jpg │ │ ├── Whip_Poor_Will_0025_796434.jpg │ │ ├── Whip_Poor_Will_0026_100456.jpg │ │ ├── Whip_Poor_Will_0027_796441.jpg │ │ ├── Whip_Poor_Will_0028_796421.jpg │ │ ├── Whip_Poor_Will_0029_796432.jpg │ │ ├── Whip_Poor_Will_0030_100412.jpg │ │ ├── Whip_Poor_Will_0031_100397.jpg │ │ ├── Whip_Poor_Will_0032_796437.jpg │ │ ├── Whip_Poor_Will_0033_82166.jpg │ │ ├── Whip_Poor_Will_0034_796413.jpg │ │ ├── Whip_Poor_Will_0035_796430.jpg │ │ ├── Whip_Poor_Will_0036_100399.jpg │ │ ├── Whip_Poor_Will_0037_796405.jpg │ │ ├── Whip_Poor_Will_0038_100443.jpg │ │ ├── Whip_Poor_Will_0039_796408.jpg │ │ ├── Whip_Poor_Will_0040_100482.jpg │ │ ├── Whip_Poor_Will_0041_796428.jpg │ │ ├── Whip_Poor_Will_0042_796419.jpg │ │ ├── Whip_Poor_Will_0043_796442.jpg │ │ ├── Whip_Poor_Will_0044_796407.jpg │ │ ├── Whip_Poor_Will_0045_796422.jpg │ │ ├── Whip_Poor_Will_0046_796440.jpg │ │ ├── Whip_Poor_Will_0047_796412.jpg │ │ ├── Whip_Poor_Will_0048_796417.jpg │ │ ├── Whip_Poor_Will_0049_83621.jpg │ │ └── Whip_Poor_Will_0050_796415.jpg │ │ └── labels.csv ├── inference_models │ ├── __init__.py │ ├── cluster_class_mapping.py │ ├── hierarchical_model.py │ └── semi_supervised_models.py ├── test │ └── test.py ├── theory │ ├── __init__.py │ └── theory.py ├── tutorials │ └── intro │ │ └── intro.ipynb └── utils │ ├── __init__.py │ ├── constants.py │ └── dataset.py ├── requirements.txt ├── setup.py └── tools └── get_cub_dataset.sh /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | 53 | # Translations 54 | *.mo 55 | *.pot 56 | 57 | # Django stuff: 58 | *.log 59 | local_settings.py 60 | db.sqlite3 61 | db.sqlite3-journal 62 | 63 | # Flask stuff: 64 | instance/ 65 | .webassets-cache 66 | 67 | # Scrapy stuff: 68 | .scrapy 69 | 70 | # Sphinx documentation 71 | docs/_build/ 72 | 73 | # PyBuilder 74 | target/ 75 | 76 | # Jupyter Notebook 77 | .ipynb_checkpoints 78 | 79 | # IPython 80 | profile_default/ 81 | ipython_config.py 82 | 83 | # pyenv 84 | .python-version 85 | 86 | # pipenv 87 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 88 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 89 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 90 | # install all needed dependencies. 91 | #Pipfile.lock 92 | 93 | # celery beat schedule file 94 | celerybeat-schedule 95 | 96 | # SageMath parsed files 97 | *.sage.py 98 | 99 | # Environments 100 | .env 101 | .venv 102 | env/ 103 | venv/ 104 | ENV/ 105 | env.bak/ 106 | venv.bak/ 107 | 108 | # Spyder project settings 109 | .spyderproject 110 | .spyproject 111 | 112 | # Rope project settings 113 | .ropeproject 114 | 115 | # mkdocs documentation 116 | /site 117 | 118 | # mypy 119 | .mypy_cache/ 120 | .dmypy.json 121 | dmypy.json 122 | 123 | # Pyre type checker 124 | .pyre/ 125 | 126 | 127 | .DS_Store 128 | .idea/ 129 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 chu-data-lab 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GOGGLES 2 | 3 | GOGGLES is a system for automatically generating probabilistic labels for image datasets based on the affinity coding paradigm. The paper can be found at https://arxiv.org/abs/1903.04552 4 | 5 | ![The affinity coding paradigm](./figures/affinity_coding.png) 6 | 7 | 8 | ## Installation 9 | 10 | ```bash 11 | git clone https://github.com/chu-data-lab/GOGGLES.git 12 | cd GOGGLES 13 | pip3 install . 14 | ``` 15 | 16 | ## Example Usage 17 | please refer to the tutorials in goggles/tutorials 18 | ```bash 19 | cd goggles/tutorials/intro 20 | jupyter-notebook 21 | ``` 22 | 23 | -------------------------------------------------------------------------------- /figures/affinity_coding.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/figures/affinity_coding.png -------------------------------------------------------------------------------- /goggles/__init__.py: -------------------------------------------------------------------------------- 1 | from .inference_models.hierarchical_model import infer_labels 2 | from .utils.dataset import GogglesDataset 3 | from .affinity_matrix_construction.construct import construct_image_affinity_matrices 4 | 5 | __version__ = '0.1' -------------------------------------------------------------------------------- /goggles/affinity_matrix_construction/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/affinity_matrix_construction/__init__.py -------------------------------------------------------------------------------- /goggles/affinity_matrix_construction/construct.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from goggles.affinity_matrix_construction.image_AF.neural_network_AFs import nn_AFs 3 | 4 | 5 | def construct_image_affinity_matrices(dataset,cache=True): 6 | """ 7 | :param GogglesDataset instance 8 | :return: a list of affinity matrices 9 | """ 10 | matrix_list = [] 11 | for layer_idx in [4,9,16,23,30]:#all max pooling layers 12 | matrix_list.extend(nn_AFs(dataset,layer_idx,10,cache)) 13 | return matrix_list 14 | -------------------------------------------------------------------------------- /goggles/affinity_matrix_construction/image_AF/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/affinity_matrix_construction/image_AF/__init__.py -------------------------------------------------------------------------------- /goggles/affinity_matrix_construction/image_AF/neural_network_AFs.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | import torch.nn.functional as F 4 | from tqdm import trange 5 | import numpy as np 6 | from goggles.utils.constants import * 7 | from .pretrained_models.vgg import Vgg16 8 | 9 | _make_cuda = lambda x: x.cuda() if torch.cuda.is_available() else x 10 | 11 | class Context: 12 | def __init__(self, model, dataset, layer_idx): 13 | self.model = model 14 | self.dataset = dataset 15 | 16 | self._layer_idx = layer_idx 17 | self._model_out_dict = dict() 18 | 19 | def get_model_output(self, image_idx): 20 | if image_idx not in self._model_out_dict: 21 | x = self.dataset[image_idx] 22 | x = x.view((1,) + x.size()) 23 | x = _make_cuda(torch.autograd.Variable(x, requires_grad=False)) 24 | 25 | self._model_out_dict[image_idx] = \ 26 | self.model.forward(x, layer_idx=self._layer_idx)[0] 27 | 28 | z = self._model_out_dict[image_idx] 29 | return z 30 | 31 | 32 | def _get_patches(z, patch_idxs, normalize=False): 33 | """ 34 | z: CxHxW 35 | patch_idxs: K 36 | """ 37 | 38 | c = z.size(0) 39 | 40 | patches = z.view(c, -1).t()[patch_idxs] 41 | if normalize: 42 | patches = F.normalize(patches, dim=1) 43 | 44 | return patches 45 | 46 | 47 | def _get_most_activated_channels(z, num_channels=5): 48 | """ 49 | z: CxHxW 50 | """ 51 | 52 | per_channel_max_activations, _ = z.max(1)[0].max(1) 53 | 54 | most_activated_channels = \ 55 | torch.topk(per_channel_max_activations, num_channels)[1] 56 | 57 | return most_activated_channels 58 | 59 | 60 | def _get_most_activated_patch_idxs_from_channels(z, channel_idxs): 61 | """ 62 | z: CxHxW 63 | channel_idxs: K 64 | """ 65 | 66 | k = channel_idxs.shape[0] 67 | 68 | most_activated_patch_idxs = \ 69 | z[channel_idxs].view(k, -1).max(1)[1] 70 | 71 | l = list(most_activated_patch_idxs.cpu().numpy()) 72 | #d_ = {p: k - i - 1 for i, p in enumerate(reversed(l))} 73 | d = [(k - i - 1, p) for i, p in enumerate(reversed(l))] 74 | d = list(sorted(d)) 75 | r,u = list(zip(*d)) 76 | #u = sorted(d.keys(), key=lambda p:d[p]) 77 | #r = [d[p] for p in u] 78 | return _make_cuda(torch.LongTensor(u)), \ 79 | _make_cuda(torch.LongTensor(r)) 80 | 81 | 82 | def _get_score_matrix_for_image(image_idx, num_max_proposals, context): 83 | score_matrix = list() 84 | column_ids = list() 85 | 86 | z = context.get_model_output(image_idx) 87 | num_patches = z.size(1) * z.size(2) 88 | ch = _get_most_activated_channels(z, num_channels=num_max_proposals) 89 | pids, ranks = _get_most_activated_patch_idxs_from_channels(z, ch) 90 | proto_patches = _get_patches(z, pids, normalize=True) 91 | 92 | for patch_idx, rank in zip(pids.cpu().numpy(), ranks.cpu().numpy()): 93 | column_ids.append([image_idx, patch_idx, rank]) 94 | 95 | for image_idx_ in range(len(context.dataset)): 96 | z_ = context.get_model_output(image_idx_) 97 | img_patches = _get_patches(z_, range(num_patches), normalize=True) 98 | scores = torch.matmul(img_patches, proto_patches.t()).max(0)[0] 99 | scores = scores.cpu().numpy() 100 | score_matrix.append(scores) 101 | return np.array(score_matrix), column_ids 102 | 103 | 104 | def nn_AFs(dataset,layer_idx, num_max_proposals,cache=False): 105 | affinity_matrix_list = [[] for _ in range(num_max_proposals)] 106 | out_filename = '.'.join([ 107 | 'v2', 108 | f'vgg16_layer{layer_idx:02d}', 109 | f'k{num_max_proposals:02d}', 110 | 'scores.npz']) 111 | out_dirpath = os.path.join(SCRATCH_DIR, 'scores') 112 | os.makedirs(out_dirpath, exist_ok=True) 113 | out_filepath = os.path.join(out_dirpath, out_filename) 114 | if cache: 115 | try: 116 | affinity_matrix_arr = np.load(out_filepath)['scores'] 117 | for i in range(num_max_proposals): 118 | affinity_matrix_list[i] = (np.squeeze(affinity_matrix_arr[i,:,:])) 119 | return affinity_matrix_list 120 | except: 121 | pass 122 | model = _make_cuda(Vgg16()) 123 | context = Context(model=model, dataset=dataset, layer_idx=layer_idx) 124 | #all_column_ids = list() 125 | for image_idx in trange(len(context.dataset)): 126 | scores, cols = _get_score_matrix_for_image( 127 | image_idx, num_max_proposals, context) 128 | for i in range(min(num_max_proposals,scores.shape[1])): 129 | affinity_matrix_list[i].append(scores[:,i]) 130 | #all_column_ids += cols 131 | for i in range(num_max_proposals): 132 | affinity_matrix_list[i] = np.array(affinity_matrix_list[i]).T 133 | if cache: 134 | print('saving output to %s' % out_filepath) 135 | np.savez( 136 | out_filepath, version=2, 137 | scores=np.array(affinity_matrix_list), 138 | num_max_proposals=num_max_proposals) 139 | return affinity_matrix_list -------------------------------------------------------------------------------- /goggles/affinity_matrix_construction/image_AF/pretrained_models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/affinity_matrix_construction/image_AF/pretrained_models/__init__.py -------------------------------------------------------------------------------- /goggles/affinity_matrix_construction/image_AF/pretrained_models/vgg.py: -------------------------------------------------------------------------------- 1 | from collections import OrderedDict 2 | 3 | import numpy as np 4 | import torch 5 | import torch.nn as nn 6 | from torchvision import models 7 | 8 | 9 | class Vgg16(nn.Module): 10 | def __init__(self, freeze=True): 11 | super(Vgg16, self).__init__() 12 | self._is_cuda = False 13 | 14 | self.input_size = 224 15 | 16 | base_model = models.vgg16(pretrained=True) 17 | 18 | features = list(base_model.features) 19 | self._features = nn.ModuleList(features).eval() 20 | 21 | classifier = list(base_model.classifier) 22 | embedding = classifier 23 | self._embedding = nn.Sequential(*embedding).eval() 24 | 25 | self._is_frozen = freeze 26 | self.freeze(freeze) 27 | 28 | self._config = None 29 | self._parse_config() 30 | 31 | def _make_cuda(self, x): 32 | return x.cuda() if self._is_cuda else x 33 | 34 | def _parse_config(self): 35 | self.zero_grad() 36 | 37 | x = self._make_cuda(torch.autograd.Variable( 38 | torch.rand(1, 3, self.input_size, self.input_size), 39 | requires_grad=False)) 40 | 41 | self._config = OrderedDict() 42 | for i, layer in enumerate(self._features): 43 | x = layer(x) 44 | self._config[i] = ( 45 | layer.__class__.__name__, 46 | tuple(x.size()[-3:]),) 47 | 48 | self.zero_grad() 49 | 50 | def cuda(self, device_id=None): 51 | self._is_cuda = True 52 | return super(Vgg16, self).cuda(device_id) 53 | 54 | def freeze(self, freeze=True): 55 | requires_grad = not freeze 56 | for parameter in self.parameters(): 57 | parameter.requires_grad = requires_grad 58 | 59 | self._is_frozen = freeze 60 | 61 | def forward(self, x, layer_idx=None): 62 | if layer_idx is None: 63 | layer_idx = len(self._config) - 1 64 | 65 | for i, model in enumerate(self._features): 66 | x = model(x) 67 | 68 | if i == layer_idx: 69 | return x 70 | 71 | def embed(self, x, layer='logits'): 72 | assert layer in ['pre_fc', 'logits'] 73 | x = self.forward(x) 74 | x = x.view(x.size(0), -1) 75 | if layer == 'logits': 76 | x = self._embedding(x) 77 | return x 78 | 79 | def get_layer_type(self, layer_idx): 80 | return self._config[layer_idx][0] 81 | 82 | def get_layer_output_dim(self, layer_idx): 83 | return self._config[layer_idx][1] 84 | 85 | def get_receptive_field(self, patch, layer_idx=None): 86 | is_originally_frozen = self._is_frozen 87 | self.zero_grad() 88 | self.freeze(False) 89 | 90 | image_size = self.input_size 91 | batch_shape = (1, 3, image_size, image_size) 92 | 93 | x = self._make_cuda(torch.autograd.Variable( 94 | torch.rand(*batch_shape), requires_grad=True)) 95 | z = self.forward(x, layer_idx=layer_idx) 96 | z_patch = patch.forward(z) 97 | 98 | torch.sum(z_patch).backward() 99 | 100 | rf = x.grad.data.cpu().numpy() 101 | rf = rf[0, 0] 102 | rf = list(zip(*np.where(np.abs(rf) > 1e-6))) 103 | 104 | (i_nw, j_nw), (i_se, j_se) = rf[0], rf[-1] 105 | 106 | rf_w, rf_h = (j_se - j_nw + 1, 107 | i_se - i_nw + 1) 108 | 109 | self.zero_grad() 110 | self.freeze(is_originally_frozen) 111 | 112 | return (i_nw, j_nw), (rf_w, rf_h) 113 | 114 | 115 | if __name__ == '__main__': 116 | # input_image_size = 224 117 | # expected_image_shape = (3, input_image_size, input_image_size) 118 | # input_tensor = torch.autograd.Variable(torch.rand(1, *expected_image_shape)) 119 | 120 | net = Vgg16() 121 | # print net(input_tensor) 122 | -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0003_65767.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0003_65767.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0006_65788.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0006_65788.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0011_65947.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0011_65947.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0014_65825.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0014_65825.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0015_65796.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0015_65796.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0016_65864.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0016_65864.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0020_66168.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0020_66168.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0023_65898.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0023_65898.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0027_65783.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0027_65783.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0028_65719.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0028_65719.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0029_65637.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0029_65637.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0030_65824.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0030_65824.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0032_65851.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0032_65851.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0036_65660.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0036_65660.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0038_65702.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0038_65702.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0039_66118.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0039_66118.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0040_65863.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0040_65863.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0041_65548.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0041_65548.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0042_65740.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0042_65740.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0043_65805.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0043_65805.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0044_66228.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0044_66228.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0047_65757.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0047_65757.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0049_65600.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0049_65600.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0051_65662.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0051_65662.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0052_66174.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0052_66174.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0055_65807.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0055_65807.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0059_65586.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0059_65586.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0063_65925.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0063_65925.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0064_65552.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0064_65552.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0066_65902.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0066_65902.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0071_65799.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0071_65799.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0074_65889.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0074_65889.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0075_65701.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0075_65701.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0077_65736.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0077_65736.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0085_66077.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0085_66077.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0086_65847.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0086_65847.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0089_66075.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0089_66075.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0090_65895.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0090_65895.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0092_65884.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0092_65884.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0094_65775.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0094_65775.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0095_65881.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0095_65881.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0098_65857.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0098_65857.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0100_65786.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0100_65786.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0102_65813.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0102_65813.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0104_65908.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0104_65908.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0106_65811.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0106_65811.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0111_65869.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0111_65869.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0114_65841.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0114_65841.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0115_65745.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0115_65745.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0118_66251.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0118_66251.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0121_65564.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0121_65564.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0124_65848.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0124_65848.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0126_65716.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0126_65716.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0128_65629.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0128_65629.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0129_65680.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0129_65680.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0130_65885.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0130_65885.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Green_Jay_0132_65886.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Green_Jay_0132_65886.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0001_796411.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0001_796411.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0002_796427.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0002_796427.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0003_796409.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0003_796409.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0004_100479.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0004_100479.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0005_796425.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0005_796425.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0006_22800.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0006_22800.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0007_796431.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0007_796431.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0008_796420.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0008_796420.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0009_796423.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0009_796423.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0010_100464.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0010_100464.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0011_796416.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0011_796416.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0012_796414.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0012_796414.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0013_796439.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0013_796439.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0015_796435.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0015_796435.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0016_796424.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0016_796424.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0017_796429.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0017_796429.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0018_796403.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0018_796403.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0019_796433.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0019_796433.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0020_100396.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0020_100396.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0021_796426.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0021_796426.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0022_796438.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0022_796438.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0023_796410.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0023_796410.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0024_100444.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0024_100444.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0025_796434.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0025_796434.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0026_100456.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0026_100456.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0027_796441.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0027_796441.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0028_796421.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0028_796421.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0029_796432.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0029_796432.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0030_100412.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0030_100412.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0031_100397.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0031_100397.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0032_796437.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0032_796437.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0033_82166.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0033_82166.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0034_796413.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0034_796413.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0035_796430.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0035_796430.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0036_100399.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0036_100399.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0037_796405.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0037_796405.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0038_100443.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0038_100443.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0039_796408.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0039_796408.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0040_100482.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0040_100482.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0041_796428.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0041_796428.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0042_796419.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0042_796419.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0043_796442.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0043_796442.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0044_796407.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0044_796407.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0045_796422.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0045_796422.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0046_796440.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0046_796440.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0047_796412.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0047_796412.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0048_796417.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0048_796417.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0049_83621.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0049_83621.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/images/Whip_Poor_Will_0050_796415.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/data/cub_dataset/images/Whip_Poor_Will_0050_796415.jpg -------------------------------------------------------------------------------- /goggles/data/cub_dataset/labels.csv: -------------------------------------------------------------------------------- 1 | y 2 | 0 3 | 0 4 | 0 5 | 0 6 | 0 7 | 0 8 | 0 9 | 0 10 | 0 11 | 0 12 | 0 13 | 0 14 | 0 15 | 0 16 | 0 17 | 0 18 | 0 19 | 0 20 | 0 21 | 0 22 | 0 23 | 0 24 | 0 25 | 0 26 | 0 27 | 0 28 | 0 29 | 0 30 | 0 31 | 0 32 | 0 33 | 0 34 | 0 35 | 0 36 | 0 37 | 0 38 | 0 39 | 0 40 | 0 41 | 0 42 | 0 43 | 0 44 | 0 45 | 0 46 | 0 47 | 0 48 | 0 49 | 0 50 | 0 51 | 0 52 | 0 53 | 0 54 | 0 55 | 0 56 | 0 57 | 0 58 | 0 59 | 1 60 | 1 61 | 1 62 | 1 63 | 1 64 | 1 65 | 1 66 | 1 67 | 1 68 | 1 69 | 1 70 | 1 71 | 1 72 | 1 73 | 1 74 | 1 75 | 1 76 | 1 77 | 1 78 | 1 79 | 1 80 | 1 81 | 1 82 | 1 83 | 1 84 | 1 85 | 1 86 | 1 87 | 1 88 | 1 89 | 1 90 | 1 91 | 1 92 | 1 93 | 1 94 | 1 95 | 1 96 | 1 97 | 1 98 | 1 99 | 1 100 | 1 101 | 1 102 | 1 103 | 1 104 | 1 105 | 1 106 | 1 107 | 1 108 | -------------------------------------------------------------------------------- /goggles/inference_models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/inference_models/__init__.py -------------------------------------------------------------------------------- /goggles/inference_models/cluster_class_mapping.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from scipy.optimize import linear_sum_assignment 3 | from goggles.theory.theory import DevSetTheory 4 | 5 | 6 | def construct_D(y_cluster,y_class): 7 | """ 8 | construct D_matrix from cluster labels and class labels 9 | :param y_cluster: 1-d numpy array of cluster ids 10 | :param y_class: 1-d numpy array of class ids 11 | :return: 2-d numpy array, D_matrix[i,j] is the number of instances 12 | in the ith class that falls into the jth cluster 13 | """ 14 | K = len(set(y_class)) 15 | D = np.zeros(shape=(K,K)) 16 | for i in range(K): 17 | for j in range(K): 18 | D[i][j] = np.sum((y_cluster==i)&(y_class==j)) 19 | D_matrix = D.T 20 | return D_matrix 21 | 22 | 23 | def solve_mapping(y_cluster,y_class,evaluate=False): 24 | """ 25 | obtain class to cluster mapping 26 | :param y_cluster: 1-d numpy array of cluster ids 27 | :param y_class: 1-d numpy array of class ids 28 | :return: a list of integers where the ith element is the corresponding cluster for the ith class 29 | """ 30 | D = construct_D(y_cluster,y_class) 31 | row_ind, col_ind = linear_sum_assignment(-D) 32 | if evaluate: 33 | #print(D) 34 | theory = DevSetTheory(d_matrix=D) 35 | print("The feasibility probability is:", theory.feasibility_test()) 36 | print("The probability of the dev set being sufficient is:", theory.dev_set_sufficiency_test()) 37 | return col_ind -------------------------------------------------------------------------------- /goggles/inference_models/hierarchical_model.py: -------------------------------------------------------------------------------- 1 | from goggles.inference_models.semi_supervised_models import SemiGMM,SemiBMM 2 | import numpy as np 3 | from tqdm import tqdm 4 | import random 5 | from goggles.affinity_matrix_construction.construct import construct_image_affinity_matrices 6 | 7 | 8 | def infer_labels(affinity_matrix_list, dev_set_indices, dev_set_labels,seed=0,evaluate=True): 9 | """ 10 | infer labels by a hierarchical inference model 11 | :param affinity_matrix_list: 12 | :param dev_set_indices: list of indices of the images in the development set 13 | :param dev_set_labels: list of labels of the images in the development set 14 | :return: 15 | """ 16 | random.seed(seed) 17 | np.random.seed(seed) 18 | n_classes = len(set(dev_set_labels)) 19 | LPs = [] 20 | for af_matrix in tqdm(affinity_matrix_list): 21 | base_model = SemiGMM(covariance_type="diag",n_components=n_classes) 22 | lp = base_model.fit_predict(af_matrix,dev_set_indices,dev_set_labels) 23 | LPs.append(lp) 24 | LPs_array = np.hstack(LPs) 25 | ensemble_model = SemiBMM(n_components=n_classes) 26 | predicted_labels = ensemble_model.fit_predict(LPs_array,dev_set_indices,dev_set_labels,evaluate) 27 | return predicted_labels -------------------------------------------------------------------------------- /goggles/inference_models/semi_supervised_models.py: -------------------------------------------------------------------------------- 1 | from sklearn.mixture import GaussianMixture 2 | from sklearn.cluster import KMeans 3 | from goggles.inference_models.cluster_class_mapping import solve_mapping 4 | import numpy as np 5 | DEL = 1e-300 6 | 7 | def update_prob_using_mapping(prob, dev_set_indices, dev_set_labels,evaluate=False): 8 | """ 9 | rearrange the columns in the prob matrix so that the ith column represents the ith class 10 | :param prob: NxK numpy array, probabilities of each example 11 | (totaling N) belonging to each class (totaling K) 12 | :param dev_set_indices: list of indices of the images in the development set 13 | :param dev_set_labels: list of labels of the images in the development set 14 | :param evaluate: evaluate the results or not. 15 | :return: the updated prob matrix 16 | """ 17 | cluster_labels = np.argmax(prob, axis=1) 18 | dev_cluster_labels = cluster_labels[dev_set_indices] 19 | cluster_class_mapping = solve_mapping(dev_cluster_labels, dev_set_labels,evaluate) 20 | prob = prob[:, cluster_class_mapping] 21 | return prob 22 | 23 | 24 | def set_prob_dev_values(prob, dev_set_indices, dev_set_labels): 25 | """ 26 | hard set in prob the values of the instances in the dev set by their labels 27 | :param prob: prob: NxK numpy array, probabilities of each example 28 | (totaling N) belonging to each class (totaling K) 29 | :param dev_set_indices: list of indices of the images in the development set 30 | :param dev_set_labels: list of labels of the images in the development set 31 | :return: the new prob matrix 32 | """ 33 | #prob[dev_set_indices, :] = 0 34 | #for i in range(len(dev_set_indices)): 35 | # prob[dev_set_indices[i], dev_set_labels[i]] = 1 36 | return prob 37 | 38 | 39 | 40 | def pmf_bernoulli(X,mu): 41 | """ 42 | probability mass function of the multi-variate bernoulli distribution 43 | :param X: data, a NxM numpy array 44 | :param mu: parameter of the bernoulli distribution, a Mx1 numpy array 45 | :return: probability of every row in X, a NX1 numpy array 46 | """ 47 | return np.exp(np.sum(X*np.log(mu+DEL)+(1-X)*np.log(1-mu+DEL),axis=1)) 48 | 49 | 50 | class ConvergenceMeter: 51 | """ 52 | meter for convergence 53 | """ 54 | def __init__(self, num_converged, rate_threshold, 55 | diff_fn=lambda a, b: abs(a - b)): 56 | self._num_converged = num_converged 57 | self._rate_threshold = rate_threshold 58 | self._diff_fn = diff_fn 59 | self._diff_history = list() 60 | self._last_val = None 61 | 62 | def offer(self, val): 63 | if self._last_val is not None: 64 | self._diff_history.append( 65 | self._diff_fn(val, self._last_val)) 66 | 67 | self._last_val = val 68 | 69 | @property 70 | def is_converged(self): 71 | if len(self._diff_history) < self._num_converged: 72 | return False 73 | 74 | return np.mean( 75 | self._diff_history[-self._num_converged:]) \ 76 | <= self._rate_threshold 77 | 78 | class SemiGMM(GaussianMixture): 79 | """ 80 | Goggles Semi-supervised Guassian Mixture model adapted from scikit-learn. 81 | The cluster-to-class mapping is performed based section 4.3 82 | """ 83 | 84 | def __init__(self, n_components=1, covariance_type='full', tol=1e-4, reg_covar=1e-6): 85 | super().__init__( 86 | n_components=n_components, covariance_type = covariance_type,tol=tol,reg_covar=reg_covar) 87 | 88 | def fit(self, X, dev_set_indices,dev_set_labels): 89 | """ 90 | :param X: data, a NxM numpy array 91 | :param dev_set_indices: list of indices of the images in the development set 92 | :param dev_set_labels: list of labels of the images in the development set 93 | :return: 94 | """ 95 | self.dev_set_indices = np.array(dev_set_indices) 96 | self.dev_set_labels = np.array(dev_set_labels) 97 | return super().fit(X) 98 | 99 | def fit_predict(self,X, dev_set_indices,dev_set_labels): 100 | """ 101 | fit a gmm and predict the class labels for all instances 102 | :param X: data, a NxM numpy array 103 | :param dev_set_indices: list of indices of the images in the development set 104 | :param dev_set_labels: list of labels of the images in the development set 105 | :return: probability of each instance (totaling N) belonging to each class (totaling K), 106 | a N X K numpy array 107 | """ 108 | self.fit(X, dev_set_indices,dev_set_labels) 109 | return self.predict_proba(X) 110 | 111 | def _estimate_log_prob_resp(self,X): 112 | log_prob_norm, log_resp = super()._estimate_log_prob_resp(X) 113 | prob = np.exp(log_resp) 114 | prob = update_prob_using_mapping(prob, self.dev_set_indices, self.dev_set_labels) 115 | prob = set_prob_dev_values(prob, self.dev_set_indices, self.dev_set_labels) 116 | log_resp = np.log(prob + DEL) 117 | return log_prob_norm, log_resp 118 | 119 | 120 | class SemiBMM: 121 | def __init__(self,n_components): 122 | self.K = n_components 123 | self.pi = np.ones(self.K)*1/self.K 124 | self.mu = np.zeros(self.K) 125 | 126 | 127 | def initalization(self,X): 128 | km = KMeans(n_clusters=self.K) 129 | y_init = km.fit_predict(X) 130 | prob = np.zeros(shape=(X.shape[0],self.K)) 131 | for i in range(X.shape[0]): 132 | prob[i,y_init[i]] = 1 133 | return prob 134 | 135 | 136 | def fit_predict(self,X, dev_set_indices,dev_set_labels,evaluate): 137 | """ 138 | fit a gmm and predict the class labels for all instances 139 | :param X: data, a NxM numpy array 140 | :param dev_set_indices: list of indices of the images in the development set 141 | :param dev_set_labels: list of labels of the images in the development set 142 | :param evaluate: If true, performs feasibility test and dev set sufficiency test 143 | :return: probability of each instance (totaling N) belonging to each class (totaling K), 144 | a N X K numpy array 145 | """ 146 | prob = self.initalization(X) 147 | self.dev_set_indices = np.array(dev_set_indices) 148 | self.dev_set_labels = np.array(dev_set_labels) 149 | convergence = ConvergenceMeter(20, 1e-6, diff_fn=lambda a, b: np.linalg.norm(a - b)) 150 | n_inter = 0 151 | max_inter = 200 152 | while not convergence.is_converged: 153 | if n_inter > max_inter: 154 | break 155 | self.M_step(X, prob) 156 | prob = self.E_step(X) 157 | convergence.offer(prob) 158 | n_inter+=1 159 | if evaluate: 160 | prob = self.E_step(X, evaluate) 161 | return prob 162 | 163 | 164 | def E_step(self,X,evaluate=False): 165 | prob = np.zeros(shape=(X.shape[0],self.K)) 166 | pi_mul_P = [] 167 | for i in range(self.K): 168 | pi_mul_P.append(self.pi[i]*pmf_bernoulli(X,self.mu[i])) 169 | pi_mul_P_sum = np.sum(pi_mul_P,axis=0) 170 | for i in range(self.K): 171 | prob[:,i] = pi_mul_P[i]/pi_mul_P_sum 172 | 173 | prob = update_prob_using_mapping(prob, self.dev_set_indices, self.dev_set_labels,evaluate) 174 | prob = set_prob_dev_values(prob, self.dev_set_indices, self.dev_set_labels) 175 | return prob 176 | 177 | 178 | def M_step(self,X, prob): 179 | Ns = np.sum(prob,axis=0) 180 | self.pi = Ns/prob.shape[1] 181 | self.mu = [np.sum((X.T*prob[:,i]).T, axis=0)/Ns[i] for i in range(self.K)] 182 | 183 | if __name__ == "__main__": 184 | data = [[1, 1, 1, 1, 0, 0, 0, 0, 0, 0], 185 | [1, 1, 1, 1, 1, 1, 0, 0, 0, 0], 186 | [0, 0, 0, 1, 1, 1, 1, 1, 1, 1], 187 | [0, 0, 0, 0, 1, 1, 1, 1, 1, 1]] 188 | bmm = SemiBMM(2) 189 | print(bmm.fit_predict(np.array(data),dev_set_indices=[0,2],dev_set_labels=[0,1])) -------------------------------------------------------------------------------- /goggles/test/test.py: -------------------------------------------------------------------------------- 1 | from goggles import construct_image_affinity_matrices, GogglesDataset,infer_labels 2 | import numpy as np 3 | import pandas as pd 4 | from sklearn.metrics import accuracy_score 5 | if __name__ == '__main__': 6 | dataset = GogglesDataset.load_all_data("../data/cub_dataset/images") 7 | afs = construct_image_affinity_matrices(dataset) 8 | dev_set_indices, dev_set_labels = [0,1,2,90,91,92],[0,0,0,1,1,1] 9 | y_true = pd.read_csv("../data/cub_dataset/labels.csv") 10 | y_true = y_true['y'].values 11 | y_true[:int(y_true.shape[0]/2)] = 0 12 | prob = infer_labels(afs,dev_set_indices,dev_set_labels) 13 | pred_labels = np.argmax(prob,axis=1).astype(int) 14 | print("accuracy", accuracy_score(y_true,pred_labels)) 15 | -------------------------------------------------------------------------------- /goggles/theory/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/theory/__init__.py -------------------------------------------------------------------------------- /goggles/theory/theory.py: -------------------------------------------------------------------------------- 1 | from scipy.stats import beta 2 | import numpy as np 3 | import math 4 | 5 | 6 | def generate_d_matrix(n, acc, K): 7 | """ 8 | generate simulated d_matrix 9 | :param n: number of labeled examples per class 10 | :param acc: accuracy 11 | :param K: number of classes 12 | :return: 13 | """ 14 | d_rows = [] 15 | for i in range(K): 16 | pvals = [(1 - acc) / (K - 1)] * K 17 | pvals[i] = 0 18 | row_i = np.squeeze(np.random.multinomial(int(n * (1 - acc)), pvals, size=1)) 19 | row_i[i] = int(n * acc) 20 | d_rows.append(row_i) 21 | d = np.squeeze(np.array(d_rows)) 22 | print(np.trace(d)/np.sum(d)) 23 | return d 24 | 25 | 26 | def n_given_sum(n_class,n_sum,n_max,x_0): 27 | n_given_sum_list = [[None for _ in range(n_sum+1)] for _ in range(n_class+1)] 28 | def _n_given_sum_(n_class,n_sum,n_max): 29 | if n_given_sum_list[n_class][n_sum] is not None: 30 | return n_given_sum_list[n_class][n_sum] 31 | if n_class == 1: 32 | if n_max < n_sum: 33 | n_given_sum_list[n_class][n_sum] = 0 34 | return 0 35 | else: 36 | n_given_sum_list[n_class][n_sum] = 1/math.factorial(x_0 - n_sum-1) 37 | return n_given_sum_list[n_class][n_sum] 38 | if n_class < 1 and n_sum!=0: 39 | return 0 40 | n = 0 41 | for i in range(min(n_max,n_sum)+1): 42 | n += 1/math.factorial(x_0-i-1)*_n_given_sum_(n_class-1,n_sum-i,n_max) 43 | n_given_sum_list[n_class][n_sum] = n 44 | return n 45 | return _n_given_sum_(n_class,n_sum,n_max) 46 | 47 | 48 | class DevSetTheory: 49 | """ 50 | development set theory 51 | """ 52 | def __init__(self,d_matrix): 53 | """ 54 | :param d_matrix: a 2-d numpy array, D_matrix[i,j] is the number of instances 55 | of the ith class that falls into the jth cluster 56 | """ 57 | self.D_matrix = d_matrix 58 | self.K = d_matrix.shape[0] 59 | self.n=100 60 | self.alpha_list, self.p_alpha_list= self.p_alphas() 61 | self.d_alpha = self.alpha_list[1]-self.alpha_list[0] 62 | 63 | def log_likelihood(self,acc): 64 | log_L = 0 65 | for i in range(self.K): 66 | for j in range(self.K): 67 | if i == j: 68 | log_L += self.D_matrix[i,j]*np.log(acc) 69 | else: 70 | log_L += self.D_matrix[i,j]*np.log(1-acc) 71 | return log_L 72 | 73 | def P_d(self): 74 | prior = beta(1,1) 75 | P = 0 76 | for acc in np.linspace(1e-6, 1-1e-6,self.n): 77 | P+=prior.pdf(acc)*np.exp(self.log_likelihood(acc))*( 1 / self.n) 78 | return P 79 | 80 | def p_alphas(self): 81 | P_d_matrix = self.P_d() 82 | prior = beta(1, 1) 83 | p_alpha_list = [] 84 | acc_list = [] 85 | for acc in np.linspace(1e-6, 1-1e-6, self.n): 86 | p = prior.pdf(acc) * np.exp(self.log_likelihood(acc)) / P_d_matrix 87 | p_alpha_list.append(p) 88 | acc_list.append(acc) 89 | return acc_list,p_alpha_list 90 | 91 | def p_acc_greater(self,target_acc): 92 | acc_greater = 0 93 | for i in range(len(self.alpha_list)): 94 | acc = self.alpha_list[i] 95 | if acc > target_acc: 96 | acc_greater+=self.p_alpha_list[i]*self.d_alpha 97 | return acc_greater 98 | 99 | def feasibility_test(self,epsilon = 0.7): 100 | """ 101 | The probability of the task being feasible 102 | :param epsilon: threshold of the estimated accuracy that make the task considered to be feasible 103 | :return: the probability of feasibility 104 | """ 105 | acc_greater = self.p_acc_greater(epsilon) 106 | acc_greater = np.clip(acc_greater, 0, 1) 107 | return acc_greater 108 | 109 | 110 | def p_one_dim(self,acc,i_dim): 111 | dev_size = int(np.sum(self.D_matrix[i_dim, :])) 112 | x_0_min = int(math.ceil((dev_size + self.K - 1) / self.K)) 113 | 114 | def p(x_0): 115 | y_sum = self.K * x_0 - dev_size - (self.K - 1) 116 | 117 | return math.factorial(dev_size) / math.factorial(x_0) * math.exp( 118 | math.log(acc) * x_0 + math.log((1 - acc) / (self.K - 1)) * (dev_size - x_0)) * \ 119 | n_given_sum(self.K - 1, y_sum, x_0 - 1, x_0) 120 | 121 | prob = 0 122 | for x_0 in range(x_0_min, dev_size + 1): 123 | prob += p(x_0) 124 | return prob 125 | 126 | def dev_set_sufficiency_test(self): 127 | """ 128 | test whether the current dev set is sufficient. 129 | This offers a guidance on whether you should use a larger dev set. 130 | :return: the lower-bound probability whether the dev set is sufficiency enough 131 | to obtain the correct class-cluster mapping 132 | """ 133 | p = 0 134 | for i in range(len(self.alpha_list)): 135 | alpha = self.alpha_list[i] 136 | p_alpha = self.p_alpha_list[i] 137 | pl = 1 138 | for i_dim in range(self.K): 139 | pl*=self.p_one_dim(alpha, i_dim) 140 | p = p + pl*p_alpha*self.d_alpha 141 | p = np.clip(p,0,1) 142 | return p 143 | 144 | 145 | if __name__ == "__main__": 146 | d_matrix = generate_d_matrix(10,0.77,2) 147 | theory = DevSetTheory(d_matrix) 148 | acc_greater = theory.p_acc_greater(0.7) 149 | ps = theory.p_alpha_list 150 | print(acc_greater) 151 | print(theory.dev_set_sufficiency_test()) 152 | #plt.plot(np.linspace(0.01, 0.99, 100),ps) 153 | #plt.show() 154 | -------------------------------------------------------------------------------- /goggles/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chu-data-lab/GOGGLES/b660b95a5c72943cafd90e6dc3cdee3e629b9164/goggles/utils/__init__.py -------------------------------------------------------------------------------- /goggles/utils/constants.py: -------------------------------------------------------------------------------- 1 | import os as _os 2 | 3 | 4 | BASE_DIR = _os.path.dirname(_os.path.dirname(_os.path.abspath(__file__))) 5 | 6 | SCRATCH_DIR = _os.path.join(BASE_DIR, '_scratch') 7 | ALL_RUNS_DIR = _os.path.join(SCRATCH_DIR, 'runs') 8 | CACHE_DIR = _os.path.join(SCRATCH_DIR, 'cache') 9 | RETRAINED_MODELS_DIR = _os.path.join(SCRATCH_DIR, 'models-retrained') 10 | 11 | AWA2_DATA_DIR = _os.path.join(SCRATCH_DIR, 'AwA2') 12 | CHNCXR_DATA_DIR = _os.path.join(SCRATCH_DIR, 'ShenzhenXRay') 13 | CUB_DATA_DIR = _os.path.join(SCRATCH_DIR, 'CUB_200_2011') 14 | GTSRB_DATA_DIR = _os.path.join(SCRATCH_DIR, 'gtsrb') 15 | KAGGLE_CHEST_XRAY_DIR = _os.path.join(SCRATCH_DIR, 'kaggle', 'chest_xray') 16 | SURFACE_DATA_DIR = _os.path.join(SCRATCH_DIR, 'surface_dataset') 17 | 18 | LOGS_DIR_NAME = 'logs' 19 | IMAGES_DIR_NAME = 'images' 20 | PROTOTYPES_DIR_NAME = 'prototypes' 21 | 22 | MODEL_FILE_NAME = 'model.pt' 23 | -------------------------------------------------------------------------------- /goggles/utils/dataset.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from PIL import Image 4 | from torch.utils.data import Dataset 5 | from torchvision import transforms 6 | 7 | 8 | class GogglesDataset(Dataset): 9 | def __init__(self,path,transform): 10 | valid_images = [".jpg", ".gif", ".png"] 11 | self._data_path = path 12 | self.images_filename_list = [] 13 | for f in os.listdir(path): 14 | ext = os.path.splitext(f)[1] 15 | if ext.lower() not in valid_images: 16 | continue 17 | self.images_filename_list.append(f) 18 | self.images_filename_list = list(sorted(self.images_filename_list)) 19 | if transform is not None: 20 | self._transform = transform 21 | else: 22 | self._transform = transforms.Compose([transforms.ToTensor()]) 23 | 24 | def __getitem__(self, idx): 25 | """ 26 | read a image only when it is used 27 | :param idx: integer 28 | :return: 29 | """ 30 | filename = self.images_filename_list[idx] 31 | try: 32 | image_file = os.path.join(self._data_path, filename) 33 | image = Image.open(image_file).convert('RGB') 34 | image = self._transform(image) 35 | except: 36 | image = None 37 | return image 38 | 39 | def __len__(self): 40 | return len(self.images_filename_list) 41 | 42 | @classmethod 43 | def load_all_data(cls, root_dir, input_image_size=224): 44 | try: 45 | transform_resize = transforms.Resize( 46 | (input_image_size, input_image_size)) 47 | except AttributeError: 48 | transform_resize = transforms.Scale( 49 | (input_image_size, input_image_size)) 50 | 51 | transform_to_tensor = transforms.ToTensor() 52 | transform_normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], 53 | std=[0.229, 0.224, 0.225]) 54 | transformation = transforms.Compose([ 55 | transform_resize, transform_to_tensor, transform_normalize]) 56 | dataset = cls( 57 | root_dir, 58 | transform=transformation) 59 | return dataset -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | numpy 2 | pandas 3 | tqdm 4 | scikit-learn 5 | scipy 6 | Pillow 7 | torch 8 | torchvision -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | 4 | import setuptools 5 | 6 | directory = os.path.dirname(os.path.abspath(__file__)) 7 | 8 | # Extract version information 9 | path = os.path.join(directory, 'goggles', '__init__.py') 10 | with open(path) as read_file: 11 | text = read_file.read() 12 | pattern = re.compile(r"^__version__ = ['\"]([^'\"]*)['\"]", re.MULTILINE) 13 | version = pattern.search(text).group(1) 14 | 15 | # Extract long_description 16 | path = os.path.join(directory, 'README.md') 17 | with open(path) as read_file: 18 | long_description = read_file.read() 19 | 20 | 21 | #extract requirements 22 | requirementPath = os.path.join(directory, 'requirements.txt') 23 | install_requires = [] 24 | if os.path.isfile(requirementPath): 25 | with open(requirementPath) as f: 26 | install_requires = f.read().splitlines() 27 | 28 | setuptools.setup( 29 | name='goggles', 30 | version=version, 31 | url='https://github.com/chu-data-lab/GOGGLES', 32 | description='A system for automatically generating probabilistic labels for image datasets based on the affinity coding paradigm', 33 | long_description_content_type='text/markdown', 34 | long_description=long_description, 35 | license='MIT License', 36 | packages=setuptools.find_packages(), 37 | include_package_data=True, 38 | install_requires = install_requires, 39 | keywords='traning-data-generation machine-learning ai information-extraction affinity-coding', 40 | classifiers=[ 41 | 'Intended Audience :: Science/Research', 42 | 'Topic :: Scientific/Engineering :: Bio-Informatics', 43 | 'Topic :: Scientific/Engineering :: Information Analysis', 44 | 'License :: OSI Approved :: Apache Software License', 45 | 'Programming Language :: Python :: 3', 46 | ], 47 | 48 | project_urls={ # Optional 49 | 'Homepage': 'https://github.com/chu-data-lab/GOGGLES', 50 | 'Source': 'https://github.com/chu-data-lab/GOGGLES', 51 | 'Bug Reports': 'https://github.com/chu-data-lab/GOGGLES/issues', 52 | 'Citation': 'https://arxiv.org/abs/1903.04552', 53 | }, 54 | ) -------------------------------------------------------------------------------- /tools/get_cub_dataset.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | TIMESTAMP=`date "+%Y-%m-%d %H:%M:%S"` 4 | 5 | declare -r CUB_DOWNLOAD_URL="http://www.vision.caltech.edu/visipedia-data/CUB-200-2011/CUB_200_2011.tgz" 6 | 7 | usage() { 8 | echo "This script downloads and extracts the CUB 2011 dataset from the following URL:" 9 | echo " - ${CUB_DOWNLOAD_URL}" 10 | echo "" 11 | echo "USAGE:" 12 | echo "\$ sh get_cub_dataset.sh /path/to/_scratch" 13 | echo "" 14 | } 15 | 16 | download() { 17 | local DOWNLOAD_URL="$1" 18 | local TARGET_DIR=$(cd "$2" && pwd) 19 | 20 | echo "[${TIMESTAMP}] Downloading ${DOWNLOAD_URL} to ${TARGET_DIR}" 21 | 22 | wget -q --show-progress "${DOWNLOAD_URL}" -P "${TARGET_DIR}" 23 | } 24 | 25 | untar_and_cleanup() { 26 | local TARGET_DIR=$(cd $(dirname "$1") && pwd) 27 | local TARGET_FILEPATH="$1" 28 | 29 | echo "[${TIMESTAMP}] Extracting data from $1" 30 | 31 | tar -C "${TARGET_DIR}" -xzf "${TARGET_FILEPATH}" 32 | rm -f "${TARGET_FILEPATH}" 33 | } 34 | 35 | if [ $# -ne 1 ]; then 36 | usage 37 | echo "ERROR: need path to _scratch directory" 38 | echo "" 39 | exit 1 40 | fi 41 | 42 | if [[ ( $1 == "--help") || $1 == "-h" ]]; then 43 | usage 44 | exit 0 45 | fi 46 | 47 | if [ ! -d "$1" ]; then 48 | usage 49 | echo "ERROR: $1 does not exist" 50 | echo "" 51 | exit 1 52 | fi 53 | 54 | declare -r SCRATCH_DIR=$(cd "$1" && pwd) 55 | 56 | DOWNLOAD_TARGET_DIR="${SCRATCH_DIR}/CUB_200_2011" 57 | DOWNLOAD_FILENAME=$(basename "${CUB_DOWNLOAD_URL}") 58 | 59 | mkdir -p "${DOWNLOAD_TARGET_DIR}" 60 | 61 | download "${CUB_DOWNLOAD_URL}" "${DOWNLOAD_TARGET_DIR}" 62 | untar_and_cleanup "${DOWNLOAD_TARGET_DIR}/${DOWNLOAD_FILENAME}" 63 | 64 | echo "[${TIMESTAMP}] Finished downloading the CUB 2011 dataset" 65 | --------------------------------------------------------------------------------