├── .gitignore ├── .gitmodules ├── README.md ├── data ├── cities │ └── mean_image.binaryproto └── data.txt ├── models └── models.txt ├── scripts ├── aws_s3_utility.py ├── city_data_utils.py ├── data_utils.py ├── file_utils.py ├── get_color.py ├── get_dataset.py ├── human_baseline.py └── linear_classification.py └── tests └── run_tests.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | bin/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | eggs/ 16 | lib/ 17 | lib64/ 18 | parts/ 19 | sdist/ 20 | var/ 21 | *.egg-info/ 22 | .installed.cfg 23 | *.egg 24 | 25 | # Installer logs 26 | pip-log.txt 27 | pip-delete-this-directory.txt 28 | 29 | # Unit test / coverage reports 30 | htmlcov/ 31 | .tox/ 32 | .coverage 33 | .cache 34 | nosetests.xml 35 | coverage.xml 36 | 37 | # Translations 38 | *.mo 39 | 40 | # Mr Developer 41 | .mr.developer.cfg 42 | .project 43 | .pydevproject 44 | 45 | # Rope 46 | .ropeproject 47 | 48 | # Django stuff: 49 | *.log 50 | *.pot 51 | 52 | # Sphinx documentation 53 | docs/_build/ 54 | 55 | # mac 56 | .DS_Store 57 | 58 | # binaries 59 | *.bin 60 | *.out 61 | 62 | #? 63 | *~ 64 | 65 | # ignore key 66 | *.key 67 | 68 | # data and models 69 | *.jpg 70 | *.png 71 | *.caffemodel 72 | *.solverstate -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "LittlePlaNet-Models"] 2 | path = LittlePlaNet-Models 3 | url = https://github.com/wulfebw/LittlePlaNet-Models.git 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # City-Recognition 2 | 3 | This project is a smale-scale spin on Google DeepMind's PlaNet geolocalization Neural Network. It was developed for CS231n at Stanford University. 4 | 5 | Google's PlaNet: http://arxiv.org/abs/1602.05314 6 | 7 | # Getting Started 8 | 9 | ## Downloading the dataset 10 | - the dataset we used cannot be included in the repo, but we have provided the [script](https://github.com/dmakian/LittlePlaNet/blob/master/scripts/get_dataset.py) we used to download the data from google street view 11 | + you need a google street view account and key to use this script 12 | + create an account [here](https://developers.google.com/maps/documentation/streetview/) 13 | + after creating an account, get an api key, and store it in a file called "api_key.key" in the main repo directory 14 | + this will allow you to run the script and download the images to use as the dataset 15 | + the dataset downloads images from various cities that have been hard-coded in the `get_dataset.py` script, so change these coordinates if you'd like to focus on different cities 16 | 17 | ## Feature extraction 18 | - this project used a pre-trained CNN to extract features for use with a linear classifier 19 | + for access to the trained models, see the [model repo](https://github.com/wulfebw/LittlePlaNet-Models) 20 | - for extracting features see [`extract_features.py`](https://github.com/wulfebw/LittlePlaNet-Models/blob/master/scripts/extract_features.py) 21 | 22 | ## Model training 23 | - after extracting features, you can use them with a variety of classification models 24 | - see [`linear_classification.py`](https://github.com/dmakian/LittlePlaNet/blob/master/scripts/linear_classification.py) for an example script for training and evaluating some baseline models on the extracted features 25 | -------------------------------------------------------------------------------- /data/cities/mean_image.binaryproto: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davidhershey/LittlePlaNet/7b39a376d7ff1020777577296d859d73f6c7a474/data/cities/mean_image.binaryproto -------------------------------------------------------------------------------- /data/data.txt: -------------------------------------------------------------------------------- 1 | place datasets here in the following format: 2 | - base level directory should be the name of the dataset 3 | - below that should be a directory containing the images 4 | - at the same level should be two text files listing the train/val image names and their labels 5 | - at the same level should be two databases, one called train_lmdb and another called val_lmdb 6 | -------------------------------------------------------------------------------- /models/models.txt: -------------------------------------------------------------------------------- 1 | place models here 2 | -------------------------------------------------------------------------------- /scripts/aws_s3_utility.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import boto 4 | 5 | from boto.s3.key import Key 6 | from boto.s3.connection import S3Connection 7 | 8 | class S3Utility(object): 9 | """ 10 | :description: An AWS S3 utility class. 11 | 12 | This is a class rather than a module because some state is required 13 | to avoid having to reestablish an s3 connection each transaction. 14 | 15 | :type access_key: string 16 | :param access_key: aws access key 17 | 18 | :type secret_key: string 19 | :param secret_key: aws secret key 20 | 21 | """ 22 | 23 | def __init__(self, access_key, secret_key, s3_bucket): 24 | self.access_key = access_key 25 | self.secret_key = secret_key 26 | self.s3_bucket = s3_bucket 27 | self._conn = None 28 | 29 | @property 30 | def conn(self): 31 | if self._conn is not None: 32 | return self._conn 33 | else: 34 | return S3Connection(self.access_key, self.secret_key) 35 | 36 | def download_file_list(self, prefix=''): 37 | """ 38 | :description: loads the name of the files in a bucket. 39 | Optionally returns only those filenames that start with prefix. 40 | """ 41 | 42 | # select the bucket, where input_s3_bucket takes the form 'bsdsdata' 43 | bucket = self.conn.get_bucket(self.s3_bucket) 44 | 45 | # collect the list of files to process - those that start with the data group id 46 | file_list = [] 47 | for key in bucket.list(): 48 | key_name = key.name.encode('utf-8') 49 | if key_name.startswith(prefix): 50 | file_list.append(key_name) 51 | 52 | return file_list 53 | 54 | def download_file(self, file_to_load, local_save_dir): 55 | """ 56 | :description: load a file from a given s3 bucket with a 57 | given name and save to a local dir 58 | 59 | :type s3_bucket: string 60 | :param s3_bucket: s3 bucket from which to load the file 61 | 62 | :type file_to_load: string 63 | :param file_to_load: the file to load 64 | 65 | :type local_save_dir: string 66 | :param local_save_dir: the local dir to which to save the downloaded file 67 | 68 | :return: the location where the file was saved 69 | """ 70 | 71 | # select the bucket, where input_s3_bucket takes the form 'bsdsdata' 72 | bucket = self.conn.get_bucket(self.s3_bucket) 73 | 74 | # set a key to the processed files list 75 | key = Key(bucket, file_to_load) 76 | key_name = key.name.encode('utf-8') 77 | 78 | # download the file to process and save in the input location 79 | save_location = os.path.join(local_save_dir, key_name) 80 | try: 81 | key.get_contents_to_filename(save_location) 82 | except boto.exception.S3ResponseError as e: 83 | raise boto.exception.S3ResponseError("key name: {} failed".format(key_name)) 84 | 85 | # return the location of the downloaded file 86 | return save_location 87 | 88 | def upload_file(self, filename_to_save_as, file_path): 89 | """ 90 | :description: uploads a single file to an s3 bucket 91 | """ 92 | # what is this? 93 | def percent_cb(complete, total): 94 | sys.stdout.write('.') 95 | sys.stdout.flush() 96 | 97 | # select the bucket, where input_s3_bucket takes the form 'bsdsdata' 98 | bucket = self.conn.get_bucket(self.s3_bucket) 99 | 100 | # send the file to the s3 bucket 101 | key = Key(bucket) 102 | key.key = filename_to_save_as 103 | key.set_contents_from_filename(file_path, cb=percent_cb, num_cb=50) 104 | 105 | def upload_directory(self, directory): 106 | """ 107 | :description: upload all the files in a directory to aws s3 108 | """ 109 | 110 | filepaths = [] 111 | for root, dirs, files in os.walk(directory): 112 | for filename in files: 113 | filepaths.append(os.path.join(root, filename)) 114 | 115 | upload_directory = os.path.basename(directory) 116 | for filepath in filepaths: 117 | dest_filepath = os.path.join(upload_directory, filepath.split(upload_directory)[-1][1:]) 118 | self.upload_file(dest_filepath, filepath) 119 | 120 | 121 | -------------------------------------------------------------------------------- /scripts/city_data_utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | :description: Utilities for loading in, preprocessing, and organizing data. 3 | """ 4 | import cv2 5 | import glob 6 | import numpy as np 7 | import os 8 | import random 9 | import scipy.io 10 | 11 | # filename and directory constants 12 | DATA_DIR = '../data/cities' 13 | OUTPUT_DIR = '../data/cities' 14 | 15 | # data split constants 16 | TRAIN_RATIO = .8 17 | VAL_RATIO = .1 18 | TEST_RATIO = .1 19 | 20 | def get_city_directories(dataset_dir): 21 | pattern = os.path.join(dataset_dir, '*/') 22 | filepaths = glob.glob(pattern) 23 | return filepaths 24 | 25 | def get_all_image_filepaths(dataset_dir, pattern): 26 | pattern = os.path.join(dataset_dir, pattern) 27 | filepaths = glob.glob(pattern) 28 | filenames = [filepath[filepath.index('cities/') + 7:] for filepath in filepaths] 29 | return filepaths, filenames 30 | 31 | def generate_train_split(labels): 32 | # randomly determine the set of train locations 33 | train_split = set() 34 | for filepath, label in labels.iteritems(): 35 | if random.random() < TRAIN_RATIO: 36 | train_split.add(filepath) 37 | return train_split 38 | 39 | def get_city_name_from_city_directory(city_directory): 40 | city_name = city_directory.split('/')[-2] 41 | return city_name 42 | 43 | def write_labels_file(dataset_dir, train_output_filepath, val_output_filepath, test_output_filepath, city_labels_filepath): 44 | 45 | # load the city directories e.g., '../data/cities/Barcelona/' 46 | city_directories = sorted(get_city_directories(dataset_dir)) 47 | 48 | # populate a dictionary with the label of each image 49 | image_labels = dict() 50 | city_labels = dict() 51 | for city_idx, city_directory in enumerate(city_directories): 52 | 53 | city_name = get_city_name_from_city_directory(city_directory) 54 | city_labels[city_name] = city_idx 55 | filepaths, filenames = get_all_image_filepaths(city_directory, pattern='*.jpg') 56 | for filename in filenames: 57 | image_labels[filename] = city_idx 58 | 59 | # randomly generate the train split (and implictly the validation split) 60 | # not by image but by location 61 | train_split = generate_train_split(image_labels) 62 | 63 | # format the lines for each data entry 64 | train_lines = [] 65 | val_lines = [] 66 | filepaths, filenames = get_all_image_filepaths(dataset_dir, pattern='*/*.jpg') 67 | for filepath, filename in zip(filepaths, filenames): 68 | 69 | # if filename in the labels then write it 70 | if filename in image_labels: 71 | label = image_labels[filename] 72 | else: 73 | print 'cant find label' 74 | continue 75 | 76 | string = '{} {}\n'.format(filename, label) 77 | if filename in train_split: 78 | train_lines.append(string) 79 | else: 80 | val_lines.append(string) 81 | 82 | # write city labels to file 83 | with open(city_labels_filepath, 'wb') as f: 84 | for city, cidx in city_labels.iteritems(): 85 | f.write('{} {}\n'.format(city, cidx)) 86 | 87 | # write train lines to file 88 | with open(train_output_filepath, 'wb') as f: 89 | f.writelines(np.random.permutation(train_lines)) 90 | 91 | # split val into test and val using mid point 92 | mid = len(val_lines) / 2 93 | 94 | # write val lines to file 95 | with open(val_output_filepath, 'wb') as f: 96 | f.writelines(np.random.permutation(val_lines[:mid])) 97 | 98 | # write test lines to file 99 | with open(test_output_filepath, 'wb') as f: 100 | f.writelines(np.random.permutation(val_lines[mid:])) 101 | 102 | if __name__ == '__main__': 103 | train_dir = os.path.join(OUTPUT_DIR) 104 | train_data_file = os.path.join(OUTPUT_DIR, 'train.txt') 105 | val_data_file = os.path.join(OUTPUT_DIR, 'val.txt') 106 | test_data_file = os.path.join(OUTPUT_DIR, 'test.txt') 107 | city_labels_file = os.path.join(OUTPUT_DIR, 'city_labels.txt') 108 | assert os.path.isdir(train_dir), 'train directory: {} not found'.format(train_dir) 109 | if os.path.exists(train_data_file): 110 | open(train_data_file, 'a').close() 111 | if os.path.exists(val_data_file): 112 | open(val_data_file, 'a').close() 113 | if os.path.exists(test_data_file): 114 | open(test_data_file, 'a').close() 115 | if os.path.exists(city_labels_file): 116 | open(city_labels_file, 'a').close() 117 | 118 | write_labels_file(train_dir, train_data_file, val_data_file, test_data_file, city_labels_file) 119 | 120 | 121 | -------------------------------------------------------------------------------- /scripts/data_utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | :description: Utilities for loading in, preprocessing, and organizing data. 3 | """ 4 | import cv2 5 | import glob 6 | import numpy as np 7 | import os 8 | import random 9 | import scipy.io 10 | 11 | # filename and directory constants 12 | DATA_DIR = '../data/crcv' 13 | OUTPUT_DIR = '../data/crcv' 14 | LABLES_FILENAME = 'GPS_Long_Lat_Compass.mat' 15 | HIST_FILENAME = 'Color_hist.mat' 16 | GIST_FILENAME = 'GIST.mat' 17 | 18 | # data split constants 19 | TRAIN_RATIO = .8 20 | VAL_RATIO = .1 21 | TEST_RATIO = .1 22 | 23 | # feature constants 24 | NUM_HIST_FEATURES = 60 25 | NUM_HIST_ROWS_PER_LOCATION = 5 26 | 27 | # preprocessing constants 28 | IMAGE_SIDE_LENGTH = 256 29 | NUM_IMAGE_CHANNELS = 3 30 | 31 | # latitude and longitude min and max for each city 32 | ORLANDO_LAT_MIN = 28 33 | ORLANDO_LAT_MAX = 29 34 | ORLANDO_LONG_MIN = -82 35 | ORLANDO_LONG_MAX = -81 36 | NYC_LAT_MIN = 40 37 | NYC_LAT_MAX = 42 38 | NYC_LONG_MIN = -75 39 | NYC_LONG_MAX = -73 40 | PITT_LAT_MIN = 40 41 | PITT_LAT_MAX = 42 42 | PITT_LONG_MIN = -81 43 | PITT_LONG_MAX = -79 44 | 45 | # city labels 46 | ORL_LABEL = 0 47 | NYC_LABEL = 1 48 | PITT_LABEL = 2 49 | 50 | def load_mat(filename): 51 | """ 52 | :description: Load and return the dictionary in a matlab file 53 | """ 54 | assert filename.endswith('.mat'), 'load_mat requires .mat file' 55 | arr = scipy.io.loadmat(os.path.join(DATA_DIR, filename)) 56 | return arr 57 | 58 | def load_labels_as_dict(): 59 | """ 60 | :description: Loads labels for each row in the dataset. Labels returned 61 | as a dict where row number is the key and the label is the value, which 62 | is one of {1 = orlando, 2 = nyc, 3 = pitt}. 63 | """ 64 | gps = load_mat(LABLES_FILENAME)['GPS_Compass'] 65 | 66 | # create mapping of 67 | labels = dict() 68 | for idx, (latitude, longitude, angle) in enumerate(gps): 69 | 70 | # orlando 71 | if latitude > ORLANDO_LAT_MIN and latitude < ORLANDO_LAT_MAX \ 72 | and longitude > ORLANDO_LONG_MIN and longitude < ORLANDO_LONG_MAX: 73 | labels[idx] = ORL_LABEL 74 | 75 | # nyc 76 | elif latitude > NYC_LAT_MIN and latitude < NYC_LAT_MAX \ 77 | and longitude > NYC_LONG_MIN and longitude < NYC_LONG_MAX: 78 | labels[idx] = NYC_LABEL 79 | 80 | # pitt 81 | elif latitude > PITT_LAT_MIN and latitude < PITT_LAT_MAX \ 82 | and longitude > PITT_LONG_MIN and longitude < PITT_LONG_MAX: 83 | labels[idx] = PITT_LABEL 84 | 85 | # invalid latitude or longitude 86 | else: 87 | raise ValueError('Latitude or longitude not in valid range.\ 88 | Longitude: {}\tLatitude: {}'.format(latitude, longitude)) 89 | 90 | return labels 91 | 92 | def load_labels_as_list(): 93 | """ 94 | :description: Loads labels for each row in the dataset. Labels returned 95 | as a list of values where the index of the value is the XXXXXXX row 96 | of the corresponding image and the label itself is one of 97 | {1 = orlando, 2 = nyc, 3 = pitt}. 98 | """ 99 | gps = load_mat(LABLES_FILENAME)['GPS_Compass'] 100 | 101 | # collect the row numbers that correspond to each city 102 | labels = [] 103 | for idx, (latitude, longitude, angle) in enumerate(gps): 104 | 105 | # orlando 106 | if latitude > ORLANDO_LAT_MIN and latitude < ORLANDO_LAT_MAX \ 107 | and longitude > ORLANDO_LONG_MIN and longitude < ORLANDO_LONG_MAX: 108 | labels.append((idx, ORL_LABEL)) 109 | 110 | # nyc 111 | elif latitude > NYC_LAT_MIN and latitude < NYC_LAT_MAX \ 112 | and longitude > NYC_LONG_MIN and longitude < NYC_LONG_MAX: 113 | labels.append((idx, NYC_LABEL)) 114 | 115 | # pitt 116 | elif latitude > PITT_LAT_MIN and latitude < PITT_LAT_MAX \ 117 | and longitude > PITT_LONG_MIN and longitude < PITT_LONG_MAX: 118 | labels.append((idx, PITT_LABEL)) 119 | 120 | # invalid latitude or longitude 121 | else: 122 | raise ValueError('Latitude or longitude not in valid range.\ 123 | Longitude: {}\tLatitude: {}'.format(latitude, longitude)) 124 | 125 | # return the label corresponding to each row 126 | return np.array(labels)[:, 1] 127 | 128 | def load_features_and_labels(feature_set='all'): 129 | """ 130 | :description: load features and corresponding labels 131 | 132 | :type feature_set: string 133 | :param feature_set: Which features to load. One of {'all', 'hist', 'gist'} 134 | """ 135 | 136 | # load the labels with the features so we can randomly permute the order 137 | labels = load_labels_as_list() 138 | 139 | # repeat the labels for each location 140 | labels = np.tile(labels, (NUM_HIST_ROWS_PER_LOCATION, 1)).T.reshape(-1) 141 | 142 | # reshape to align with features before permute 143 | labels = labels.reshape((-1, NUM_HIST_ROWS_PER_LOCATION)) 144 | 145 | # load features 146 | if feature_set == 'all': 147 | hist = load_mat(HIST_FILENAME)['color_hist'] 148 | gist = load_mat(GIST_FILENAME)['RefImGIST'] 149 | feats = np.hstack((hist, gist)) 150 | elif feature_set == 'hist': 151 | feats = load_mat(HIST_FILENAME)['color_hist'] 152 | elif feature_set == 'gist': 153 | feats = load_mat(GIST_FILENAME)['RefImGIST'] 154 | else: 155 | raise ValueError("""feature_set must be one of {'all', 'hist', 'gist'}""") 156 | 157 | # remove all zero rows - rows with data are separated by 5 all zero 158 | # rows, i'm not sure why but remove them here 159 | feats = feats[np.any(feats != 0, axis=1)] 160 | feats = feats.reshape((-1, NUM_HIST_ROWS_PER_LOCATION, feats.shape[-1])) 161 | 162 | # permute features and labels 163 | # we want the test set to always be the same images, so set random seed before permuting 164 | np.random.seed(42) 165 | idxs = np.random.permutation(len(feats)) 166 | feats = feats[idxs] 167 | feats = feats.reshape((-1, feats.shape[-1])) 168 | labels = labels[idxs] 169 | labels = labels.reshape((-1)) 170 | 171 | # train, val, test split 172 | train_idx = int(len(feats) * TRAIN_RATIO) 173 | val_idx = int(len(feats) * (TRAIN_RATIO + VAL_RATIO)) 174 | train = feats[:train_idx] 175 | y_train = labels[:train_idx] 176 | val = feats[train_idx: val_idx] 177 | y_val = labels[train_idx: val_idx] 178 | test = feats[val_idx:] 179 | y_test = labels[val_idx:] 180 | 181 | # compute mean and std on train data 182 | means = np.mean(train, axis=0) 183 | stds = np.std(train, axis=0) 184 | 185 | # and use those stats to normalize train, val, test 186 | train = (train - means) / stds 187 | val = (val - means) / stds 188 | test = (test - means) / stds 189 | 190 | return train, y_train, val, y_val, test, y_test 191 | 192 | def subtract_mean_image(imgs): 193 | mean_rgb_values = imgs.mean(axis=(0,1,2)) 194 | imgs -= mean_rgb_values 195 | return imgs 196 | 197 | def resize_image(img, side_length): 198 | """ 199 | :description: Resize an image proportionally. 200 | """ 201 | # resize proportionally and then crop to avoid warping image 202 | h, w, c = img.shape 203 | new_h, new_w = side_length, side_length 204 | if h > w: 205 | new_h = side_length * int(h / float(w)) 206 | else: 207 | new_w = side_length * int(w / float(h)) 208 | 209 | # crop center 210 | img = cv2.resize(img, (new_h, new_w)) 211 | h_offset = (new_h - side_length) / 2 212 | w_offset = (new_w - side_length) / 2 213 | img = img[h_offset:h_offset + side_length, w_offset:w_offset + side_length] 214 | return img 215 | 216 | def write_images_to_directory(output_dir, imgs, filenames): 217 | for img, filename in zip(imgs, filenames): 218 | if filename is not None: 219 | filepath = os.path.join(output_dir, filename) 220 | cv2.imwrite(filepath, img) 221 | 222 | def image_filename_to_id(filename): 223 | return int(filename[: filename.rindex('_')]) 224 | 225 | def get_image_filepaths(dataset_dir): 226 | pattern = os.path.join(dataset_dir, '*.jpg') 227 | filepaths = glob.glob(pattern) 228 | filenames = [filepath[filepath.rindex('/') + 1:] for filepath in filepaths] 229 | return filepaths, filenames 230 | 231 | def preprocess_crcv(dataset_dir, output_dir): 232 | # get filepaths of images to process and their corresponding ids 233 | filepaths, img_filenames = get_image_filepaths(dataset_dir) 234 | num_imgs = len(filepaths) 235 | 236 | # allocate containers for processed images and filenames 237 | imgs = np.empty((num_imgs, IMAGE_SIDE_LENGTH, IMAGE_SIDE_LENGTH, NUM_IMAGE_CHANNELS)) 238 | filenames = [None] * num_imgs 239 | 240 | # load and resize each image 241 | for idx, (filepath, filename) in enumerate(zip(filepaths, img_filenames)): 242 | print('processing filepath: {}'.format(filepath)) 243 | img = cv2.imread(filepath) 244 | if img is None: 245 | print('image filepath failed to load: {}'.format(filepath)) 246 | else: 247 | resized_img = resize_image(img, IMAGE_SIDE_LENGTH) 248 | imgs[idx] = resized_img 249 | filenames[idx] = filename 250 | 251 | # subtract mean image from images 252 | #imgs = subtract_mean_image(imgs) 253 | 254 | # write the preprocessed images to dir 255 | write_images_to_directory(output_dir, imgs, filenames) 256 | 257 | def generate_train_split(labels): 258 | # randomly determine the set of train locations 259 | train_split = set() 260 | for loc, label in labels.iteritems(): 261 | if random.random() < TRAIN_RATIO: 262 | train_split.add(loc) 263 | return train_split 264 | 265 | def write_labels_file(dataset_dir, train_output_filepath, val_output_filepath, test_output_filepath): 266 | # collect the filepaths in the directory 267 | filepaths, filenames = get_image_filepaths(dataset_dir) 268 | 269 | # load labels 270 | labels = load_labels_as_dict() 271 | 272 | # randomly generate the train split (and implictly the validation split) 273 | # not by image but by location 274 | train_split = generate_train_split(labels) 275 | 276 | # format the lines for each data entry 277 | train_lines = [] 278 | val_lines = [] 279 | for filepath, filename in zip(filepaths, filenames): 280 | img_id = image_filename_to_id(filename) 281 | if filename[-5] == '5': 282 | continue 283 | if img_id in labels: 284 | label = labels[img_id] 285 | else: 286 | print 'cant find label' 287 | string = '{} {}\n'.format(filename, label) 288 | if img_id in train_split: 289 | train_lines.append(string) 290 | else: 291 | val_lines.append(string) 292 | 293 | # write train lines to file 294 | with open(train_output_filepath, 'wb') as f: 295 | f.writelines(np.random.permutation(train_lines)) 296 | 297 | # split val into test and val using mid point 298 | mid = len(val_lines) / 2 299 | 300 | # write val lines to file 301 | with open(val_output_filepath, 'wb') as f: 302 | f.writelines(np.random.permutation(val_lines[:mid])) 303 | 304 | # write test lines to file 305 | with open(test_output_filepath, 'wb') as f: 306 | f.writelines(np.random.permutation(val_lines[mid:])) 307 | 308 | if __name__ == '__main__': 309 | # dataset_dir = os.path.join(DATA_DIR, 'part9') 310 | # output_dir = os.path.join(OUTPUT_DIR, 'part9_resized') 311 | # assert os.path.isdir(dataset_dir), 'dataset directory: {} not found'.format(dataset_dir) 312 | # if not os.path.isdir(output_dir): 313 | # os.mkdir(output_dir) 314 | # preprocess_crcv(dataset_dir, output_dir) 315 | 316 | train_dir = os.path.join(OUTPUT_DIR, 'resized') 317 | train_data_file = os.path.join(OUTPUT_DIR, 'train.txt') 318 | val_data_file = os.path.join(OUTPUT_DIR, 'val.txt') 319 | test_data_file = os.path.join(OUTPUT_DIR, 'test.txt') 320 | assert os.path.isdir(train_dir), 'train directory: {} not found'.format(train_dir) 321 | assert os.path.exists(train_data_file), 'train data file: {} not found'.format(train_data_file) 322 | assert os.path.exists(val_data_file), 'val data file: {} not found'.format(val_data_file) 323 | assert os.path.exists(test_data_file), 'test data file: {} not found'.format(test_data_file) 324 | write_labels_file(train_dir, train_data_file, val_data_file, test_data_file) 325 | 326 | 327 | -------------------------------------------------------------------------------- /scripts/file_utils.py: -------------------------------------------------------------------------------- 1 | 2 | import os 3 | 4 | import aws_s3_utility 5 | 6 | 7 | def is_valid(key): 8 | return key.replace('-','').isalnum() 9 | 10 | def load_key(filepath): 11 | assert os.path.exists(filepath), 'filepath: {} not found'.format(filepath) 12 | 13 | key = None 14 | with open(filepath, 'rb') as f: 15 | key = f.readline() 16 | if is_valid(key): 17 | return key 18 | else: 19 | raise ValueError('invalid key: {}'.format(key)) 20 | 21 | def upload_directory_to_aws(directory): 22 | ak = load_key('../access_key.key') 23 | sk = load_key('../secret_key.key') 24 | bucket = 'littleplanet' 25 | aws_util = aws_s3_utility.S3Utility(ak, sk, bucket) 26 | aws_util.upload_directory(directory) 27 | 28 | if __name__ == '__main__': 29 | directory_names = ['Barcelona', 'DC', 'Detroit', 'London', 'Moscow', 'NYC', 'Paris', 'Rio', 'SanFran', 'Sydney'] 30 | for directory_name in directory_names: 31 | directory_path = os.path.join('../imgs', directory_name) 32 | upload_directory_to_aws(directory_path) 33 | -------------------------------------------------------------------------------- /scripts/get_color.py: -------------------------------------------------------------------------------- 1 | # http://charlesleifer.com/blog/using-python-and-k-means-to-find-the-dominant-colors-in-images/ 2 | from collections import namedtuple 3 | from math import sqrt 4 | import random 5 | 6 | from PIL import Image # pip install pillow 7 | 8 | Point = namedtuple('Point', ('coords', 'n', 'ct')) 9 | Cluster = namedtuple('Cluster', ('points', 'center', 'n')) 10 | 11 | 12 | def get_points(img): 13 | points = [] 14 | w, h = img.size 15 | for count, color in img.getcolors(w * h): 16 | points.append(Point(color, 3, count)) 17 | return points 18 | 19 | 20 | rtoh = lambda rgb: '#%s' % ''.join(('%02x' % p for p in rgb)) 21 | 22 | 23 | def get_color(image, n=1): 24 | img = Image.open(image) 25 | img.thumbnail((200, 200)) 26 | w, h = img.size 27 | 28 | points = get_points(img) 29 | clusters = kmeans(points, n, 1) 30 | rgbs = [map(int, c.center.coords) for c in clusters] 31 | return map(rtoh, rgbs) 32 | 33 | 34 | def euclidean(p1, p2): 35 | return sqrt(sum([(p1.coords[i] - p2.coords[i]) ** 2 for i in range(p1.n)])) 36 | 37 | 38 | def calculate_center(points, n): 39 | vals = [0.0 for i in range(n)] 40 | plen = 0 41 | for p in points: 42 | plen += p.ct 43 | for i in range(n): 44 | vals[i] += (p.coords[i] * p.ct) 45 | return Point([(v / plen) for v in vals], n, 1) 46 | 47 | 48 | def kmeans(points, k, min_diff): 49 | clusters = [Cluster([p], p, p.n) for p in random.sample(points, k)] 50 | 51 | while 1: 52 | plists = [[] for i in range(k)] 53 | 54 | for p in points: 55 | smallest_distance = float('Inf') 56 | for i in range(k): 57 | distance = euclidean(p, clusters[i].center) 58 | if distance < smallest_distance: 59 | smallest_distance = distance 60 | idx = i 61 | plists[idx].append(p) 62 | 63 | diff = 0 64 | for i in range(k): 65 | old = clusters[i] 66 | center = calculate_center(plists[i], old.n) 67 | new = Cluster(plists[i], center, old.n) 68 | clusters[i] = new 69 | diff = max(diff, euclidean(old.center, new.center)) 70 | 71 | if diff < min_diff: 72 | break 73 | 74 | return clusters 75 | -------------------------------------------------------------------------------- /scripts/get_dataset.py: -------------------------------------------------------------------------------- 1 | 2 | import math 3 | import os 4 | import random 5 | import sys 6 | from multiprocessing.pool import ThreadPool 7 | import urllib 8 | 9 | import file_utils 10 | 11 | cities = { 12 | "Paris" : (48.8567,2.3508), 13 | "London" : (51.5072,-0.1275), 14 | "Barcelona" : (41.3833,2.1833), 15 | "Moscow" : (55.7500,37.6167), 16 | "Sydney" : (-33.8650,151.2094), 17 | "Rio" : (-22.9068,-43.1729), 18 | "NYC" : (40.7127,-74.0059), 19 | "SanFran" : (37.7833,-122.4167), 20 | "Detroit" : (42.3314,-83.0458), 21 | "DC" : (38.9047,-77.0164) 22 | } 23 | 24 | # radius of the Earth 25 | R = 6378.1 26 | 27 | # radius of images around center of city 28 | IMAGE_RADIUS = 10 29 | 30 | # number of images to download from each city 31 | NUM_IMAGES_PER_CITY = 200 32 | 33 | # size of failed-download image 34 | FAILED_DOWNLOAD_IMAGE_SIZE = 3554 35 | 36 | # place key in a file in the Geo-Localization directory 37 | # as the only text in the file on one line 38 | KEY_FILEPATH = "../api_key.key" 39 | API_KEY = file_utils.load_key(KEY_FILEPATH) 40 | GOOGLE_URL = ("http://maps.googleapis.com/maps/api/streetview?sensor=false&" 41 | "size=256x256&fov=120&key=" + API_KEY) 42 | IMAGES_DIR = '../imgs/' 43 | 44 | 45 | def download_images_for_city(city, lat, lon): 46 | print 'downloading images of {}'.format(city) 47 | num_imgs = 0 48 | misses = 0 49 | cur_directory = os.path.join(IMAGES_DIR, city) 50 | if not os.path.exists(cur_directory): 51 | os.makedirs(cur_directory) 52 | 53 | while num_imgs < 100: 54 | 55 | # randomly select latitude and longitude in the city 56 | brng = math.radians(random.uniform(0, 360)) # bearing is 90 degrees converted to radians. 57 | d = random.uniform(0, IMAGE_RADIUS) 58 | lat_rad = math.radians(lat) # current lat point converted to radians 59 | lon_rad = math.radians(lon) # current long point converted to radians 60 | rand_lat = math.asin(math.sin(lat_rad)*math.cos(d/R) + 61 | math.cos(lat_rad)*math.sin(d/R)*math.cos(brng)) 62 | rand_lon = lon_rad + math.atan2(math.sin(brng)*math.sin(d/R)*math.cos(lat_rad), 63 | math.cos(d/R)-math.sin(lat_rad)*math.sin(rand_lat)) 64 | rand_lat = math.degrees(rand_lat) 65 | rand_lon = math.degrees(rand_lon) 66 | 67 | # download image 68 | filename = 'lat-{}-lon-{}.jpg'.format(round(rand_lat, 4), round(rand_lon, 4)) 69 | filepath = os.path.join(cur_directory, filename) 70 | url = GOOGLE_URL + "&location=" + str(rand_lat) + ","+ str(rand_lon) 71 | urllib.urlretrieve(url, filepath) 72 | 73 | # check if the downloaded image was invalid and if so remove it 74 | if os.path.isfile(filepath): 75 | size = os.path.getsize(filepath) 76 | if size == FAILED_DOWNLOAD_IMAGE_SIZE: 77 | os.remove(filepath) 78 | misses += 1 79 | else: 80 | num_imgs += 1 81 | 82 | print 'invalid photo of {} downloaded {} times'.format(city, misses) 83 | file_utils.upload_directory_to_aws(cur_directory) 84 | 85 | def download_images(): 86 | 87 | # download images for each city in a different thread 88 | num_threads = 4 89 | pool = ThreadPool(num_threads) 90 | for city, (lat, lon) in cities.iteritems(): 91 | pool.apply_async(download_images_for_city, (city, lat, lon)) 92 | 93 | pool.close() 94 | pool.join() 95 | 96 | if __name__ == '__main__': 97 | download_images() 98 | -------------------------------------------------------------------------------- /scripts/human_baseline.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | import cv2 4 | import numpy as np 5 | import os 6 | import random 7 | import shutil 8 | import urllib 9 | 10 | import data_utils 11 | 12 | LABELS_URL = 'http://www.cs.ucf.edu/~aroshan/index_files/Dataset_PitOrlManh/GPS_Long_Lat_Compass.mat' 13 | URL = 'http://www.cs.ucf.edu/~aroshan/index_files/Dataset_PitOrlManh/images/{}_{}.jpg' 14 | DATASET_DIR = 'images' 15 | MIN_ID = 1 16 | MAX_ID = 10000 17 | PERSPECTIVES = [0,1,2,3,4,5] 18 | 19 | def get_images(num_images, dataset_dir): 20 | """ save num_images random images to dataset_dir """ 21 | # first download labels 22 | request_url = LABELS_URL 23 | outfile = os.path.join(DATASET_DIR, data_utils.LABLES_FILENAME) 24 | 25 | # then download images 26 | for idx in range(num_images): 27 | img_id = str(random.randint(MIN_ID, MAX_ID)).zfill(6) 28 | perspective = str(random.choice(PERSPECTIVES)) 29 | request_url = URL.format(img_id, perspective) 30 | print 'downloading image at: {}'.format(request_url) 31 | outfile = os.path.join(DATASET_DIR, '{}_{}.jpg'.format(img_id, perspective)) 32 | urllib.urlretrieve(request_url, outfile) 33 | 34 | def get_human_baseline(dataset_dir): 35 | # collect the filepaths in the directory 36 | filepaths, filenames = data_utils.get_image_filepaths(dataset_dir) 37 | filepaths = np.random.permutation(filepaths) 38 | 39 | # load labels 40 | labels = data_utils.load_labels_as_dict() 41 | 42 | # init counts 43 | incorrect = 0 44 | correct = 0 45 | 46 | # iterate through images 47 | count = 0 48 | for filepath, filename in zip(filepaths, filenames): 49 | count += 1 50 | 51 | # get label 52 | img_id = data_utils.image_filename_to_id(filename) 53 | if img_id in labels: 54 | label = labels[img_id] 55 | else: 56 | print 'cant find label' 57 | continue 58 | 59 | # load image 60 | img = cv2.imread(filepath) 61 | img = cv2.resize(img, (227,227)) 62 | 63 | # show image 64 | cv2.imshow('where was this picture taken? 1=ORL, 2=NYC, 3=PITT', img) 65 | 66 | # request a label until a valid one is given 67 | pred_label = None 68 | while pred_label not in [1,2,3]: 69 | 70 | try: 71 | pred_label = raw_input() 72 | if pred_label == 'q': 73 | return 74 | pred_label = int(pred_label) 75 | except: 76 | print 'please enter 1 for Orlando, 2 for NYC, 3 for Pittsburg' 77 | 78 | if label != pred_label: 79 | incorrect += 1 80 | else: 81 | correct += 1 82 | 83 | # close image 84 | cv2.destroyWindow('where was this picture taken? 1=ORL, 2=NYC, 3=PITT') 85 | 86 | if count % 10 == 0: 87 | print 'current accuracy: {}'.format(correct / float(correct + incorrect)) 88 | print 'current ratio: {} correct, {} incorrect'.format(correct, incorrect) 89 | 90 | if __name__ == '__main__': 91 | num_images = 200 92 | dataset_dir = DATASET_DIR 93 | if not os.path.isdir(dataset_dir): 94 | os.mkdir(dataset_dir) 95 | get_images(num_images, dataset_dir) 96 | get_human_baseline(dataset_dir) 97 | shutil.rmtree(dataset_dir) 98 | -------------------------------------------------------------------------------- /scripts/linear_classification.py: -------------------------------------------------------------------------------- 1 | """ 2 | :description: Classifying locations using linear classifiers and extracted features. 3 | """ 4 | import argparse 5 | from sklearn import linear_model 6 | 7 | import data_utils 8 | 9 | def run_classifier(feature_set, loss): 10 | """ 11 | :description: Fit a classifier and check its validation accuracy. 12 | 13 | :type feature_set: string 14 | :param feature_set: which feature set to use, options are {'all', 'hist', 'gist'} 15 | 16 | :type loss: string 17 | :param loss: which loss function to use 18 | """ 19 | train, y_train, val, y_val, test, y_test = data_utils.load_features_and_labels(feature_set) 20 | classifier = linear_model.SGDClassifier(loss, n_iter=50, alpha=1e-4, verbose=1) 21 | classifier.fit(train, y_train) 22 | acc = classifier.score(val, y_val) 23 | print 'validation accuracy: {}'.format(acc) 24 | acc = classifier.score(test, y_test) 25 | print 'validation accuracy: {}'.format(acc) 26 | 27 | def parse_args(): 28 | """ 29 | :description: Parse command line arguments. 30 | """ 31 | parser = argparse.ArgumentParser() 32 | parser.add_argument('-f', dest='feature_set', default='all', 33 | help="""{'all', 'hist', 'gist'}""") 34 | parser.add_argument('-l', dest='loss', default='hinge', 35 | help="""{'hinge', 'log', 'modified_huber', 'perceptron'}""") 36 | args = parser.parse_args() 37 | return args.feature_set, args.loss 38 | 39 | if __name__ == '__main__': 40 | feature_set, loss = parse_args() 41 | run_classifier(feature_set, loss) 42 | -------------------------------------------------------------------------------- /tests/run_tests.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/davidhershey/LittlePlaNet/7b39a376d7ff1020777577296d859d73f6c7a474/tests/run_tests.py --------------------------------------------------------------------------------