├── LICENSE ├── README.md ├── anno1602.py └── screenshot.png /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Lucas Beyer 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # laser-detection-annotator 2 | A tool to annotate/label detections in a stream of laser data. Can show supporting video if available. 3 | 4 | [![Screenshot of the annotator](screenshot.png)](screenshot.png) 5 | 6 | **TODO**: Documentation both of usage and of customization! 7 | 8 | Usage: 9 | 10 | ``` 11 | python anno1602.py [--dry-run] subpath-basename 12 | ``` 13 | 14 | Have a look at the code, all configuration parameters are at the beginning and commented. 15 | -------------------------------------------------------------------------------- /anno1602.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | from getpass import getuser 5 | 6 | # Settings of the annotation process 7 | ############ 8 | 9 | # How many frames does a batch scan. 10 | # This excludes the very last frame, since we start at 0. 11 | batchsize = 100 12 | # Show every tickskip-th frame for annotation. 13 | tickskip = 5 14 | # Skip so many batches after each. Example: 3 means batch, skip, skip, skip, batch, ... 15 | batchskip = 3 16 | 17 | # Everything further away than this is dropped. 18 | # This is because many lasers put NaN to some number, e.g. 29.999 19 | laser_cutoff = 14 20 | 21 | # Where to load the laser csv data and images from. 22 | # TODO: Describe better! 23 | basedir = "/work/" + getuser() + "/strands/wheelchair/dumped/" 24 | # Where to save the detections to. 25 | # TODO: Describe better! 26 | savedir = "/media/" + getuser() + "/NSA1/strands/wheelchair/" 27 | 28 | # The field-of-view of the laser you're using. 29 | # From https://github.com/lucasb-eyer/strands_karl/blob/5a2dd60/launch/karl_robot.launch#L25 30 | # TODO: change to min/max for supporting non-zero-centered lasers. 31 | laserFoV = 225 32 | 33 | # The field-of-view of the supportive camera you're using. 34 | # From https://www.asus.com/3D-Sensor/Xtion_PRO_LIVE/specifications/ 35 | # TODO: change to min/max for supporting non-zero-centered cameras. 36 | cameraFoV = 58 37 | 38 | # The size of the camera is needed for pre-generating the image-axes in the plot for efficiency. 39 | camsize = (480, 640) 40 | 41 | # Radius of the circle around the cursor, in data-units. 42 | # From https://thesegamesiplay.files.wordpress.com/2015/03/wheelchair.jpg 43 | circrad = 1.22/2 44 | 45 | # TODO: make the types of labels configurable? Although, does it even matter? 46 | 47 | # End of settings 48 | ############ 49 | 50 | import sys 51 | import os 52 | import json 53 | import time 54 | 55 | import numpy as np 56 | import matplotlib as mpl 57 | import matplotlib.pyplot as plt 58 | from matplotlib.widgets import Cursor, AxesWidget 59 | 60 | try: 61 | import cv2 62 | def imread(fname): 63 | img = cv2.imread(fname) 64 | if img is not None: 65 | img = img[:,:,::-1] # BGR to RGB 66 | return img 67 | except ImportError: 68 | # Python 2/3 compatibility 69 | try: 70 | FileNotFoundError 71 | except NameError: 72 | FileNotFoundError = IOError 73 | 74 | from matplotlib.image import imread as mpl_imread 75 | def imread(fname): 76 | try: 77 | return mpl_imread(fname) 78 | except FileNotFoundError: 79 | return None 80 | 81 | laserFoV = np.radians(laserFoV) 82 | cameraFoV = np.radians(cameraFoV) 83 | 84 | if len(sys.argv) < 2: 85 | print("Usage: {} relative/path/to_file.bag".format(sys.argv[0])) 86 | print() 87 | print("relative to {}".format(basedir)) 88 | print("To perform a dry run without saving results use --dry-run or -n.") 89 | print("To change into person annotation mode use --person or -p.") 90 | sys.exit(1) 91 | 92 | # Very crude, not robust argv handling. 93 | name = None 94 | dryrun = False 95 | person_mode = False 96 | 97 | for arg in sys.argv[1:]: 98 | if arg in ("--dry-run", "-n"): 99 | dryrun = True 100 | elif arg in ("--person", "-p"): 101 | person_mode = True 102 | else: 103 | if name is None: 104 | name = arg 105 | else: 106 | print("Cannot parse arguments, please only specify a single path to the data, and/or flags for dry run or person only annotations.") 107 | sys.exit(1) 108 | 109 | 110 | 111 | def mkdirs(fname): 112 | """ Make directories necessary for creating `fname`. """ 113 | dname = os.path.dirname(fname) 114 | if not os.path.isdir(dname): 115 | os.makedirs(dname) 116 | 117 | 118 | # **TODO**: put into common toolbox repo. 119 | def xy_to_rphi(x, y): 120 | # Note: axes rotated by 90 by intent, so that 0 is top. 121 | return np.hypot(x, y), np.arctan2(-x, y) 122 | 123 | 124 | def rphi_to_xy(r, phi): 125 | return r * -np.sin(phi), r * np.cos(phi) 126 | 127 | 128 | def scan_to_xy(scan, thresh=None): 129 | s = np.array(scan, copy=True) 130 | if thresh is not None: 131 | s[s > thresh] = np.nan 132 | angles = np.linspace(-laserFoV/2, laserFoV/2, len(scan)) 133 | return rphi_to_xy(scan, angles) 134 | 135 | 136 | def imload(name, *seqs): 137 | for s in seqs: 138 | fname = "{}{}_dir/{}.jpg".format(basedir, name, int(s)) 139 | im = imread(fname) 140 | if im is not None: 141 | return im 142 | print("WARNING: Couldn't find any of " + ' ; '.join(map(str, map(int, seqs)))) 143 | return np.zeros(camsize + (3,), dtype=np.uint8) 144 | 145 | 146 | class Anno1602: 147 | def __init__(self, batches, scans, seqs, laser_thresh=laser_cutoff, circrad=circrad, xlim=None, ylim=None, dryrun=False, person_mode=False): 148 | self.batches = batches 149 | self.scans = scans 150 | self.seqs = seqs 151 | self.b = 0 152 | self.i = 0 153 | self.laser_thresh = laser_thresh 154 | self.circrad = circrad 155 | self.xlim = xlim 156 | self.ylim = ylim 157 | self.dryrun = dryrun 158 | self.person_mode = person_mode 159 | 160 | # Build the figure and the axes. 161 | self.fig = plt.figure(figsize=(10,10)) 162 | gs = mpl.gridspec.GridSpec(3, 2, width_ratios=(3, 1)) 163 | 164 | self.axlaser = plt.subplot(gs[:,0]) 165 | axprev, axpres, axnext = plt.subplot(gs[0,1]), plt.subplot(gs[1,1]), plt.subplot(gs[2,1]) 166 | 167 | self.imprev = axprev.imshow(np.random.randint(255, size=camsize + (3,)), interpolation='nearest', animated=True) 168 | self.impres = axpres.imshow(np.random.randint(255, size=camsize + (3,)), interpolation='nearest', animated=True) 169 | self.imnext = axnext.imshow(np.random.randint(255, size=camsize + (3,)), interpolation='nearest', animated=True) 170 | axprev.axis('off') 171 | axpres.axis('off') 172 | axnext.axis('off') 173 | 174 | gs.tight_layout(self.fig) # Make better use of the space we have. 175 | 176 | self.circ = MouseCircle(self.axlaser, radius=self.circrad, linewidth=1, fill=False) 177 | 178 | # Configure interaction 179 | self.fig.canvas.mpl_connect('button_press_event', self.click) 180 | self.fig.canvas.mpl_connect('scroll_event', self.scroll) 181 | self.fig.canvas.mpl_connect('key_press_event', self.key) 182 | 183 | # Labels!! 184 | self.wheelchairs = [[None for i in b] for b in batches] 185 | self.walkingaids = [[None for i in b] for b in batches] 186 | self.persons = [[None for i in b] for b in batches] 187 | self.load() 188 | 189 | # The pause is needed for the OSX backend. 190 | plt.pause(0.0001) 191 | self.replot() 192 | 193 | def save(self): 194 | if self.dryrun: 195 | return 196 | 197 | mkdirs(savedir + name) 198 | 199 | def _doit(f, data): 200 | for ib, batch in enumerate(data): 201 | if batch is not None: 202 | for i, seq in enumerate(batch): 203 | if seq is not None: 204 | f.write('{},['.format(self.seqs[self.batches[ib][i]])) 205 | f.write(','.join('[{},{}]'.format(*xy_to_rphi(x,y)) for x,y in seq)) 206 | f.write(']\n') 207 | 208 | if self.person_mode: 209 | with open(savedir + name + ".wp", "w+") as f: 210 | _doit(f, self.persons) 211 | 212 | else: 213 | with open(savedir + name + ".wc", "w+") as f: 214 | _doit(f, self.wheelchairs) 215 | 216 | with open(savedir + name + ".wa", "w+") as f: 217 | _doit(f, self.walkingaids) 218 | 219 | def load(self): 220 | def _doit(f, whereto): 221 | # loading a file can be done here and should "just" be reading it 222 | # line-by-line, de-json-ing the second half of `,` and recording it in 223 | # a dict with the sequence number which is the first half of `,`. 224 | data = {} 225 | for line in f: 226 | seq, tail = line.split(',', 1) 227 | data[int(seq)] = [rphi_to_xy(r, phi) for r,phi in json.loads(tail)] 228 | 229 | # Then, in second pass, go through b/i and check for `seqs[batches[b][i]]` 230 | # in that dict, and use that. 231 | for ib, batch in enumerate(whereto): 232 | for i, _ in enumerate(batch): 233 | batch[i] = data.get(self.seqs[self.batches[ib][i]], None) 234 | 235 | try: 236 | with open(savedir + name + ".wc", "r") as f: 237 | _doit(f, self.wheelchairs) 238 | 239 | with open(savedir + name + ".wa", "r") as f: 240 | _doit(f, self.walkingaids) 241 | 242 | with open(savedir + name + ".wp", "r") as f: 243 | _doit(f, self.persons) 244 | except FileNotFoundError: 245 | pass # That's ok, just means no annotations yet. 246 | 247 | def replot(self, newbatch=True): 248 | batch = self.batches[self.b] 249 | 250 | # This doesn't really belong here, but meh, it needs to happen as soon as a new batch is opened, so here. 251 | # We want to save a batch, even empty, as soon as it has been seen. 252 | if newbatch: 253 | for i, _ in enumerate(self.batches[self.b]): 254 | if self.wheelchairs[self.b][i] is None: 255 | self.wheelchairs[self.b][i] = [] 256 | if self.walkingaids[self.b][i] is None: 257 | self.walkingaids[self.b][i] = [] 258 | if self.persons[self.b][i] is None: 259 | self.persons[self.b][i] = [] 260 | 261 | self.axlaser.clear() 262 | self.axlaser.scatter(*scan_to_xy(self.scans[batch[self.i]], self.laser_thresh), s=10, color='#E24A33', alpha=0.5, lw=0) 263 | 264 | # Camera frustum to help orientation. 265 | self.axlaser.plot([0, -self.laser_thresh*np.sin(cameraFoV/2)], [0, self.laser_thresh*np.cos(cameraFoV/2)], 'k:') 266 | self.axlaser.plot([0, self.laser_thresh*np.sin(cameraFoV/2)], [0, self.laser_thresh*np.cos(cameraFoV/2)], 'k:') 267 | 268 | # Jitter size a little so we can easily see multiple click mistakes. 269 | for x,y in self.wheelchairs[self.b][self.i] or []: 270 | self.axlaser.scatter(x, y, marker='+', s=np.random.uniform(40,60), color='#348ABD') 271 | for x,y in self.walkingaids[self.b][self.i] or []: 272 | self.axlaser.scatter(x, y, marker='x', s=np.random.uniform(40,60), color='#988ED5') 273 | for x,y in self.persons[self.b][self.i] or []: 274 | self.axlaser.scatter(x, y, marker='o', s=np.random.uniform(15,100), color='#50B948', facecolors='none') 275 | 276 | # Fix aspect ratio and visible region. 277 | if self.xlim is not None: 278 | self.axlaser.set_xlim(*self.xlim) 279 | if self.ylim is not None: 280 | self.axlaser.set_ylim(*self.ylim) 281 | self.axlaser.set_aspect('equal', adjustable='box') # Force axes to have equal scale. 282 | 283 | b = self.seqs[batch[self.i]] 284 | self.impres.set_data(imload(name, b, b-1, b+1, b-2, b+2)) 285 | 286 | if newbatch: 287 | a = self.seqs[batch[0]] 288 | self.imprev.set_data(imload(name, a, a+1, a+2, a+tickskip, a+batchsize//10, a+batchsize//5, a+batchsize//4)) 289 | c = self.seqs[batch[-1]] 290 | self.imnext.set_data(imload(name, c, c-1, c-2, c-tickskip, c-batchsize//10, c-batchsize//5, c-batchsize//4)) 291 | 292 | self.fig.suptitle("{}: Batch {}/{} frame {}, seq {}".format(name, self.b+1, len(self.batches), self.i*tickskip, self.seqs[batch[self.i]])) 293 | self.fig.canvas.draw() 294 | self.circ._update() 295 | 296 | def click(self, e): 297 | if self._ignore(e): 298 | return 299 | 300 | # In some rare cases, there is no xdata or ydata (not sure when exactly) 301 | # so we skip them instea of adding them to the list and causing bugs later on! 302 | if e.xdata is None or e.ydata is None: 303 | return 304 | 305 | if self.person_mode: 306 | if e.button == 1: 307 | self.persons[self.b][self.i].append((e.xdata, e.ydata)) 308 | elif e.button == 2: 309 | self._clear(e.xdata, e.ydata) 310 | else: 311 | if e.button == 1: 312 | self.wheelchairs[self.b][self.i].append((e.xdata, e.ydata)) 313 | elif e.button == 3: 314 | self.walkingaids[self.b][self.i].append((e.xdata, e.ydata)) 315 | elif e.button == 2: 316 | self._clear(e.xdata, e.ydata) 317 | 318 | self.replot(newbatch=False) 319 | 320 | def scroll(self, e): 321 | if self._ignore(e): 322 | return 323 | 324 | if e.button == 'down': 325 | for _ in range(int(-e.step)): 326 | self._nexti() 327 | elif e.button == 'up': 328 | for _ in range(int(e.step)): 329 | self._previ() 330 | self.replot(newbatch=False) 331 | 332 | def key(self, e): 333 | if self._ignore(e): 334 | return 335 | 336 | newbatch = False 337 | if e.key in ("left", "a"): 338 | self._nexti() 339 | elif e.key in ("right", "d"): 340 | self._previ() 341 | elif e.key in ("down", "s", "pagedown"): 342 | self._prevb() 343 | newbatch = True 344 | elif e.key in ("up", "w", "pageup"): 345 | self._nextb() 346 | newbatch = True 347 | elif e.key == "c": 348 | self._clear(e.xdata, e.ydata) 349 | else: 350 | print(e.key) 351 | 352 | self.replot(newbatch=newbatch) 353 | 354 | def _nexti(self): 355 | self.i = max(0, self.i-1) 356 | 357 | def _previ(self): 358 | self.i = min(len(self.batches[self.b])-1, self.i+1) 359 | 360 | def _nextb(self): 361 | self.save() 362 | self.b += 1 363 | self.i = 0 364 | 365 | # We decided to close after the last batch. 366 | if self.b == len(self.batches): 367 | self.b -= 1 # Because it gets redrawn once before closed... 368 | plt.close(self.fig) 369 | 370 | def _prevb(self): 371 | self.save() 372 | if self.b == 0: 373 | self.b = len(self.batches)-1 374 | else: 375 | self.b = self.b-1 376 | self.i = 0 377 | 378 | def _clear(self, mx, my): 379 | try: 380 | if self.person_mode: 381 | self.persons[self.b][self.i] = [(x,y) for x,y in self.persons[self.b][self.i] if np.hypot(mx-x, my-y) > self.circrad] 382 | else: 383 | self.wheelchairs[self.b][self.i] = [(x,y) for x,y in self.wheelchairs[self.b][self.i] if np.hypot(mx-x, my-y) > self.circrad] 384 | self.walkingaids[self.b][self.i] = [(x,y) for x,y in self.walkingaids[self.b][self.i] if np.hypot(mx-x, my-y) > self.circrad] 385 | except TypeError: 386 | import pdb ; pdb.set_trace() # THERE IS A RARE BUG HERE. CALL LUCAS 387 | 388 | def _ignore(self, e): 389 | # https://scipy.github.io/old-wiki/pages/Cookbook/Matplotlib/Interactive_Plotting.html#Handling_click_events_while_zoomed 390 | # But we don't do `not e.inaxes` because that one is only set when the mouse moves, meaning 391 | # when we switch the frame, it will not be `inaxes` as long as we don't move the mouse! 392 | return plt.get_current_fig_manager().toolbar.mode != '' 393 | 394 | 395 | # Annoyingly large amount of code for just a circle around the mouse cursor! 396 | class MouseCircle(AxesWidget): 397 | def __init__(self, ax, radius, **circprops): 398 | AxesWidget.__init__(self, ax) 399 | 400 | self.connect_event('motion_notify_event', self.onmove) 401 | self.connect_event('draw_event', self.storebg) 402 | 403 | circprops['animated'] = True 404 | circprops['radius'] = radius 405 | 406 | self.circ = plt.Circle((0,0), **circprops) 407 | self.ax.add_artist(self.circ) 408 | 409 | self.background = None 410 | 411 | def storebg(self, e): 412 | if not self.ignore(e): 413 | self.background = self.canvas.copy_from_bbox(self.ax.bbox) 414 | 415 | def onmove(self, e): 416 | if not self.ignore(e) and self.canvas.widgetlock.available(self): 417 | self.circ.center = (e.xdata, e.ydata) # (None,None) -> invisible if out of axis. 418 | self._update() 419 | 420 | def _update(self): 421 | if self.background is not None: 422 | self.canvas.restore_region(self.background) 423 | self.ax.draw_artist(self.circ) 424 | self.canvas.blit(bbox=None) # Note: not passing `self.ax.bbox` as else the other axes stay old... 425 | 426 | 427 | if __name__ == "__main__": 428 | # Loading all data first 429 | print("Loading data...") ; sys.stdout.flush() 430 | data = np.genfromtxt(basedir + name + ".csv", delimiter=",") 431 | seqs, scans = data[:,0].astype(np.uint32), data[:,1:-1] # Last column is always empty for ease of dumping. 432 | print("Loaded {} scans".format(len(seqs))) ; sys.stdout.flush() 433 | 434 | # Chunking into minibatches. 435 | print("Chunking into batches...") ; sys.stdout.flush() 436 | batches = [] 437 | for bstart in np.arange(tickskip, len(seqs)-tickskip, batchsize*(batchskip+1)): 438 | batches.append(np.arange(bstart, min(bstart+batchsize-1, len(seqs)), tickskip)) 439 | print("Created {} batches".format(len(batches))) ; sys.stdout.flush() 440 | 441 | # Determine the view-space. 442 | xr, yr = scan_to_xy(np.full(scans.shape[1], laser_cutoff, dtype=np.float32)) 443 | 444 | print("Starting annotator...") ; sys.stdout.flush() 445 | print("Person mode: {}, Dry run: {}".format(person_mode, dryrun)) ; sys.stdout.flush() 446 | anno = Anno1602(batches, scans, seqs, laser_cutoff, xlim=(min(xr), max(xr)), ylim=(min(yr), max(yr)), dryrun=dryrun, person_mode=person_mode) 447 | 448 | t0 = time.time() 449 | anno.replot() 450 | plt.show() 451 | t1 = time.time() 452 | 453 | print("You annotated {} batches in {:.0f}s, i.e. took {:.1f}s per batch.".format(len(batches), t1-t0, (t1-t0)/len(batches))) 454 | -------------------------------------------------------------------------------- /screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lucasb-eyer/laser-detection-annotator/97c3b2bbd5c26513cce5ef716b1be42259e91efb/screenshot.png --------------------------------------------------------------------------------