├── Model Tutorial - PDF.pdf ├── CONTRIBUTING ├── CODE_OF_CONDUCT.md ├── README.md ├── LICENSE └── pathfinder.py /Model Tutorial - PDF.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/many-to-many-dijkstra/HEAD/Model Tutorial - PDF.pdf -------------------------------------------------------------------------------- /CONTRIBUTING: -------------------------------------------------------------------------------- 1 | The code provided here is to provide guidance for other teams seeking to replicate and extend our work. It's documentary in nature, so we don't anticipate modifying it or accepting contributed code changes. 2 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | 3 | Facebook has adopted a Code of Conduct that we expect project participants to adhere to. 4 | Please read the [full text](https://code.fb.com/codeofconduct/) 5 | so that you can understand what actions will and will not be tolerated. 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Pathfinder 2 | 3 | A many-to-many variant of the Dijkstra's shortest distance algorithm. 4 | 5 | Originally developed to predict the locations of medium-voltage electrical distribution grid infrastructure using publicly available data sources. 6 | 7 | How pathfinder works is described in the Model Tutorial in this repo. 8 | Notes on implementation and operation are embedded as comments within the code. 9 | 10 | ## License 11 | Pathfinder is MIT licensed, as found in the LICENSE file. 12 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Facebook, Inc. and its affiliates. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /pathfinder.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (c) Facebook, Inc. and its affiliates. 3 | 4 | This source code is licensed under the MIT license found in the 5 | LICENSE file in the root directory of this source tree. 6 | """ 7 | 8 | """ 9 | The pathfinder algorithm. 10 | """ 11 | from __future__ import absolute_import 12 | from __future__ import division 13 | from __future__ import print_function 14 | from __future__ import unicode_literals 15 | from __future__ import with_statement 16 | import heapq 17 | import os 18 | import sys 19 | import time 20 | 21 | import matplotlib 22 | matplotlib.use('Agg') 23 | import matplotlib.pyplot as plt 24 | from numba import autojit 25 | import numpy as np 26 | 27 | 28 | def seek( 29 | origins, 30 | targets=None, 31 | weights=None, 32 | path_handling='link', 33 | debug=False, 34 | film=False, 35 | ): 36 | """ 37 | Find the shortest paths between *any* origin and *each* target. 38 | 39 | Pathfinder is a modified version of Dijkstra's algorithm 40 | (https://en.wikipedia.org/wiki/Dijkstra's_algorithm) for finding 41 | the shortest distance between two points in a graph. It differs in 42 | a few important ways: 43 | 44 | * It finds the shortest distance between a target point and the 45 | nearest of a set of origin points. This is then repeated for 46 | each target point. 47 | * It assumes a gridded topology. In other words, it knows that 48 | each node only 49 | touches its neighbors to the north, south, east, west, 50 | northeast, northwest, southeast, and southwest. 51 | 52 | Like Dijkstra's, Pathfinder assumes that all weights are 0 or greater. 53 | Negative weights are set to zero. All input arrays (origins, 54 | targets, and weights) need to have the same number of rows and columns. 55 | 56 | @param origins, targets: 2D numpy array of ints 57 | Any non-zero values are the locations of origin and target points, 58 | respectively. Note that both may be modified. Target points may 59 | be removed, once paths to them are found and origins may be 60 | augmented (see path_handling param below). 61 | If targets is not supplied, no targets are assumed and a targets 62 | array of all zeros is created. This is useful for calculating minimum 63 | distances from a set of origins to all points of the grid. 64 | @param weights: 2D numpy array of floats 65 | The cost of visiting a grid square, zero or greater. 66 | For favorable (easy to traverse) grid locations, this is low. 67 | For unfavorable grid locations, this is high. 68 | If not supplied, a weights array of all ones is used. This is 69 | useful for calculating as-the-crow-flies distance. 70 | @param path_handling: string 71 | One of {'link', 'assimilate', 'none', 'l', 'a', 'n'}. 72 | Determines how to handle paths between target and origins, 73 | once they are found. 74 | 75 | * 'link' or 'l' adds a target to the origins once it is found, 76 | as well as the path connecting them. This mode 77 | is good for growing a network by connecting nodes, as we do here 78 | when planning or estimating an electrical grid. 79 | * 'assimilate' or 'a' adds a target to the origins once it is found, 80 | but does not add the path connecting them. This mode 81 | is good for growing a network by adding nodes that 82 | have no physical connection between them, as in planning an 83 | ad-hoc wireless network. 84 | * 'none' or 'n' doesn't add a target to the to the origins 85 | once it is found. This mode is good for finding a path 86 | from a backbone or trunk to many leaf nodes, 87 | as in planning fiber backhaul routing. 88 | 89 | @param debug: boolean 90 | If True, provide text updates on the algorithm's progress. 91 | @param film: boolean 92 | If True, periodically save snapshots of the algorithm's progress. 93 | 94 | @retun results: dict 95 | 'paths': 2D numpy array of ints 96 | 1 where paths have been found, and 0 everywhere else. 97 | 'distance: 2D numpy array of floats 98 | The length of the shortest path (the sum of the weights of grid 99 | cells traversed) from the nearest origin point to every point 100 | on the grid. Origin points have a distance of zero, and it 101 | goes up from there the further away you get. 102 | 'rendering': 2D numpy array of floats 103 | An image representing the final state of the algorithm, including 104 | paths found and distances calculated. 105 | """ 106 | if weights is None: 107 | weights = np.ones(origins.shape) 108 | if targets is None: 109 | targets = np.zeros(origins.shape, dtype=np.int8) 110 | assert targets.shape == origins.shape 111 | assert targets.shape == weights.shape 112 | path_handling = path_handling.lower() 113 | assert path_handling in ['none', 'n', 'assimilate', 'a', 'link', 'l'] 114 | n_rows, n_cols = origins.shape 115 | if path_handling[0] == 'n': 116 | path_handling = 0 117 | if path_handling[0] == 'a': 118 | path_handling = 1 119 | if path_handling[0] == 'l': 120 | path_handling = 2 121 | 122 | iteration = 0 123 | not_visited = 9999999999. 124 | 125 | if film: 126 | frame_rate = int(1e4) 127 | frame_counter = 100000 128 | frame_dirname = 'frames' 129 | try: 130 | os.mkdir(frame_dirname) 131 | except Exception: 132 | # NBD 133 | pass 134 | 135 | cwd = os.getcwd() 136 | try: 137 | os.chdir(frame_dirname) 138 | for filename in os.listdir('.'): 139 | os.remove(filename) 140 | except Exception: 141 | print('Frame deletion failed') 142 | finally: 143 | os.chdir(cwd) 144 | 145 | rendering = 1. / (2. * weights) 146 | rendering = np.minimum(rendering, 1.) 147 | target_locations = np.where(targets) 148 | n_targets = target_locations[0].size 149 | n_targets_remaining = n_targets 150 | n_targets_remaining_update = n_targets 151 | for i_target, row in enumerate(target_locations[0]): 152 | col = target_locations[1][i_target] 153 | wid = 8 154 | rendering[ 155 | row - wid: 156 | row + wid + 1, 157 | col - wid: 158 | col + wid + 1] = .5 159 | 160 | # The distance array shows the shortest weighted distance from 161 | # each point in the grid to the nearest origin point. 162 | distance = np.ones((n_rows, n_cols)) * not_visited 163 | origin_locations = np.where(origins != 0) 164 | distance[origin_locations] = 0. 165 | 166 | # The paths array shows each of the paths that are discovered 167 | # from targets to their nearest origin point. 168 | paths = np.zeros((n_rows, n_cols), dtype=np.int8) 169 | 170 | # The halo is the set of points under evaluation. They surround 171 | # the origin points and expand outward, forming a growing halo 172 | # around the set of origins that eventually enevlops targets. 173 | # It is implemented using a heap queue, so that the halo point 174 | # nearest to an origin is always the next one that gets evaluated. 175 | halo = [] 176 | for i, origin_row in enumerate(origin_locations[0]): 177 | origin_col = origin_locations[1][i] 178 | heapq.heappush(halo, (0., (origin_row, origin_col))) 179 | 180 | # The temporary array for tracking locations to add to the halo. 181 | # This gets overwritten with each iteration. 182 | new_locs = np.zeros((int(1e6), 3)) 183 | n_new_locs = 0 184 | 185 | while len(halo) > 0: 186 | iteration += 1 187 | if debug: 188 | if (n_targets_remaining > n_targets_remaining_update or 189 | iteration % 1e4 == 0.): 190 | n_targets_remaining = n_targets_remaining_update 191 | print('\r {num} targets of {total} reached, {rem} remaining, {halo_len} to try ' 192 | .format( 193 | num=n_targets - n_targets_remaining, 194 | total=n_targets, 195 | rem=n_targets_remaining, 196 | halo_len=len(halo), 197 | ), end='') 198 | sys.stdout.flush() 199 | if film: 200 | if iteration % frame_rate == 0: 201 | frame_counter = render( 202 | distance, 203 | frame_counter, 204 | frame_dirname, 205 | not_visited, 206 | rendering, 207 | ) 208 | 209 | # Reinitialize locations to add. 210 | new_locs[:n_new_locs, :] = 0. 211 | n_new_locs = 0 212 | 213 | # Retrieve and check the location with shortest distance. 214 | (distance_here, (row_here, col_here)) = heapq.heappop(halo) 215 | n_new_locs, n_targets_remaining_update = nb_loop( 216 | col_here, 217 | distance, 218 | distance_here, 219 | n_cols, 220 | n_new_locs, 221 | n_rows, 222 | n_targets_remaining, 223 | new_locs, 224 | not_visited, 225 | origins, 226 | path_handling, 227 | paths, 228 | row_here, 229 | targets, 230 | weights, 231 | ) 232 | for i_loc in range(n_new_locs): 233 | loc = (int(new_locs[i_loc, 1]), int(new_locs[i_loc, 2])) 234 | heapq.heappush(halo, (new_locs[i_loc, 0], loc)) 235 | 236 | if debug: 237 | print('\r ', end='') 238 | sys.stdout.flush() 239 | print('') 240 | # Add the newfound paths to the visualization. 241 | rendering = 1. / (1. + distance / 10.) 242 | rendering[np.where(origins)] = 1. 243 | rendering[np.where(paths)] = .8 244 | results = {'paths': paths, 'distance': distance, 'rendering': rendering} 245 | return results 246 | 247 | 248 | def render( 249 | distance, 250 | frame_counter, 251 | frame_dirname, 252 | not_visited, 253 | rendering, 254 | ): 255 | """ 256 | Turn the progress of the algorithm into a pretty picture. 257 | """ 258 | progress = rendering.copy() 259 | visited_locs = np.where(distance < not_visited) 260 | progress[visited_locs] = 1. / (1. + distance[visited_locs] / 10.) 261 | filename = 'pathfinder_frame_' + str(frame_counter) + '.png' 262 | cmap = 'inferno' 263 | dpi = 1200 264 | plt.figure(33374) 265 | plt.clf() 266 | plt.imshow( 267 | progress, 268 | origin='higher', 269 | interpolation='nearest', 270 | cmap=plt.get_cmap(cmap), 271 | vmax=1., 272 | vmin=0., 273 | ) 274 | filename_full = os.path.join(frame_dirname, filename) 275 | plt.savefig(filename_full, dpi=dpi) 276 | frame_counter += 1 277 | return frame_counter 278 | 279 | 280 | @autojit(nopython=True) 281 | def nb_trace_back( 282 | distance, 283 | n_new_locs, 284 | new_locs, 285 | not_visited, 286 | origins, 287 | path_handling, 288 | paths, 289 | target, 290 | weights, 291 | ): 292 | """ 293 | Connect each found electrified target to the grid through 294 | the shortest available path. 295 | """ 296 | # Handle the case where you find more than one target. 297 | path = [] 298 | distance_remaining = distance[target] 299 | current_location = target 300 | while distance_remaining > 0.: 301 | path.append(current_location) 302 | (row_here, col_here) = current_location 303 | # Check each of the neighbors for the lowest distance to grid. 304 | neighbors = [ 305 | ((row_here - 1, col_here), 1.), 306 | ((row_here + 1, col_here), 1.), 307 | ((row_here, col_here + 1), 1.), 308 | ((row_here, col_here - 1), 1.), 309 | ((row_here - 1, col_here - 1), 2.**.5), 310 | ((row_here + 1, col_here - 1), 2.**.5), 311 | ((row_here - 1, col_here + 1), 2.**.5), 312 | ((row_here + 1, col_here + 1), 2.**.5), 313 | ] 314 | lowest_distance = not_visited 315 | # It's confusing, but keep in mind that 316 | # distance[neighbor] is the distance from the neighbor position 317 | # to the grid, while neighbor_distance is 318 | # the distance *through* 319 | # the neighbor position to the grid. It is distance[neighbor] 320 | # plus the distance to the neighbor from the current position. 321 | for (neighbor, scale) in neighbors: 322 | if neighbor not in path: 323 | distance_from_neighbor = scale * weights[current_location] 324 | neighbor_distance = (distance[neighbor] + 325 | distance_from_neighbor) 326 | if neighbor_distance < lowest_distance: 327 | lowest_distance = neighbor_distance 328 | best_neighbor = neighbor 329 | 330 | # This will fail if caught in a local minimum. 331 | if distance_remaining < distance[best_neighbor]: 332 | distance_remaining = 0. 333 | continue 334 | 335 | distance_remaining = distance[best_neighbor] 336 | current_location = best_neighbor 337 | 338 | # Add this new path. 339 | for i_loc, loc in enumerate(path): 340 | paths[loc] = 1 341 | # If paths are to be linked, include the entire paths as origins and 342 | # add them to new_locs. If targets are to be assimilated, just add 343 | # the target (the first point on the path) to origins and new_locs. 344 | if path_handling == 2 or ( 345 | path_handling == 1 and i_loc == 0): 346 | origins[loc] = 1 347 | distance[loc] = 0. 348 | new_locs[n_new_locs, 0] = 0. 349 | new_locs[n_new_locs, 1] = loc[0] 350 | new_locs[n_new_locs, 2] = loc[1] 351 | n_new_locs += 1 352 | 353 | return n_new_locs 354 | 355 | 356 | @autojit(nopython=True) 357 | def nb_loop( 358 | col_here, 359 | distance, 360 | distance_here, 361 | n_cols, 362 | n_new_locs, 363 | n_rows, 364 | n_targets_remaining, 365 | new_locs, 366 | not_visited, 367 | origins, 368 | path_handling, 369 | paths, 370 | row_here, 371 | targets, 372 | weights, 373 | ): 374 | """ 375 | This is the meat of the computation. 376 | Pull the computationally expensive operations from seek() 377 | out into their own function that can be pre-compiled using numba. 378 | """ 379 | # Calculate the distance for each of the 8 neighbors. 380 | neighbors = [ 381 | ((row_here - 1, col_here), 1.), 382 | ((row_here + 1, col_here), 1.), 383 | ((row_here, col_here + 1), 1.), 384 | ((row_here, col_here - 1), 1.), 385 | ((row_here - 1, col_here - 1), 2.**.5), 386 | ((row_here + 1, col_here - 1), 2.**.5), 387 | ((row_here - 1, col_here + 1), 2.**.5), 388 | ((row_here + 1, col_here + 1), 2.**.5), 389 | ] 390 | 391 | for (neighbor, scale) in neighbors: 392 | weight = scale * weights[neighbor] 393 | neighbor_distance = distance_here + weight 394 | 395 | if distance[neighbor] == not_visited: 396 | if targets[neighbor]: 397 | n_new_locs = nb_trace_back( 398 | distance, 399 | n_new_locs, 400 | new_locs, 401 | not_visited, 402 | origins, 403 | path_handling, 404 | paths, 405 | neighbor, 406 | weights, 407 | ) 408 | targets[neighbor] = 0 409 | n_targets_remaining -= 1 410 | if neighbor_distance < distance[neighbor]: 411 | distance[neighbor] = neighbor_distance 412 | if (neighbor[0] > 0 and 413 | neighbor[0] < n_rows - 1 and 414 | neighbor[1] > 0 and 415 | neighbor[1] < n_cols - 1): 416 | new_locs[n_new_locs, 0] = distance[neighbor] 417 | new_locs[n_new_locs, 1] = neighbor[0] 418 | new_locs[n_new_locs, 2] = neighbor[1] 419 | n_new_locs += 1 420 | return n_new_locs, n_targets_remaining 421 | --------------------------------------------------------------------------------