├── .gitattributes ├── .gitignore ├── LICENSE ├── README.md ├── build └── lib │ └── mapmatcher │ ├── __init__.py │ └── mapmatcher.py ├── dist └── mapmatcher-1.0.win32.exe ├── example.PNG ├── install.txt ├── mapMatch.PNG ├── mapMatch.mapMatch.pyt.xml ├── mapMatch.pyt ├── mapMatch.pyt.xml ├── mapmatcher ├── __init__.py ├── __init__.pyc ├── mapmatcher.py └── mapmatcher.pyc ├── setup.py ├── testSegments.cpg ├── testSegments.dbf ├── testSegments.prj ├── testSegments.sbn ├── testSegments.sbx ├── testSegments.shp ├── testSegments.shp.xml ├── testSegments.shx ├── testTrack.cpg ├── testTrack.dbf ├── testTrack.prj ├── testTrack.sbn ├── testTrack.sbx ├── testTrack.shp ├── testTrack.shx ├── testTrack_pth.cpg ├── testTrack_pth.dbf ├── testTrack_pth.prj ├── testTrack_pth.sbn ├── testTrack_pth.sbx ├── testTrack_pth.shp ├── testTrack_pth.shp.xml └── testTrack_pth.shx /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | 4 | # Custom for Visual Studio 5 | *.cs diff=csharp 6 | 7 | # Standard to msysgit 8 | *.doc diff=astextplain 9 | *.DOC diff=astextplain 10 | *.docx diff=astextplain 11 | *.DOCX diff=astextplain 12 | *.dot diff=astextplain 13 | *.DOT diff=astextplain 14 | *.pdf diff=astextplain 15 | *.PDF diff=astextplain 16 | *.rtf diff=astextplain 17 | *.RTF diff=astextplain 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Windows image file caches 2 | Thumbs.db 3 | ehthumbs.db 4 | 5 | # Folder config file 6 | Desktop.ini 7 | 8 | # Recycle Bin used on file shares 9 | $RECYCLE.BIN/ 10 | 11 | # Windows Installer files 12 | *.cab 13 | *.msi 14 | *.msm 15 | *.msp 16 | 17 | # Windows shortcuts 18 | *.lnk 19 | 20 | # ========================= 21 | # Operating System Files 22 | # ========================= 23 | 24 | # OSX 25 | # ========================= 26 | 27 | .DS_Store 28 | .AppleDouble 29 | .LSOverride 30 | 31 | # Thumbnails 32 | ._* 33 | 34 | # Files that might appear in the root of a volume 35 | .DocumentRevisions-V100 36 | .fseventsd 37 | .Spotlight-V100 38 | .TemporaryItems 39 | .Trashes 40 | .VolumeIcon.icns 41 | 42 | # Directories potentially created on remote AFP share 43 | .AppleDB 44 | .AppleDesktop 45 | Network Trash Folder 46 | Temporary Items 47 | .apdisk 48 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 simonscheider 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Map Matcher 2 | 3 | This python script allows map matching (matching of tracking points to a network) 4 | in arcpy using a Hidden Markov model with 5 | probabilities parameterized based on spatial + network distances. 6 | Follows the ideas in Newson, Krumm (2009): 7 | "Hidden markov Map Matching through noise and sparseness" 8 | 9 | Author: Simon Scheider 10 | 11 | Created: 16/03/2017 12 | 13 | 14 | ## Installation 15 | 16 | To install as a toolbox in ArcGIS, see [mapMaptch.pyt](#mapmatchpyt-arcgis-python-toolbox) 17 | 18 | The code is written in Python 2.7 and depends on: 19 | 20 | * arcpy (ships with ArcGIS and its own Python 2.7) 21 | * [networkx](https://networkx.github.io) 22 | 23 | `python pip install networkx` 24 | 25 | * note: requires installing GDAL first, which can be obtained as a windows wheel from [here](http://www.lfd.uci.edu/~gohlke/pythonlibs/) 26 | and then installed with pip locally: 27 | 28 | `python pip install GDAL-2.1.3-cp27-cp27m-win32.whl` 29 | 30 | To install the mapmatcher Python module, simply download and execute this windows executable: 31 | - [mapmatching/dist/mapmatcher-1.0.win32.exe](https://github.com/simonscheider/mapmatching/blob/master/dist/mapmatcher-1.0.win32.exe). 32 | 33 | ## Usage 34 | 35 | Example: 36 | 37 | `from mapmatcher import mapmatcher` 38 | 39 | `arcpy.env.workspace = 'C:/Users/simon/Documents/GitHub/mapmatching'` 40 | 41 | `opt = mapmatcher.mapMatch('testTrack.shp', 'testSegments.shp')` 42 | 43 | `#outputs testTrack_pth.shp` 44 | 45 | `mapmatcher.exportPath(opt, 'testTrack.shp')` 46 | 47 | The last method saves a new shape file named _testTrack_pth.shp_ in the current arcpy workspace, containing a sequence of segments to which the track was mapped. 48 | 49 | Results are shown here: 50 | 51 | 52 | 53 | 54 | The main method is _mapMatch_. Based on the Viterbi algorithm for Hidden Markov models, 55 | see https://en.wikipedia.org/wiki/Viterbi_algorithm, it gets trackpoints and segments, and returns the most probable segment path (a list of segments) for the list of points. 56 | 57 | ### Method _mapMatch_: 58 | * @param **track** = a shape file (filename) with point geometries representing a track, can be unprojected (WGS84). The order of points in this file should reflect the temporal order. 59 | 60 | * @param **segments** = a shape file of network segments, should be _projected_ (in meter) to compute Euclidean distances properly (e.g. GCS Amersfoord). _Note_: To compute network distances, the script turns this network into a graph using [networkx](https://networkx.github.io), based on coincidence of segment end points. It is therefore important that logically connected segments are also geometrically connected (no geometrical errors). Other than this, the script does have other requirements for the network. 61 | 62 | * @param _decayconstantNet_ (optional) = the network distance (in meter) after which the match probability falls under 0.34 ([exponential decay](https://en.wikipedia.org/wiki/Exponential_decay)). Default is 30 meters. This distance parameter depends on the intervals between successing points in the track. 63 | 64 | * @param _decayConstantEu_ (optional) = the Euclidean distance (in meter) after which the match probability falls under 0.34 (exponential decay). Default is 10 meters. This distance parameter depends on the measurement accuracy of tracking points. 65 | 66 | * @param _maxDist_ (optional) = the Euclidean distance threshold (in meter) for taking into account segments candidates. Default is 50 meters. Depends also on measurement accuracy of track points. 67 | 68 | * result = delivers back a path (a list of segment ids). 69 | 70 | #### Note: 71 | Depending on the type of movement, optional parameters need to be fine tuned to get optimal results. For example, when tracking frequency is very slow, then track points are far apart, and then _decayconstantNet_ needs to be increased accordingly. 72 | 73 | ### Method _exportPath_ : 74 | Exports the path into a shape file named _segments_pth.shp_ inside the current ArcGIS workspace. 75 | 76 | 77 | # mapMatch.pyt (ArcGIS Python toolbox) 78 | 79 | To use the Python method as an ArcGIS toolbox, you need to do the following: 80 | 81 | 1. In your ArcGIS Python version (e.g. Folder `C:\Python27\ArcGIS10.3\Lib\site-packages`), install required modules for GDAL and networkx in a cmd window: 82 | 83 | - if you have not installed it yet, install pip (http://pip.readthedocs.io/en/latest/installing/). 84 | - Download a suitable GDAL wheel from [here](http://www.lfd.uci.edu/~gohlke/pythonlibs/). Then execute: 85 | - `python pip install GDAL-2.1.3-cp27-cp27m-win32.whl` 86 | - `python pip install networkx` 87 | 88 | 2. Install mapmatcher Python module by downloading and executing the windows executable [mapmatching/dist/mapmatcher-1.0.win32.exe](https://github.com/simonscheider/mapmatching/blob/master/dist/mapmatcher-1.0.win32.exe). Make sure you select exactly the Python installation that ships with your ArcGIS as a target folder. 89 | 90 | 3. Download the ArcGIS Python toolbox [mapMatch.pyt](https://github.com/simonscheider/mapmatching/blob/master/mapMatch.pyt), together with meta data files [mapMatch.mapMatch.pyt.xml](https://github.com/simonscheider/mapmatching/blob/master/mapMatch.mapMatch.pyt.xml) and [mapMatch.pyt.xml](https://github.com/simonscheider/mapmatching/blob/master/mapMatch.pyt.xml) and drop it anywhere on your computer. 91 | 92 | 4. Now you can open the toolbox by clicking on it inside an ArcGIS Catalog Window: 93 | 94 | 95 | The tool saves a new shape file named _NameofInputTrack_pth.shp_ inside the current ArcGIS workspace that contains the path of segments to which the track was mapped. When executing, make sure the network is as small as possible to speed up. 96 | -------------------------------------------------------------------------------- /build/lib/mapmatcher/__init__.py: -------------------------------------------------------------------------------- 1 | import mapmatcher 2 | -------------------------------------------------------------------------------- /build/lib/mapmatcher/mapmatcher.py: -------------------------------------------------------------------------------- 1 | """ 2 | ------------------------------------------------------------------------------- 3 | # Name: mapMatcher 4 | # Purpose: This python script allows map matching (matching of track points to a network) 5 | # in arcpy using a Hidden Markov model with 6 | # probabilities parameterized based on spatial + network distances. 7 | # Follows the ideas in Newson, Krumm (2009): 8 | # "Hidden markov Map Matching through noise and sparseness" 9 | # 10 | # Example usage under '__main__' 11 | # 12 | # Author: Simon Scheider 13 | # 14 | # Created: 01/03/2017 15 | # Copyright: (c) simon 2017 16 | # Licence: 17 | 18 | The code is written in Python 2.7 and depends on: 19 | 20 | * arcpy (ships with ArcGIS and its own Python 2.7) 21 | * networkx (# python pip install networkx (https://networkx.github.io)) 22 | (note: requires installing GDAL first, which can be obtained as a wheel from 23 | http://www.lfd.uci.edu/~gohlke/pythonlibs/ and then installed with pip locally: 24 | python pip install GDAL-2.1.3-cp27-cp27m-win32.whl 25 | ) 26 | 27 | #------------------------------------------------------------------------------- 28 | """ 29 | 30 | __author__ = "Simon Scheider" 31 | __copyright__ = "" 32 | 33 | 34 | import sys 35 | 36 | try: 37 | from math import exp, sqrt 38 | import os 39 | import arcpy 40 | arcpy.env.overwriteOutput = True 41 | import networkx as nx 42 | import time 43 | 44 | except ImportError: 45 | print "Error: missing one of the libraries (arcpy, networkx)" 46 | sys.exit() 47 | 48 | 49 | 50 | def mapMatch(track, segments, decayconstantNet = 30, decayConstantEu = 10, maxDist = 50, addfullpath = True): 51 | """ 52 | The main method. Based on the Viterbi algorithm for Hidden Markov models, 53 | see https://en.wikipedia.org/wiki/Viterbi_algorithm. 54 | It gets trackpoints and segments, and returns the most probable segment path (a list of segments) for the list of points. 55 | Inputs: 56 | @param track = a shape file (filename) representing a track, can also be unprojected (WGS84) 57 | @param segments = a shape file of network segments, should be projected (in meter) to compute Euclidean distances properly (e.g. GCS Amersfoord) 58 | @param decayconstantNet (optional) = the network distance (in meter) after which the match probability falls under 0.34 (exponential decay). (note this is the inverse of lambda). 59 | This depends on the point frequency of the track (how far are track points separated?) 60 | @param decayConstantEu (optional) = the Euclidean distance (in meter) after which the match probability falls under 0.34 (exponential decay). (note this is the inverse of lambda). 61 | This depends on the positional error of the track points (how far can points deviate from their true position?) 62 | @param maxDist (optional) = the Euclidean distance threshold (in meter) for taking into account segments candidates. 63 | @param addfullpath (optional, True or False) = whether a contiguous full segment path should be outputted. If not, a 1-to-1 list of segments matching each track point is outputted. 64 | 65 | note: depending on the type of movement, optional parameters need to be fine tuned to get optimal results. 66 | """ 67 | #Make sure passed in parameters are floats 68 | decayconstantNet = float(decayconstantNet) 69 | decayConstantEu = float(decayConstantEu) 70 | maxDist= float(maxDist) 71 | 72 | #gest start time 73 | start_time = time.time() 74 | 75 | #this array stores, for each point in a track, probability distributions over segments, together with the (most probable) predecessor segment taking into account a network distance 76 | V = [{}] 77 | 78 | #get track points, build network graph (graph, endpoints, lengths) and get segment info from arcpy 79 | points = getTrackPoints(track, segments) 80 | r = getSegmentInfo(segments) 81 | endpoints = r[0] 82 | lengths = r[1] 83 | graph = getNetworkGraph(segments,lengths) 84 | pathnodes = [] #set of pathnodes to prevent loops 85 | 86 | #init first point 87 | sc = getSegmentCandidates(points[0], segments, decayConstantEu, maxDist) 88 | for s in sc: 89 | V[0][s] = {"prob": sc[s], "prev": None, "path": [], "pathnodes":[]} 90 | # Run Viterbi when t > 0 91 | for t in range(1, len(points)): 92 | V.append({}) 93 | #Store previous segment candidates 94 | lastsc = sc 95 | #Get segment candidates and their a-priori probabilities (based on Euclidean distance for current point t) 96 | sc = getSegmentCandidates(points[t], segments, decayConstantEu, maxDist) 97 | for s in sc: 98 | max_tr_prob = 0 99 | prev_ss = None 100 | path = [] 101 | for prev_s in lastsc: 102 | #determine the highest network transition probability from previous candidates to s and get the corresponding network path 103 | pathnodes = V[t-1][prev_s]["pathnodes"][-10:] 104 | n = getNetworkTransP(prev_s, s, graph, endpoints, lengths, pathnodes, decayconstantNet) 105 | np = n[0] #This is the network transition probability 106 | tr_prob = V[t-1][prev_s]["prob"]*np 107 | #this selects the most probable predecessor candidate and the path to it 108 | if tr_prob > max_tr_prob: 109 | max_tr_prob = tr_prob 110 | prev_ss = prev_s 111 | path = n[1] 112 | if n[2] != None: 113 | pathnodes.append(n[2]) 114 | #The final probability of a candidate is the product of a-priori and network transitional probability 115 | max_prob = sc[s] * max_tr_prob 116 | V[t][s] = {"prob": max_prob, "prev": prev_ss, "path": path, "pathnodes":pathnodes} 117 | 118 | #Now max standardize all p-values to prevent running out of digits 119 | maxv = max(value["prob"] for value in V[t].values()) 120 | maxv = (1 if maxv == 0 else maxv) 121 | for s in V[t].keys(): 122 | V[t][s]["prob"]=V[t][s]["prob"]/maxv 123 | 124 | 125 | intertime1 = time.time() 126 | print("--- Viterbi forward: %s seconds ---" % (intertime1 - start_time)) 127 | #print V 128 | 129 | #opt is the result: a list of (matched) segments [s1, s2, s3,...] in the exact order of the point track: [p1, p2, p3,...] 130 | opt = [] 131 | 132 | # get the highest probability at the end of the track 133 | max_prob = max(value["prob"] for value in V[-1].values()) 134 | previous = None 135 | if max_prob == 0: 136 | print " probabilities fall to zero (network distances in data are too large, try increasing network decay parameter)" 137 | 138 | # Get most probable ending state and its backtrack 139 | for st, data in V[-1].items(): 140 | if data["prob"] == max_prob: 141 | opt.append(st) 142 | previous = st 143 | break 144 | ## print " previous: "+str(previous) 145 | ## print " max_prob: "+str(max_prob) 146 | ## print " V -1: "+str(V[-1].items()) 147 | 148 | # Follow the backtrack till the first observation to fish out most probable states and corresponding paths 149 | for t in range(len(V) - 2, -1, -1): 150 | #Get the subpath between last and most probable previous segment and add it to the resulting path 151 | path = V[t + 1][previous]["path"] 152 | opt[0:0] =(path if path !=None else []) 153 | #Insert the previous segment 154 | opt.insert(0, V[t + 1][previous]["prev"]) 155 | previous = V[t + 1][previous]["prev"] 156 | intertime2 = time.time() 157 | print("--- Viterbi backtracking: %s seconds ---" % (intertime2 - intertime1)) 158 | 159 | #Clean the path (remove double segments and crossings) (only in full path option) 160 | print "path length before cleaning :" +str(len(opt)) 161 | opt = cleanPath(opt, endpoints) 162 | intertime3 = time.time() 163 | print("--- Path cleaning: %s seconds ---" % (intertime3 - intertime2)) 164 | print "final length: "+str(len(opt)) 165 | pointstr= [str(g.firstPoint.X)+' '+str(g.firstPoint.Y) for g in points] 166 | optstr= [str(i) for i in opt] 167 | print 'The path for points ['+' '.join(pointstr)+'] is: ' 168 | print '[' + ' '.join(optstr) + '] with highest probability of %s' % max_prob 169 | 170 | #If only a single segment candidate should be returned for each point: 171 | if addfullpath == False: 172 | opt = getpointMatches(points,opt) 173 | optstr= [str(i) for i in opt] 174 | print "Individual point matches: "+'[' + ' '.join(optstr) + ']' 175 | intertime4 = time.time() 176 | print("--- Picking point matches: %s seconds ---" % (intertime4 - intertime3)) 177 | 178 | return opt 179 | 180 | #Fishes out a 1-to-1 list of path segments nearest to the list of points in the track (not contiguous, may contain repeated segments) 181 | def getpointMatches(points, path): 182 | qr = '"OBJECTID" IN ' +str(tuple(path)) 183 | arcpy.SelectLayerByAttribute_management('segments_lyr',"NEW_SELECTION", qr) 184 | opta = [] 185 | for point in points: 186 | sdist = 100000 187 | candidate = '' 188 | cursor = arcpy.da.SearchCursor('segments_lyr', ["OBJECTID", "SHAPE@"]) 189 | for row in cursor: 190 | #compute the spatial distance 191 | dist = point.distanceTo(row[1]) 192 | if dist