├── .gitignore ├── CONTRIBUTING.md ├── LargeNetworkAnalysisTools.ParallelCalculateLocations.pyt.xml ├── LargeNetworkAnalysisTools.SolveLargeAnalysisWithKnownPairs.pyt.xml ├── LargeNetworkAnalysisTools.SolveLargeODCostMatrix.pyt.xml ├── LargeNetworkAnalysisTools.pyt ├── README.md ├── conference-slides ├── DevSummit2020_SolvingLargeTransportationAnalysisProblems.pdf ├── DevSummit2021_SolvingLargeTransportationAnalysisProblems.pdf └── DevSummit2022_SolvingLargeTransportationAnalysisProblems.pdf ├── helpers.py ├── images ├── CalculateLocations_SQL_1.png ├── CalculateLocations_SQL_2.png ├── ParallelCalculateLocations_Dialog.png ├── ParallelCalculateLocations_SQL.png ├── SolveLargeAnalysisWithKnownODPairs_ManyToMany_Dialog.png ├── SolveLargeAnalysisWithKnownODPairs_OneToOne_Dialog.png └── SolveLargeODCostMatrix_Dialog.png ├── license.txt ├── od_config.py ├── parallel_calculate_locations.py ├── parallel_odcm.py ├── parallel_route_pairs.py ├── rt_config.py ├── solve_large_odcm.py ├── solve_large_route_pair_analysis.py └── unittests ├── __init__.py ├── input_data_helper.py ├── portal_credentials.py ├── test_ParallelCalculateLocations_tool.py ├── test_SolveLargeAnalysisWithKnownPairs_tool.py ├── test_SolveLargeODCostMatrix_tool.py ├── test_helpers.py ├── test_parallel_calculate_locations.py ├── test_parallel_odcm.py ├── test_parallel_route_pairs.py ├── test_solve_large_odcm.py ├── test_solve_large_route_pair_analysis.py └── unittests_README.txt /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | odcm_outputs.txt 3 | 4 | *.pyc 5 | 6 | LargeNetworkAnalysisTools.pyt.xml 7 | 8 | /unittests/TestInput 9 | /unittests/TestOutput 10 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Esri welcomes contributions from anyone and everyone. Please see our [guidelines for contributing](https://github.com/esri/contributing). -------------------------------------------------------------------------------- /LargeNetworkAnalysisTools.ParallelCalculateLocations.pyt.xml: -------------------------------------------------------------------------------- 1 | 2 | 20230330153136001.0TRUE20230405160344001500000005000ItemDescriptionc:\program files\arcgis\pro\Resources\Help\gp<DIV STYLE="text-align:Left;"><DIV><P><SPAN>The point feature class or layer whose network locations you want to calculate.</SPAN></P></DIV></DIV><DIV STYLE="text-align:Left;"><DIV><P><SPAN>The catalog path to the output feature class.</SPAN></P></DIV></DIV><DIV STYLE="text-align:Left;"><DIV><P><SPAN>Network dataset or network dataset layer to use when calculating network locations.</SPAN></P></DIV></DIV><DIV STYLE="text-align:Left;"><DIV><P><SPAN>Defines the number of features that will be in each chunk in the parallel processing.</SPAN></P></DIV></DIV><DIV STYLE="text-align:Left;"><DIV><P><SPAN>Defines the maximum number of parallel processes to run at once. Do not exceed the number of logical processors of your machine.</SPAN></P></DIV></DIV><DIV STYLE="text-align:Left;"><DIV><P><SPAN>Network travel mode to use when calculating network locations.</SPAN></P></DIV></DIV><DIV STYLE="text-align:Left;"><DIV><P><SPAN>The maximum search distance that will be used when locating the input features on the network. Features that are outside the search tolerance will be left unlocated.</SPAN></P></DIV></DIV><DIV STYLE="text-align:Left;"><DIV><P><SPAN>The network dataset source feature classes on which input features are allowed to locate.</SPAN></P></DIV></DIV><DIV STYLE="text-align:Left;"><DIV><P><SPAN>An optional query for each network dataset source feature class filtering the source features that can be located on.</SPAN></P></DIV></DIV>

<DIV STYLE="text-align:Left;"><DIV><P><SPAN>Calculate network locations for a large dataset by chunking it and solving in parallel.</SPAN></P></DIV></DIV>

Parallel Calculate Locationsnetwork locationscalculate locationsnetwork analystparallelArcToolbox Tool20230405 3 | -------------------------------------------------------------------------------- /LargeNetworkAnalysisTools.SolveLargeAnalysisWithKnownPairs.pyt.xml: -------------------------------------------------------------------------------- 1 | 2 | 20220401095418001.0TRUE2023010694021001500000005000ItemDescriptionc:\program files\arcgis\pro\Resources\Help\gp<DIV STYLE="text-align:Left;"><DIV><DIV><P><SPAN>The feature class or layer containing the origins. For the one-to-one Origin-Destination Assignment Type, the origins dataset must have a field populated with the ID of the destination the origin is assigned to.</SPAN></P></DIV></DIV></DIV><DIV STYLE="text-align:Left;"><DIV><DIV><P><SPAN>A field in origins representing the origin's unique ID. For the many-to-many Origin-Destination Assignment Type, the values in the Origin-Destination Pair Table's origin ID field must correspond to these unique origin IDs.</SPAN></P></DIV></DIV></DIV><DIV STYLE="text-align:Left;"><DIV><DIV><P><SPAN>The feature class or layer containing the destinations.</SPAN></P></DIV></DIV></DIV><DIV STYLE="text-align:Left;"><P><SPAN>A field in destinations representing the destination's unique ID. For the one-to-one Origin-Destination Assignment Type, the values in the origins table's Assigned Destination Field should correspond to these unique destination IDs. For the many-to-many Origin-Destination Assignment Type, the values in the Origin-Destination Pair Table's destination ID field must correspond to these unique destination IDs.</SPAN></P></DIV><DIV STYLE="text-align:Left;"><P><SPAN>A text string indicating which type of preassigned origin-destination pairs to use for the analysis. The options are:</SPAN></P><UL><LI><P><SPAN>A field in Origins defines the assigned Destination (one-to-one)</SPAN></P></LI><LI><P><SPAN>A separate table defines the origin-destination pairs (many-to-many)</SPAN></P></LI></UL></DIV><DIV STYLE="text-align:Left;"><P><SPAN>A field in Origins indicating the ID of the destination each origin is assigned to. Any origin with a null value or a value that does not match a valid destination ID will be ignored in the analysis. This parameter is only applicable for the one-to-one Origin-Destination Assignment Type.</SPAN></P></DIV><DIV STYLE="text-align:Left;"><DIV><P><SPAN>A table or CSV file defining origin-destination pairs. The table must be populated with a column of origin IDs matching values in the Origin Unique ID Field of the Origins table and a column of destination IDs matching values in the Destination Unique ID Field of the Destinations table. This parameter is only applicable for the many-to-many Origin-Destination Assignment Type.</SPAN></P></DIV></DIV><DIV STYLE="text-align:Left;"><DIV><P><SPAN>The field name in the Origin-Destination Pair Table defining the origin IDs. This parameter is only applicable for the many-to-many Origin-Destination Assignment Type.</SPAN></P></DIV></DIV><DIV STYLE="text-align:Left;"><DIV><P><SPAN>The field name in the Origin-Destination Pair Table defining the destination IDs. This parameter is only applicable for the many-to-many Origin-Destination Assignment Type.</SPAN></P></DIV></DIV><DIV STYLE="text-align:Left;"><DIV><DIV><P><SPAN>Network dataset, network dataset layer, or portal URL to use when calculating the Route analysis.</SPAN></P></DIV></DIV></DIV><DIV STYLE="text-align:Left;"><DIV><DIV><P><SPAN>Network travel mode to use when calculating the Route analysis.</SPAN></P></DIV></DIV></DIV><DIV STYLE="text-align:Left;"><DIV><DIV><P><SPAN>The time units the output travel time will be reported in.</SPAN></P></DIV></DIV></DIV><DIV STYLE="text-align:Left;"><DIV><DIV><P><SPAN>The distance units the output travel distance will be reported in.</SPAN></P></DIV></DIV></DIV><DIV STYLE="text-align:Left;"><DIV><DIV><P><SPAN>Defines the chunk size for parallel Route calculations, the number of origin-destination routes to calculate simultaneously. For example, if you want to process a maximum of 1000 origins and 1000 destinations in a single chunk, for a total of 1000 paired routes, set this parameter to 1000.</SPAN></P></DIV></DIV></DIV><DIV STYLE="text-align:Left;"><DIV><DIV><P><SPAN>Defines the maximum number of parallel processes to run at once. Do not exceed the number of logical processors of your machine.</SPAN></P></DIV></DIV></DIV><DIV STYLE="text-align:Left;"><DIV><P><SPAN>Path to the output feature class that will contain the calculated routes between origins and their assigned destinations.</SPAN></P></DIV></DIV><DIV STYLE="text-align:Left;"><DIV><DIV><P><SPAN>The start time of day for the analysis. No value indicates a time neutral analysis.</SPAN></P></DIV></DIV></DIV><DIV STYLE="text-align:Left;"><DIV><DIV><P><SPAN>Point, line, or polygon barriers to use in the OD Cost Matrix analysis. This parameter is optional.</SPAN></P></DIV></DIV></DIV><DIV STYLE="text-align:Left;"><P><SPAN>When you solve a network analysis, the input points must "locate" on the network used for the analysis. If origins and destinations are used more than once, it is more efficient to calculate the network location fields up front and re-use them. Set this parameter to True to pre-calculate the network location fields. This is recommended unless:</SPAN></P><UL><LI><P><SPAN>You are using a portal URL as the network data source. In this case, pre-calculating network locations is not possible, and the parameter is hidden.</SPAN></P></LI><LI><P><SPAN>You have already pre-calculated the network location fields using the network dataset and travel mode you are using for this analysis. In this case, you can save time by not precalculating them again.</SPAN></P></LI><LI><P><SPAN>Each destination has only one assigned origin. In this case, there is no efficiency gain in calculating the location fields in advance.</SPAN></P></LI></UL></DIV><DIV STYLE="text-align:Left;"><P><SPAN>A Boolean indicating whether to sort origins by their assigned destination prior to commencing the parallel solve. Using sorted data will improve the efficiency of the solve slightly. If your input data is already sorted, or if no origins are assigned to the same destinations, then sorting is not useful, and you should set this parameter to false. This parameter is only applicable for the one-to-one Origin-Destination Assignment Type.</SPAN></P></DIV><DIV STYLE="text-align:Left;"><DIV><DIV><P><SPAN>A Boolean indicating whether to reverse the direction of travel and calculate the route from the destination to the origin. The default is false. This parameter is only applicable for the one-to-one Origin-Destination Assignment Type.</SPAN></P></DIV></DIV></DIV>

<DIV STYLE="text-align:Left;"><DIV><DIV><P><SPAN>Calculate the travel time and distance and generate routes between known origin-destination pairs by chunking up the problem and solving in parallel.</SPAN></P></DIV></DIV></DIV>

Solve Large Analysis With Known OD PairsRoute pairsparallel processingOD Cost MatrixNetwork AnalystArcToolbox Tool20230106 3 | -------------------------------------------------------------------------------- /LargeNetworkAnalysisTools.SolveLargeODCostMatrix.pyt.xml: -------------------------------------------------------------------------------- 1 | 2 | 20210326134543001.0TRUE2023052585647001500000005000ItemDescriptionc:\program files\arcgis\pro\Resources\Help\gp<DIV STYLE="text-align:Left;"><DIV><DIV><P><SPAN>The feature class or layer containing the origins.</SPAN></P></DIV></DIV></DIV><DIV STYLE="text-align:Left;"><DIV><DIV><P><SPAN>The feature class or layer containing the destinations.</SPAN></P></DIV></DIV></DIV><DIV STYLE="text-align:Left;"><DIV><P><SPAN>Network dataset, network dataset layer, or portal URL to use when calculating the OD Cost Matrix.</SPAN></P></DIV></DIV><DIV STYLE="text-align:Left;"><DIV><DIV><P><SPAN>Network travel mode to use when calculating the OD Cost Matrix</SPAN></P></DIV></DIV></DIV><DIV STYLE="text-align:Left;"><DIV><P><SPAN>The time units the output Total_Time field will be reported in.</SPAN></P></DIV></DIV><DIV STYLE="text-align:Left;"><DIV><P><SPAN>The distance units the output Total_Distance field will be reported in.</SPAN></P></DIV></DIV><DIV STYLE="text-align:Left;"><DIV><P><SPAN>Defines the chunk size for parallel OD Cost Matrix calculations. For example, if you want to process a maximum of 1000 origins and 1000 destinations in a single chunk, set this parameter to 1000.</SPAN></P></DIV></DIV><DIV STYLE="text-align:Left;"><DIV><DIV><P><SPAN>Defines the maximum number of parallel processes to run at once. Do not exceed the number of logical processors of your machine.</SPAN></P></DIV></DIV></DIV><DIV STYLE="text-align:Left;"><DIV><P><SPAN>Path to the output feature class that will contain the updated origins, which may be spatially sorted and have added fields. The OriginOID field in the Output OD Lines Feature Class refers to the ObjectID of the Output Updated Origins and not the original input origins.</SPAN></P></DIV></DIV><DIV STYLE="text-align:Left;"><DIV><DIV><P><SPAN>Path to the output feature class that will contain the updated destinations, which may be spatially sorted and have added fields. The DestinationOID field in the Output OD Lines Feature Class refers to the ObjectID of the Output Updated Destinations and not the original input destinations.</SPAN></P></DIV></DIV></DIV><DIV STYLE="text-align:Left;"><DIV><P><SPAN>The desired output format for the OD Cost Matrix Lines. The available choices are:</SPAN></P><UL><LI><P><SPAN>"Feature class" - A single, combined feature class. This option is the slowest to create and will likely fail for extremely large problems.</SPAN></P></LI><LI><P><SPAN>"CSV files" - A set of .csv files. Each file represents the OD Cost Matrix Lines output, without shape geometry, for a chunk of origins and a chunk of destinations, using the naming scheme `ODLines_O_#_#_D_#_#.csv`, where the `#` signs represent the ObjectID ranges of the origins and destinations in the chunk. If you have set a value for the Number of Destinations to Find for Each Origin parameter, you may find some output files using the naming scheme `ODLines_O_#_#.csv` because results from all destinations have been combined into one file.</SPAN></P></LI><LI><P><SPAN>"Apache Arrow files" - A set of Apache Arrow files. Each file represents the OD Cost Matrix Lines output, without shape geometry, for a chunk of origins and a chunk of destinations, using the naming scheme `ODLines_O_#_#_D_#_#.arrow`, where the `#` signs represent the ObjectID ranges of the origins and destinations in the chunk. If you have set a value for the Number of Destinations to Find for Each Origin parameter, you may find some output files using the naming scheme `ODLines_O_#_#.arrow` because results from all destinations have been combined into one file. This option is not available in versions of ArcGIS Pro prior to 2.9 and is not available if the network data source is a service.</SPAN></P></LI></UL></DIV></DIV><DIV STYLE="text-align:Left;"><DIV><DIV><P><SPAN>Path to the output feature class that will contain the OD Cost Matrix Lines output computed by the tool. The schema of this feature class is described in the </SPAN><A href="https://pro.arcgis.com:443/en/pro-app/latest/arcpy/network-analyst/origindestinationcostmatrix-output-data-types.htm" target="ESRI_SECTION1_9FF9489173C741DD95472F21B5AD8374" STYLE="text-decoration:underline;"><SPAN>arcpy documentation</SPAN></A><SPAN>.</SPAN></P></DIV></DIV></DIV><DIV STYLE="text-align:Left;"><DIV><P><SPAN>Path to a folder, which will be created by the tool, that will contain the CSV or Arrow files representing the OD Cost Matrix Lines results if the Output OD Cost Matrix Format parameter value is "CSV files" or "Apache Arrow files". The schema of the files is described in the </SPAN><A href="https://pro.arcgis.com:443/en/pro-app/latest/arcpy/network-analyst/origindestinationcostmatrix-output-data-types.htm" target="ESRI_SECTION1_9FF9489173C741DD95472F21B5AD8374" STYLE="text-decoration:underline;"><SPAN>arcpy documentation</SPAN></A><SPAN>.</SPAN></P></DIV></DIV><DIV STYLE="text-align:Left;"><DIV><P><SPAN>Impedance cutoff limiting the search distance for each origin. For example, you could set up the problem to find only destinations within a 15 minute drive time of the origins. This parameter is optional. Leaving it blank uses no cutoff.</SPAN></P><UL><LI><P><SPAN> If your travel mode has time-based impedance units, Cutoff represents a time and is interpreted in the units specified in the Time Units parameter.</SPAN></P></LI><LI><P><SPAN> If your travel mode has distance-based impedance units, Cutoff represents a distance and is interpreted in the units specified in the Distance Units parameter.</SPAN></P></LI><LI><P><SPAN> If your travel mode has other units (not time- or distance-based), Cutoff should be specified in the units of your travel mode's impedance attribute.</SPAN></P></LI></UL></DIV></DIV><DIV STYLE="text-align:Left;"><DIV><P><SPAN>The number of destinations to find for each origin. For example, setting this to 3 will result in the output including the travel time and distance from each origin to its three closest destinations. This parameter is optional. Leaving it blank results in finding the travel time and distance from each origin to all destinations.</SPAN></P></DIV></DIV><DIV STYLE="text-align:Left;"><DIV><P><SPAN>The start time of day for the analysis. No value indicates a time neutral analysis.</SPAN></P></DIV></DIV><DIV STYLE="text-align:Left;"><DIV><P><SPAN>Point, line, and polygon barriers to use in the OD Cost Matrix analysis.</SPAN></P></DIV></DIV><DIV STYLE="text-align:Left;"><DIV><DIV><P><SPAN>When you solve a network analysis, the input points must "locate" on the network used for the analysis. When chunking your inputs to solve in parallel, inputs may be used many times. Rather than calculating the network location fields for each input every time it is used, it is more efficient to calculate all the network location fields up front and re-use them. Set this parameter to True to pre-calculate the network location fields. This is recommended for every situation unless:</SPAN></P><UL><LI><P><SPAN> You are using a portal URL as the network data source. In this case, pre-calculating network locations is not possible, and the parameter is hidden.</SPAN></P></LI><LI><P><SPAN> You have already pre-calculated the network location fields using the network dataset and travel mode you are using for this analysis. In this case, you can save time by not precalculating them again.</SPAN></P></LI></UL></DIV></DIV></DIV><DIV STYLE="text-align:Left;"><DIV><P><SPAN>Whether to spatially sort origins and destinations prior to commencing the parallel solve. Using sorted data will improve the efficiency of the chunking behavior significantly, and it may reduce the number of credits consumed if you're using a service that charges credits. If your input data is already sorted, then sorting is not useful, and you should set this parameter to false. Otherwise, you should set his parameter to true. Note, however, that spatial sorting is only available if you have the Advanced license. Spatial sorting will be skipped automatically if you don't have the necessary license, and the parameter will be hidden in the tool dialog.</SPAN></P></DIV></DIV>

<DIV STYLE="text-align:Left;"><DIV><DIV><P><SPAN>Sample script tool to solve a large OD Cost Matrix problem by chunking up the data and solving the chunks in parallel.</SPAN></P></DIV></DIV></DIV>

Solve Large OD Cost MatrixNetwork AnalystOD Cost MatrixlargeoriginsdestinationsArcToolbox Tool20230525 3 | -------------------------------------------------------------------------------- /conference-slides/DevSummit2020_SolvingLargeTransportationAnalysisProblems.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/large-network-analysis-tools/b8503c5d606260f53acad6fd3bff612c242e2da4/conference-slides/DevSummit2020_SolvingLargeTransportationAnalysisProblems.pdf -------------------------------------------------------------------------------- /conference-slides/DevSummit2021_SolvingLargeTransportationAnalysisProblems.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/large-network-analysis-tools/b8503c5d606260f53acad6fd3bff612c242e2da4/conference-slides/DevSummit2021_SolvingLargeTransportationAnalysisProblems.pdf -------------------------------------------------------------------------------- /conference-slides/DevSummit2022_SolvingLargeTransportationAnalysisProblems.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/large-network-analysis-tools/b8503c5d606260f53acad6fd3bff612c242e2da4/conference-slides/DevSummit2022_SolvingLargeTransportationAnalysisProblems.pdf -------------------------------------------------------------------------------- /images/CalculateLocations_SQL_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/large-network-analysis-tools/b8503c5d606260f53acad6fd3bff612c242e2da4/images/CalculateLocations_SQL_1.png -------------------------------------------------------------------------------- /images/CalculateLocations_SQL_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/large-network-analysis-tools/b8503c5d606260f53acad6fd3bff612c242e2da4/images/CalculateLocations_SQL_2.png -------------------------------------------------------------------------------- /images/ParallelCalculateLocations_Dialog.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/large-network-analysis-tools/b8503c5d606260f53acad6fd3bff612c242e2da4/images/ParallelCalculateLocations_Dialog.png -------------------------------------------------------------------------------- /images/ParallelCalculateLocations_SQL.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/large-network-analysis-tools/b8503c5d606260f53acad6fd3bff612c242e2da4/images/ParallelCalculateLocations_SQL.png -------------------------------------------------------------------------------- /images/SolveLargeAnalysisWithKnownODPairs_ManyToMany_Dialog.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/large-network-analysis-tools/b8503c5d606260f53acad6fd3bff612c242e2da4/images/SolveLargeAnalysisWithKnownODPairs_ManyToMany_Dialog.png -------------------------------------------------------------------------------- /images/SolveLargeAnalysisWithKnownODPairs_OneToOne_Dialog.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/large-network-analysis-tools/b8503c5d606260f53acad6fd3bff612c242e2da4/images/SolveLargeAnalysisWithKnownODPairs_OneToOne_Dialog.png -------------------------------------------------------------------------------- /images/SolveLargeODCostMatrix_Dialog.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/large-network-analysis-tools/b8503c5d606260f53acad6fd3bff612c242e2da4/images/SolveLargeODCostMatrix_Dialog.png -------------------------------------------------------------------------------- /license.txt: -------------------------------------------------------------------------------- 1 | Apache License - 2.0 2 | 3 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 4 | 5 | 1. Definitions. 6 | 7 | "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. 8 | 9 | "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. 10 | 11 | "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control 12 | with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management 13 | of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial 14 | ownership of such entity. 15 | 16 | "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. 17 | 18 | "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, 19 | and configuration files. 20 | 21 | "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to 22 | compiled object code, generated documentation, and conversions to other media types. 23 | 24 | "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice 25 | that is included in or attached to the work (an example is provided in the Appendix below). 26 | 27 | "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the 28 | editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes 29 | of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, 30 | the Work and Derivative Works thereof. 31 | 32 | "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work 33 | or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual 34 | or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of 35 | electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on 36 | electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for 37 | the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing 38 | by the copyright owner as "Not a Contribution." 39 | 40 | "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and 41 | subsequently incorporated within the Work. 42 | 43 | 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, 44 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, 45 | publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 46 | 47 | 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, 48 | non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, 49 | sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are 50 | necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was 51 | submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work 52 | or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You 53 | under this License for that Work shall terminate as of the date such litigation is filed. 54 | 55 | 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, 56 | and in Source or Object form, provided that You meet the following conditions: 57 | 58 | 1. You must give any other recipients of the Work or Derivative Works a copy of this License; and 59 | 60 | 2. You must cause any modified files to carry prominent notices stating that You changed the files; and 61 | 62 | 3. You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices 63 | from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and 64 | 65 | 4. If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a 66 | readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the 67 | Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the 68 | Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever 69 | such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. 70 | You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, 71 | provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to 72 | Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your 73 | modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with 74 | the conditions stated in this License. 75 | 76 | 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You 77 | to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, 78 | nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 79 | 80 | 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except 81 | as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 82 | 83 | 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides 84 | its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, 85 | any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for 86 | determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under 87 | this License. 88 | 89 | 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required 90 | by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, 91 | including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the 92 | use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or 93 | any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 94 | 95 | 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a 96 | fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting 97 | such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree 98 | to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your 99 | accepting any such warranty or additional liability. 100 | 101 | END OF TERMS AND CONDITIONS -------------------------------------------------------------------------------- /od_config.py: -------------------------------------------------------------------------------- 1 | """Defines OD Cost matrix solver object properties that are not specified 2 | in the tool dialog. 3 | 4 | A list of OD cost matrix solver properties is documented here: 5 | https://pro.arcgis.com/en/pro-app/latest/arcpy/network-analyst/odcostmatrix.htm 6 | 7 | You can include any of them in the dictionary in this file, and the tool will 8 | use them. However, travelMode, timeUnits, distanceUnits, defaultImpedanceCutoff, 9 | defaultDestinationCount, and timeOfDay will be ignored because they are specified 10 | in the tool dialog. 11 | 12 | Copyright 2022 Esri 13 | Licensed under the Apache License, Version 2.0 (the "License"); 14 | you may not use this file except in compliance with the License. 15 | You may obtain a copy of the License at 16 | http://www.apache.org/licenses/LICENSE-2.0 17 | Unless required by applicable law or agreed to in writing, software 18 | distributed under the License is distributed on an "AS IS" BASIS, 19 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 20 | See the License for the specific language governing permissions and 21 | limitations under the License. 22 | """ 23 | import arcpy 24 | 25 | # These properties are set by the tool dialog or can be specified as command line arguments. Do not set the values for 26 | # these properties in the OD_PROPS dictionary below because they will be ignored. 27 | OD_PROPS_SET_BY_TOOL = [ 28 | "travelMode", "timeUnits", "distanceUnits", "defaultImpedanceCutoff", "defaultDestinationCount", "timeOfDay"] 29 | 30 | # You can customize these properties to your needs, and the parallel OD cost matrix calculations will use them. 31 | OD_PROPS = { 32 | "accumulateAttributeNames": [], 33 | "allowSaveLayerFile": False, 34 | "ignoreInvalidLocations": True, 35 | "lineShapeType": arcpy.nax.LineShapeType.NoLine, 36 | "overrides": "", 37 | # "searchQuery": [], # This parameter is very network specific. Only uncomment if you are using it. 38 | "searchTolerance": 5000, 39 | "searchToleranceUnits": arcpy.nax.DistanceUnits.Meters, 40 | "timeZone": arcpy.nax.TimeZoneUsage.LocalTimeAtLocations, 41 | } 42 | -------------------------------------------------------------------------------- /parallel_calculate_locations.py: -------------------------------------------------------------------------------- 1 | """Calculate the network locations for a large dataset by chunking the 2 | inputs and solving in parallel. 3 | 4 | This is a sample script users can modify to fit their specific needs. 5 | 6 | Note: Unlike in the core Calculate Locations tool, this tool generates 7 | a new feature class instead of merely adding fields to the original. 8 | A new feature class must be generated during the parallel processing, 9 | and as a result, the ObjectIDs may change, so we ask the user to specify 10 | an output feature class path instead of overwriting the original. If you 11 | need the original ObjectIDs, please calculate an additional field to track 12 | them before calling this tool. We also do this to avoid accidentally deleting 13 | the user's original data if the tool errors. 14 | 15 | This script is intended to be called as a subprocess from a other scripts 16 | so that it can launch parallel processes with concurrent.futures. It must be 17 | called as a subprocess because the main script tool process, when running 18 | within ArcGIS Pro, cannot launch parallel subprocesses on its own. 19 | 20 | This script should not be called directly from the command line. 21 | 22 | Copyright 2024 Esri 23 | Licensed under the Apache License, Version 2.0 (the "License"); 24 | you may not use this file except in compliance with the License. 25 | You may obtain a copy of the License at 26 | http://www.apache.org/licenses/LICENSE-2.0 27 | Unless required by applicable law or agreed to in writing, software 28 | distributed under the License is distributed on an "AS IS" BASIS, 29 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 30 | See the License for the specific language governing permissions and 31 | limitations under the License. 32 | """ 33 | # pylint: disable=logging-fstring-interpolation 34 | import os 35 | import uuid 36 | import logging 37 | import shutil 38 | import time 39 | import traceback 40 | import argparse 41 | 42 | import arcpy 43 | 44 | import helpers 45 | 46 | DELETE_INTERMEDIATE_OUTPUTS = True # Set to False for debugging purposes 47 | 48 | # Change logging.INFO to logging.DEBUG to see verbose debug messages 49 | LOG_LEVEL = logging.INFO 50 | 51 | 52 | class LocationCalculator( 53 | helpers.JobFolderMixin, helpers.LoggingMixin, helpers.MakeNDSLayerMixin 54 | ): # pylint:disable = too-many-instance-attributes 55 | """Used for calculating network locations for a designated chunk of the input datasets.""" 56 | 57 | def __init__(self, **kwargs): 58 | """Initialize the location calculator for the given inputs. 59 | 60 | Expected arguments: 61 | - input_fc 62 | - network_data_source 63 | - travel_mode 64 | - search_tolerance 65 | - search_criteria 66 | - search_query 67 | - scratch_folder 68 | """ 69 | self.input_fc = kwargs["input_fc"] 70 | self.network_data_source = kwargs["network_data_source"] 71 | self.travel_mode = kwargs["travel_mode"] 72 | self.scratch_folder = kwargs["scratch_folder"] 73 | self.search_tolerance = kwargs.get("search_tolerance", None) 74 | self.search_criteria = kwargs.get("search_criteria", None) 75 | self.search_query = kwargs.get("search_query", None) 76 | 77 | # Create a job ID and a folder for this job 78 | self._create_job_folder() 79 | 80 | # Setup the class logger. Logs for each parallel process are not written to the console but instead to a 81 | # process-specific log file. 82 | self.setup_logger("CalcLocs") 83 | 84 | # Create a network dataset layer if needed 85 | self._make_nds_layer() 86 | 87 | # Define output feature class path for this chunk (set during feature selection) 88 | self.out_fc = None 89 | 90 | # Prepare a dictionary to store info about the analysis results 91 | self.job_result = { 92 | "jobId": self.job_id, 93 | "jobFolder": self.job_folder, 94 | "outputFC": "", 95 | "oidRange": None, 96 | "logFile": self.log_file 97 | } 98 | 99 | def _subset_inputs(self, oid_range): 100 | """Create a layer from the input feature class that contains only the OIDs for this chunk. 101 | 102 | Args: 103 | oid_range (list): Input feature class ObjectID range to select for this chunk 104 | """ 105 | # Copy the subset of features in this OID range to a feature class in the job gdb so we can calculate locations 106 | # on it without interference from other parallel processes 107 | self.logger.debug("Subsetting features for this chunk...") 108 | out_gdb = self._create_output_gdb() 109 | self.out_fc = os.path.join(out_gdb, f"Locs_{oid_range[0]}_{oid_range[1]}") 110 | oid_field_name = arcpy.Describe(self.input_fc).oidFieldName 111 | where_clause = ( 112 | f"{oid_field_name} >= {oid_range[0]} " 113 | f"And {oid_field_name} <= {oid_range[1]}" 114 | ) 115 | self.logger.debug(f"Where clause: {where_clause}") 116 | arcpy.conversion.FeatureClassToFeatureClass( 117 | self.input_fc, 118 | os.path.dirname(self.out_fc), 119 | os.path.basename(self.out_fc), 120 | where_clause 121 | ) 122 | 123 | def calculate_locations(self, oid_range): 124 | """Calculate locations for a chunk of the input feature class with the designated OID range.""" 125 | self._subset_inputs(oid_range) 126 | self.logger.debug("Calculating locations...") 127 | helpers.run_gp_tool( 128 | self.logger, 129 | arcpy.na.CalculateLocations, 130 | [ 131 | self.out_fc, 132 | self.network_data_source 133 | ], { 134 | "search_tolerance": self.search_tolerance, 135 | "search_criteria": self.search_criteria, 136 | "search_query": self.search_query, 137 | "travel_mode": self.travel_mode 138 | } 139 | ) 140 | self.job_result["outputFC"] = self.out_fc 141 | self.job_result["oidRange"] = tuple(oid_range) 142 | 143 | 144 | def calculate_locations_for_chunk(chunk, calc_locs_settings): 145 | """Calculate locations for a range of OIDs in the input dataset. 146 | 147 | Args: 148 | chunk (list): OID range to calculate locations for. Specified as a list of [start range, end range], inclusive. 149 | calc_locs_settings (dict): Dictionary of kwargs for the LocationCalculator class. 150 | 151 | Returns: 152 | dict: Dictionary of job results for the chunk 153 | """ 154 | location_calculator = LocationCalculator(**calc_locs_settings) 155 | location_calculator.calculate_locations(chunk) 156 | location_calculator.teardown_logger() 157 | return location_calculator.job_result 158 | 159 | 160 | class ParallelLocationCalculator: 161 | """Calculate network locations for a large dataset by chunking the dataset and calculating in parallel.""" 162 | 163 | def __init__( # pylint: disable=too-many-locals, too-many-arguments 164 | self, logger, input_features, output_features, network_data_source, chunk_size, max_processes, 165 | travel_mode=None, search_tolerance=None, search_criteria=None, search_query=None 166 | ): 167 | """Calculate network locations for the input features in parallel. 168 | 169 | Run the Calculate Locations tool on chunks of the input dataset in parallel and and recombine the results. 170 | Refer to the Calculate Locations tool documentation for more information about the input parameters. 171 | https://pro.arcgis.com/en/pro-app/latest/tool-reference/network-analyst/calculate-locations.htm 172 | 173 | Args: 174 | logger (logging.logger): Logger class to use for messages that get written to the GP window. Set up using 175 | helpers.configure_global_logger(). 176 | input_features (str): Catalog path to input features to calculate locations for 177 | output_features (str): Catalog path to the location where the output updated feature class will be saved. 178 | Unlike in the core Calculate Locations tool, this tool generates a new feature class instead of merely 179 | adding fields to the original. A new feature class must be generated during the parallel processing, 180 | and as a result, the ObjectIDs may change, so we ask the user to specify an output feature class path 181 | instead of overwriting the original. If you need the original ObjectIDs, please calculate an additional 182 | field to track them before calling this tool. 183 | network_data_source (str): Network data source catalog path 184 | chunk_size (int): Maximum features to be processed in one chunk 185 | max_processes (int): Maximum number of parallel processes allowed 186 | travel_mode (str, optional): String-based representation of a travel mode (name or JSON) 187 | search_tolerance (str, optional): Linear Unit string representing the search distance to use when locating 188 | search_criteria (list, optional): Defines the network sources that can be used for locating 189 | search_query (list, optional): Defines queries to use per network source when locating. 190 | """ 191 | self.input_features = input_features 192 | self.output_features = output_features 193 | self.max_processes = max_processes 194 | 195 | # Set up logger that will write to the GP window 196 | self.logger = logger 197 | 198 | # Scratch folder to store intermediate outputs from the OD Cost Matrix processes 199 | unique_id = uuid.uuid4().hex 200 | self.scratch_folder = os.path.join( 201 | arcpy.env.scratchFolder, "CalcLocs_" + unique_id) # pylint: disable=no-member 202 | self.logger.info(f"Intermediate outputs will be written to {self.scratch_folder}.") 203 | os.mkdir(self.scratch_folder) 204 | 205 | # Dictionary of static input settings to send to the parallel location calculator 206 | self.calc_locs_inputs = { 207 | "input_fc": self.input_features, 208 | "network_data_source": network_data_source, 209 | "travel_mode": travel_mode, 210 | "search_tolerance": search_tolerance, 211 | "search_criteria": search_criteria, 212 | "search_query": search_query, 213 | "scratch_folder": self.scratch_folder 214 | } 215 | 216 | # List of intermediate output feature classes created by each process 217 | self.temp_out_fcs = {} 218 | 219 | # Construct OID ranges for the input data chunks 220 | self.ranges = helpers.get_oid_ranges_for_input(self.input_features, chunk_size) 221 | 222 | def calc_locs_in_parallel(self): 223 | """Calculate locations in parallel.""" 224 | # Calculate locations in parallel 225 | job_results = helpers.run_parallel_processes( 226 | self.logger, calculate_locations_for_chunk, [self.calc_locs_inputs], self.ranges, 227 | len(self.ranges), self.max_processes, 228 | "Calculating locations", "Calculate Locations" 229 | ) 230 | for result in job_results: 231 | # Parse the results dictionary and store components for post-processing. 232 | # Store the ranges as dictionary keys to facilitate sorting. 233 | self.temp_out_fcs[result["oidRange"]] = result["outputFC"] 234 | 235 | # Rejoin the chunked feature classes into one. 236 | self.logger.info("Rejoining chunked data...") 237 | self._rejoin_chunked_output() 238 | 239 | # Clean up 240 | # Delete the job folders if the job succeeded 241 | if DELETE_INTERMEDIATE_OUTPUTS: 242 | self.logger.info("Deleting intermediate outputs...") 243 | try: 244 | shutil.rmtree(self.scratch_folder, ignore_errors=True) 245 | except Exception: # pylint: disable=broad-except 246 | # If deletion doesn't work, just throw a warning and move on. This does not need to kill the tool. 247 | self.logger.warning( 248 | f"Unable to delete intermediate Calculate Locations output folder {self.scratch_folder}.") 249 | 250 | self.logger.info("Finished calculating locations in parallel.") 251 | 252 | def _rejoin_chunked_output(self): 253 | """Merge the chunks into a single feature class. 254 | 255 | Create an empty final output feature class and populate it using InsertCursor, as this tends to be faster than 256 | using the Merge geoprocessing tool. 257 | """ 258 | self.logger.debug("Creating output feature class...") 259 | 260 | # Handle ridiculously huge outputs that may exceed the number of rows allowed in a 32-bit OID feature class 261 | kwargs = {} 262 | if helpers.arcgis_version >= "3.2": # 64-bit OIDs were introduced in ArcGIS Pro 3.2. 263 | num_inputs = int(arcpy.management.GetCount(self.input_features).getOutput(0)) 264 | if num_inputs > helpers.MAX_ALLOWED_FC_ROWS_32BIT: 265 | # Use a 64bit OID field in the output feature class 266 | kwargs = {"oid_type": "64_BIT"} 267 | 268 | # Create the final output feature class 269 | template_fc = self.temp_out_fcs[tuple(self.ranges[0])] 270 | desc = arcpy.Describe(template_fc) 271 | helpers.run_gp_tool( 272 | self.logger, 273 | arcpy.management.CreateFeatureclass, [ 274 | os.path.dirname(self.output_features), 275 | os.path.basename(self.output_features), 276 | "POINT", 277 | template_fc, # template feature class to transfer full schema 278 | "SAME_AS_TEMPLATE", 279 | "SAME_AS_TEMPLATE", 280 | desc.spatialReference 281 | ], 282 | kwargs 283 | ) 284 | 285 | # Insert the rows from all the individual output feature classes into the final output 286 | self.logger.debug("Inserting rows into output feature class from output chunks...") 287 | fields = ["SHAPE@"] + [f.name for f in desc.fields] 288 | with arcpy.da.InsertCursor(self.output_features, fields) as cur: # pylint: disable=no-member 289 | # Get rows from the output feature class from each chunk in the original order 290 | for chunk in self.ranges: 291 | for row in arcpy.da.SearchCursor(self.temp_out_fcs[tuple(chunk)], fields): # pylint: disable=no-member 292 | cur.insertRow(row) 293 | 294 | 295 | def launch_parallel_calc_locs(): 296 | """Read arguments passed in via subprocess and run the parallel calculate locations. 297 | 298 | This script is intended to be called via subprocess via a client module. Users should not call this script 299 | directly from the command line. 300 | 301 | We must launch this script via subprocess in order to support parallel processing from an ArcGIS Pro script tool, 302 | which cannot do parallel processing directly. 303 | """ 304 | # Create the parser 305 | parser = argparse.ArgumentParser(description=globals().get("__doc__", ""), fromfile_prefix_chars='@') 306 | 307 | # Define Arguments supported by the command line utility 308 | 309 | # --input-features parameter 310 | help_string = "The full catalog path to the input features to calculate locations for." 311 | parser.add_argument( 312 | "-if", "--input-features", action="store", dest="input_features", help=help_string, required=True) 313 | 314 | # --output-features parameter 315 | help_string = "The full catalog path to the output features." 316 | parser.add_argument( 317 | "-of", "--output-features", action="store", dest="output_features", help=help_string, required=True) 318 | 319 | # --network-data-source parameter 320 | help_string = "The full catalog path to the network dataset that will be used for calculating locations." 321 | parser.add_argument( 322 | "-n", "--network-data-source", action="store", dest="network_data_source", help=help_string, required=True) 323 | 324 | # --chunk-size parameter 325 | help_string = "Maximum number of features that can be in one chunk for parallel processing." 326 | parser.add_argument( 327 | "-ch", "--chunk-size", action="store", dest="chunk_size", type=int, help=help_string, required=True) 328 | 329 | # --max-processes parameter 330 | help_string = "Maximum number parallel processes to use for calculating locations." 331 | parser.add_argument( 332 | "-mp", "--max-processes", action="store", dest="max_processes", type=int, help=help_string, required=True) 333 | 334 | # --travel-mode parameter 335 | help_string = ( 336 | "The name or JSON string representation of the travel mode from the network data source that will be used for " 337 | "calculating locations." 338 | ) 339 | parser.add_argument("-tm", "--travel-mode", action="store", dest="travel_mode", help=help_string, required=False) 340 | 341 | # --search-tolerance parameter 342 | help_string = "Linear Unit string representing the search distance to use when locating." 343 | parser.add_argument( 344 | "-st", "--search-tolerance", action="store", dest="search_tolerance", help=help_string, required=False) 345 | 346 | # --search-criteria parameter 347 | help_string = "Defines the network sources that can be used for locating." 348 | parser.add_argument( 349 | "-sc", "--search-criteria", action="store", dest="search_criteria", help=help_string, required=False) 350 | 351 | # --search-query parameter 352 | help_string = "Defines queries to use per network source when locating." 353 | parser.add_argument( 354 | "-sq", "--search-query", action="store", dest="search_query", help=help_string, required=False) 355 | 356 | try: 357 | logger = helpers.configure_global_logger(LOG_LEVEL) 358 | 359 | # Get arguments as dictionary. 360 | args = vars(parser.parse_args()) 361 | args["logger"] = logger 362 | 363 | # Initialize a parallel location calculator class 364 | cl_calculator = ParallelLocationCalculator(**args) 365 | # Calculate network locations in parallel chunks 366 | start_time = time.time() 367 | cl_calculator.calc_locs_in_parallel() 368 | logger.info(f"Parallel Calculate Locations completed in {round((time.time() - start_time) / 60, 2)} minutes") 369 | 370 | except Exception: # pylint: disable=broad-except 371 | logger.error("Error in parallelization subprocess.") 372 | errs = traceback.format_exc().splitlines() 373 | for err in errs: 374 | logger.error(err) 375 | raise 376 | 377 | finally: 378 | helpers.teardown_logger(logger) 379 | 380 | if __name__ == "__main__": 381 | # This script should always be launched via subprocess as if it were being called from the command line. 382 | with arcpy.EnvManager(overwriteOutput=True): 383 | launch_parallel_calc_locs() 384 | -------------------------------------------------------------------------------- /rt_config.py: -------------------------------------------------------------------------------- 1 | """Defines Route solver object properties that are not specified 2 | in the tool dialog. 3 | 4 | A list of Route solver properties is documented here: 5 | https://pro.arcgis.com/en/pro-app/latest/arcpy/network-analyst/route.htm 6 | 7 | You can include any of them in the dictionary in this file, and the tool will 8 | use them. However, travelMode, timeUnits, distanceUnits, and timeOfDay 9 | will be ignored because they are specified in the tool dialog. 10 | 11 | Copyright 2022 Esri 12 | Licensed under the Apache License, Version 2.0 (the "License"); 13 | you may not use this file except in compliance with the License. 14 | You may obtain a copy of the License at 15 | http://www.apache.org/licenses/LICENSE-2.0 16 | Unless required by applicable law or agreed to in writing, software 17 | distributed under the License is distributed on an "AS IS" BASIS, 18 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 19 | See the License for the specific language governing permissions and 20 | limitations under the License. 21 | """ 22 | import arcpy 23 | 24 | # These properties are set by the tool dialog or can be specified as command line arguments. Do not set the values for 25 | # these properties in the RT_PROPS dictionary below because they will be ignored. 26 | RT_PROPS_SET_BY_TOOL = ["travelMode", "timeUnits", "distanceUnits", "timeOfDay"] 27 | 28 | # You can customize these properties to your needs, and the parallel Route calculations will use them. 29 | RT_PROPS = { 30 | 'accumulateAttributeNames': [], 31 | 'allowSaveLayerFile': False, 32 | 'allowSaveRouteData': False, 33 | 'directionsDistanceUnits': arcpy.nax.DistanceUnits.Kilometers, 34 | 'directionsLanguage': "en", 35 | 'directionsStyle': arcpy.nax.DirectionsStyle.Desktop, 36 | 'findBestSequence': False, 37 | 'ignoreInvalidLocations': True, 38 | 'overrides': "", 39 | 'preserveFirstStop': False, 40 | 'preserveLastStop': False, 41 | 'returnDirections': False, 42 | 'returnRouteEdges': True, 43 | 'returnRouteJunctions': False, 44 | 'returnRouteTurns': False, 45 | 'returnToStart': False, 46 | 'routeShapeType': arcpy.nax.RouteShapeType.TrueShapeWithMeasures, 47 | # 'searchQuery': "", # This parameter is very network specific. Only uncomment if you are using it. 48 | 'searchTolerance': 5000, 49 | 'searchToleranceUnits': arcpy.nax.DistanceUnits.Meters, 50 | 'timeZone': arcpy.nax.TimeZoneUsage.LocalTimeAtLocations, 51 | 'timeZoneForTimeWindows': arcpy.nax.TimeZoneUsage.LocalTimeAtLocations, 52 | } 53 | -------------------------------------------------------------------------------- /unittests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/large-network-analysis-tools/b8503c5d606260f53acad6fd3bff612c242e2da4/unittests/__init__.py -------------------------------------------------------------------------------- /unittests/input_data_helper.py: -------------------------------------------------------------------------------- 1 | """Helper for unit tests to create required inputs. 2 | 3 | Copyright 2024 Esri 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | """ 14 | import os 15 | import csv 16 | import arcpy 17 | 18 | 19 | def get_tract_centroids_with_store_id_fc(sf_gdb): 20 | """Create the TractCentroids_wStoreID feature class in the SanFrancisco.gdb/Analysis for use in unit tests.""" 21 | new_fc = os.path.join(sf_gdb, "Analysis", "TractCentroids_wStoreID") 22 | if arcpy.Exists(new_fc): 23 | # The feature class exists already, so there's no need to do anything. 24 | return new_fc 25 | # Copy the tutorial dataset's TractCentroids feature class to the new feature class 26 | print(f"Creating {new_fc} for test input...") 27 | orig_fc = os.path.join(sf_gdb, "Analysis", "TractCentroids") 28 | if not arcpy.Exists(orig_fc): 29 | raise ValueError(f"{orig_fc} is missing.") 30 | arcpy.management.Copy(orig_fc, new_fc) 31 | # Add and populate the StoreID field 32 | # Also add a pre-populated CurbApproach field to test field transfer 33 | arcpy.management.AddField(new_fc, "StoreID", "TEXT", field_length=8) 34 | arcpy.management.AddField(new_fc, "CurbApproach", "SHORT") 35 | store_ids = [ # Pre-assigned store IDs to add to TractCentroids 36 | 'Store_6', 'Store_6', 'Store_11', 'Store_11', 'Store_11', 'BadStore', 'Store_11', 'Store_11', 'Store_11', '', 37 | 'Store_11', 'Store_11', 'Store_6', 'Store_11', 'Store_11', 'Store_11', 'Store_11', 'Store_1', 'Store_7', 38 | 'Store_1', 'Store_1', 'Store_2', 'Store_2', 'Store_2', 'Store_1', 'Store_2', 'Store_7', 'Store_1', 'Store_2', 39 | 'Store_2', 'Store_7', 'Store_7', 'Store_7', 'Store_4', 'Store_4', 'Store_3', 'Store_3', 'Store_3', 'Store_3', 40 | 'Store_19', 'Store_14', 'Store_19', 'Store_19', 'Store_14', 'Store_19', 'Store_16', 'Store_14', 'Store_14', 41 | 'Store_14', 'Store_14', 'Store_14', 'Store_14', 'Store_14', 'Store_14', 'Store_14', 'Store_14', 'Store_14', 42 | 'Store_14', 'Store_7', 'Store_7', 'Store_7', 'Store_7', 'Store_7', 'Store_7', 'Store_7', 'Store_7', 'Store_12', 43 | 'Store_12', 'Store_12', 'Store_12', 'Store_12', 'Store_12', 'Store_12', 'Store_7', 'Store_12', 'Store_12', 44 | 'Store_12', 'Store_12', 'Store_12', 'Store_12', 'Store_12', 'Store_12', 'Store_12', 'Store_4', 'Store_12', 45 | 'Store_4', 'Store_4', 'Store_4', 'Store_4', 'Store_4', 'Store_4', 'Store_13', 'Store_13', 'Store_13', 46 | 'Store_13', 'Store_5', 'Store_13', 'Store_13', 'Store_5', 'Store_5', 'Store_12', 'Store_12', 'Store_14', 47 | 'Store_14', 'Store_12', 'Store_12', 'Store_14', 'Store_14', 'Store_14', 'Store_14', 'Store_14', 'Store_12', 48 | 'Store_14', 'Store_12', 'Store_14', 'Store_14', 'Store_14', None, 'Store_12', 'Store_14', 'Store_14', 49 | 'Store_14', 'Store_14', 'Store_3', 'Store_2', 'Store_3', 'Store_3', 'Store_3', 'Store_3', 'Store_4', 'Store_3', 50 | 'Store_2', 'Store_3', 'Store_16', 'Store_3', 'Store_3', 'Store_3', 'Store_3', 'Store_18', 'Store_16', 51 | 'Store_16', 'Store_16', 'Store_15', 'Store_16', 'Store_16', 'Store_14', 'Store_16', 'Store_3', 'Store_3', 52 | 'Store_13', 'Store_3', 'Store_16', 'Store_16', 'Store_16', 'Store_16', 'Store_14', 'Store_14', 'Store_16', 53 | 'Store_16', 'Store_16', 'Store_16', 'Store_16', 'Store_17', 'Store_15', 'Store_17', 'Store_17', 'Store_17', 54 | 'Store_17', 'Store_15', 'Store_15', 'Store_15', 'Store_3', 'Store_15', 'Store_15', 'Store_15', 'Store_15', 55 | 'Store_15', 'Store_15', 'Store_15', 'Store_15', 'Store_15', 'Store_15', 'Store_16', 'Store_19', 'Store_19', 56 | 'Store_19', 'Store_15', 'Store_19', 'Store_15', 'Store_16', 'Store_19', 'Store_19', 'Store_19', 'Store_18', 57 | 'Store_18', 'Store_15', 'Store_18', 'Store_18', 'Store_18', 'Store_15', 'Store_18', 'Store_17', 'Store_18', 58 | 'Store_15', 'Store_15', 'Store_16', 'Store_19', 'Store_15'] 59 | with arcpy.da.UpdateCursor(new_fc, ["StoreID", "CurbApproach"]) as cur: # pylint: disable=no-member 60 | idx = 0 61 | for _ in cur: 62 | cur.updateRow([store_ids[idx], 2]) 63 | idx += 1 64 | return new_fc 65 | 66 | 67 | def get_tract_centroids_with_cutoff(sf_gdb): 68 | """Create the TractCentroids_Cutoff feature class in the SanFrancisco.gdb/Analysis for use in unit tests.""" 69 | new_fc = os.path.join(sf_gdb, "Analysis", "TractCentroids_Cutoff") 70 | if arcpy.Exists(new_fc): 71 | # The feature class exists already, so there's no need to do anything. 72 | return new_fc 73 | # Copy the tutorial dataset's TractCentroids feature class to the new feature class 74 | print(f"Creating {new_fc} for test input...") 75 | orig_fc = os.path.join(sf_gdb, "Analysis", "TractCentroids") 76 | if not arcpy.Exists(orig_fc): 77 | raise ValueError(f"{orig_fc} is missing.") 78 | arcpy.management.Copy(orig_fc, new_fc) 79 | # Add and populate the Cutoff field 80 | arcpy.management.AddField(new_fc, "Cutoff", "DOUBLE") 81 | with arcpy.da.UpdateCursor(new_fc, ["NAME", "Cutoff"]) as cur: # pylint: disable=no-member 82 | # Give 060816029.00 a cutoff value of 15 and leave the rest null 83 | for row in cur: 84 | if row[0] == "060816029.00": 85 | cur.updateRow([row[0], 15]) 86 | return new_fc 87 | 88 | 89 | def get_stores_with_dest_count(sf_gdb): 90 | """Create the Stores_DestCount feature class in the SanFrancisco.gdb/Analysis for use in unit tests.""" 91 | new_fc = os.path.join(sf_gdb, "Analysis", "Stores_DestCount") 92 | if arcpy.Exists(new_fc): 93 | # The feature class exists already, so there's no need to do anything. 94 | return new_fc 95 | # Copy the tutorial dataset's Stores feature class to the new feature class 96 | print(f"Creating {new_fc} for test input...") 97 | orig_fc = os.path.join(sf_gdb, "Analysis", "Stores") 98 | if not arcpy.Exists(orig_fc): 99 | raise ValueError(f"{orig_fc} is missing.") 100 | arcpy.management.Copy(orig_fc, new_fc) 101 | # Add and populate the Cutoff field 102 | arcpy.management.AddField(new_fc, "TargetDestinationCount", "LONG") 103 | with arcpy.da.UpdateCursor(new_fc, ["NAME", "TargetDestinationCount"]) as cur: # pylint: disable=no-member 104 | # Give Store_1 a TargetDestinationCount of 3 and Store_2 a TargetDestinationCount of 2 105 | # and leave the rest as null 106 | for row in cur: 107 | if row[0] == "Store_1": 108 | cur.updateRow([row[0], 3]) 109 | if row[0] == "Store_2": 110 | cur.updateRow([row[0], 2]) 111 | return new_fc 112 | 113 | 114 | def get_od_pair_csv(input_data_folder): 115 | """Create the od_pairs.csv input file in the input data folder for use in unit testing.""" 116 | od_pair_file = os.path.join(input_data_folder, "od_pairs.csv") 117 | if os.path.exists(od_pair_file): 118 | # The OD pair file already exists, so no need to do anything. 119 | return od_pair_file 120 | print(f"Creating {od_pair_file} for test input...") 121 | od_pairs = [ 122 | ["06075011400", "Store_13"], 123 | ["06075011400", "Store_19"], 124 | ["06075011400", "Store_11"], 125 | ["06075021800", "Store_9"], 126 | ["06075021800", "Store_12"], 127 | ["06075013000", "Store_3"], 128 | ["06075013000", "Store_10"], 129 | ["06075013000", "Store_1"], 130 | ["06075013000", "Store_8"], 131 | ["06075013000", "Store_12"], 132 | ["06081602500", "Store_12"], 133 | ["06081602500", "Store_25"], 134 | ["06081602500", "Store_9"], 135 | ["06081602500", "Store_17"], 136 | ["06081602500", "Store_21"], 137 | ["06075030400", "Store_7"], 138 | ["06075030400", "Store_5"], 139 | ["06075030400", "Store_21"], 140 | ["06075030400", "Store_19"], 141 | ["06075030400", "Store_23"], 142 | ["06075045200", "Store_1"], 143 | ["06075045200", "Store_5"], 144 | ["06075045200", "Store_6"], 145 | ["06075012600", "Store_19"], 146 | ["06075012600", "Store_5"], 147 | ["06075012600", "Store_23"], 148 | ["06075012600", "Store_15"], 149 | ["06075060700", "Store_19"], 150 | ["06081601400", "Store_7"], 151 | ["06081601400", "Store_15"], 152 | ["06081601400", "Store_2"], 153 | ["06075023001", "Store_22"], 154 | ["06075032800", "Store_25"], 155 | ["06081601800", "Store_13"], 156 | ["06075013100", "Store_25"], 157 | ["06075013100", "Store_9"], 158 | ["06075013100", "Store_23"], 159 | ["06081600600", "Store_22"], 160 | ["06081600600", "Store_12"], 161 | ["06081600600", "Store_1"], 162 | ["06081600600", "Store_21"], 163 | ["06075012000", "Store_22"], 164 | ["06075031400", "Store_20"], 165 | ["06075031400", "Store_24"], 166 | ["06081601000", "Store_2"], 167 | ["06075026004", "Store_16"], 168 | ["06075026004", "Store_7"], 169 | ["06075020300", "Store_17"], 170 | ["06075020300", "Store_13"], 171 | ["06075060502", "Store_18"], 172 | ["06075011000", "Store_21"], 173 | ["06075011000", "Store_19"], 174 | ["06075011000", "Store_12"], 175 | ["06075011000", "Store_22"], 176 | ["06075026404", "Store_17"], 177 | ["06075026404", "Store_9"], 178 | ["06081601601", "Store_9"], 179 | ["06075021500", "Store_22"], 180 | ["06075021500", "Store_9"], 181 | ["06075026302", "Store_21"], 182 | ["06075026302", "Store_15"], 183 | ["06075026302", "Store_23"], 184 | ["06075026302", "Store_24"] 185 | ] 186 | with open(od_pair_file, "w", newline='', encoding="utf-8") as f: 187 | w = csv.writer(f) 188 | w.writerows(od_pairs) 189 | return od_pair_file 190 | 191 | 192 | def get_od_pairs_fgdb_table(sf_gdb): 193 | """Create the ODPairs fgdb input table in SanFrancisco.gdb for use in unit testing.""" 194 | od_pair_table = os.path.join(sf_gdb, "ODPairs") 195 | if arcpy.Exists(od_pair_table): 196 | # The OD pair table already exists, so no need to do anything. 197 | return od_pair_table 198 | input_data_folder = os.path.dirname(sf_gdb) 199 | od_pair_csv = get_od_pair_csv(input_data_folder) 200 | print(f"Creating {od_pair_table} for test input...") 201 | arcpy.management.CreateTable(sf_gdb, "ODPairs") 202 | arcpy.management.AddFields( 203 | od_pair_table, 204 | [["OriginID", "TEXT", "OriginID", 25], ["DestinationID", "TEXT", "DestinationID", 45]] 205 | ) 206 | with arcpy.da.InsertCursor(od_pair_table, ["OriginID", "DestinationID"]) as cur: # pylint: disable=no-member 207 | with open(od_pair_csv, "r", encoding="utf-8") as f: 208 | for row in csv.reader(f): 209 | cur.insertRow(row) 210 | 211 | return od_pair_table 212 | -------------------------------------------------------------------------------- /unittests/portal_credentials.py: -------------------------------------------------------------------------------- 1 | """Portal credentials for unit tests.""" 2 | # Do not check in your actual credentials into the repo. 3 | PORTAL_URL = "" 4 | PORTAL_USERNAME = "" 5 | PORTAL_PASSWORD = "" 6 | PORTAL_TRAVEL_MODE = "" 7 | -------------------------------------------------------------------------------- /unittests/test_ParallelCalculateLocations_tool.py: -------------------------------------------------------------------------------- 1 | """Unit tests for the ParallelCalculateLocations script tool. The test cases focus 2 | on making sure the tool parameters work correctly. 3 | 4 | Copyright 2023 Esri 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | """ 15 | # pylint: disable=import-error, invalid-name 16 | 17 | import os 18 | import datetime 19 | import unittest 20 | import arcpy 21 | 22 | CWD = os.path.dirname(os.path.abspath(__file__)) 23 | 24 | 25 | class TestSolveLargeODCostMatrixTool(unittest.TestCase): 26 | """Test cases for the SolveLargeODCostMatrix script tool.""" 27 | 28 | @classmethod 29 | def setUpClass(self): # pylint: disable=bad-classmethod-argument 30 | self.maxDiff = None 31 | 32 | tbx_path = os.path.join(os.path.dirname(CWD), "LargeNetworkAnalysisTools.pyt") 33 | arcpy.ImportToolbox(tbx_path) 34 | 35 | self.input_data_folder = os.path.join(CWD, "TestInput") 36 | sf_gdb = os.path.join(self.input_data_folder, "SanFrancisco.gdb") 37 | self.test_points = os.path.join(sf_gdb, "Analysis", "TractCentroids") 38 | self.local_nd = os.path.join(sf_gdb, "Transportation", "Streets_ND") 39 | tms = arcpy.nax.GetTravelModes(self.local_nd) 40 | self.local_tm_time = tms["Driving Time"] 41 | 42 | # Create a unique output directory and gdb for this test 43 | self.scratch_folder = os.path.join( 44 | CWD, "TestOutput", 45 | "Output_ParallelCalcLocs_Tool_" + datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) 46 | os.makedirs(self.scratch_folder) 47 | self.output_gdb = os.path.join(self.scratch_folder, "outputs.gdb") 48 | arcpy.management.CreateFileGDB(os.path.dirname(self.output_gdb), os.path.basename(self.output_gdb)) 49 | 50 | def test_tool_defaults(self): 51 | """Test that the tool runs and works with only required parameters.""" 52 | # The input feature class should not be overwritten by this tool, but copy it first just in case. 53 | fc_to_precalculate = os.path.join(self.output_gdb, "PrecalcFC_InputD") 54 | arcpy.management.Copy(self.test_points, fc_to_precalculate) 55 | output_fc = os.path.join(self.output_gdb, "PrecalcFC_OutputD") 56 | arcpy.LargeNetworkAnalysisTools.ParallelCalculateLocations( # pylint: disable=no-member 57 | fc_to_precalculate, 58 | output_fc, 59 | self.local_nd, 60 | ) 61 | self.assertTrue(arcpy.Exists(output_fc), "Output feature class does not exist") 62 | 63 | def test_tool_nondefaults(self): 64 | """Test that the tool runs and works with all input parameters.""" 65 | # The input feature class should not be overwritten by this tool, but copy it first just in case. 66 | fc_to_precalculate = os.path.join(self.output_gdb, "PrecalcFC_InputND") 67 | arcpy.management.Copy(self.test_points, fc_to_precalculate) 68 | output_fc = os.path.join(self.output_gdb, "PrecalcFC_OutputND") 69 | arcpy.LargeNetworkAnalysisTools.ParallelCalculateLocations( # pylint: disable=no-member 70 | fc_to_precalculate, 71 | output_fc, 72 | self.local_nd, 73 | 30, 74 | 4, 75 | self.local_tm_time, 76 | "5000 Meters", 77 | ["Streets"], 78 | [["Streets", "ObjectID <> 1"]] 79 | ) 80 | self.assertTrue(arcpy.Exists(output_fc), "Output feature class does not exist") 81 | 82 | def test_error_invalid_query_sources(self): 83 | """Test for correct error when an invalid network source name is specified in the search query.""" 84 | with self.assertRaises(arcpy.ExecuteError) as ex: 85 | arcpy.LargeNetworkAnalysisTools.ParallelCalculateLocations( # pylint: disable=no-member 86 | self.test_points, 87 | os.path.join(self.output_gdb, "Junk"), 88 | self.local_nd, 89 | 30, 90 | 4, 91 | self.local_tm_time, 92 | "5000 Meters", 93 | ["Streets"], 94 | [["Streets", "ObjectID <> 1"], ["BadSourceName", "ObjectID <> 2"]] # Invalid source name 95 | ) 96 | actual_messages = str(ex.exception) 97 | # Check for expected GP message 98 | self.assertIn("30254", actual_messages) 99 | 100 | def test_error_duplicate_query_sources(self): 101 | """Test for correct error when a network source is specified more than once in the search query.""" 102 | with self.assertRaises(arcpy.ExecuteError) as ex: 103 | arcpy.LargeNetworkAnalysisTools.ParallelCalculateLocations( # pylint: disable=no-member 104 | self.test_points, 105 | os.path.join(self.output_gdb, "Junk"), 106 | self.local_nd, 107 | 30, 108 | 4, 109 | self.local_tm_time, 110 | "5000 Meters", 111 | ["Streets"], 112 | [["Streets", "ObjectID <> 1"], ["Streets", "ObjectID <> 2"]] # Duplicate query 113 | ) 114 | actual_messages = str(ex.exception) 115 | # Check for expected GP message 116 | self.assertIn("30255", actual_messages) 117 | 118 | def test_error_invalid_query(self): 119 | """Test for correct error when an invalid search query is specified.""" 120 | with self.assertRaises(arcpy.ExecuteError) as ex: 121 | arcpy.LargeNetworkAnalysisTools.ParallelCalculateLocations( # pylint: disable=no-member 122 | self.test_points, 123 | os.path.join(self.output_gdb, "Junk"), 124 | self.local_nd, 125 | 30, 126 | 4, 127 | self.local_tm_time, 128 | "5000 Meters", 129 | ["Streets"], 130 | [["Streets", "NAME = 1"]] # Bad query syntax 131 | ) 132 | actual_messages = str(ex.exception) 133 | # Check for expected validation message 134 | self.assertIn("An invalid SQL statement was used", actual_messages) 135 | 136 | 137 | if __name__ == '__main__': 138 | unittest.main() 139 | -------------------------------------------------------------------------------- /unittests/test_SolveLargeAnalysisWithKnownPairs_tool.py: -------------------------------------------------------------------------------- 1 | """Unit tests for the SolveLargeAnalysisWithKnownPairs script tool. 2 | 3 | The test cases focus on making sure the tool parameters work correctly. 4 | 5 | Copyright 2023 Esri 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | # pylint: disable=import-error, invalid-name 17 | 18 | import sys 19 | import os 20 | import datetime 21 | import unittest 22 | from copy import deepcopy 23 | import arcpy 24 | import portal_credentials 25 | import input_data_helper 26 | 27 | CWD = os.path.dirname(os.path.abspath(__file__)) 28 | sys.path.append(os.path.dirname(CWD)) 29 | import helpers # noqa: E402, pylint: disable=wrong-import-position 30 | 31 | 32 | class TestSolveLargeAnalysisWithKnownPairsTool(unittest.TestCase): 33 | """Test cases for the SolveLargeAnalysisWithKnownPairs script tool.""" 34 | 35 | @classmethod 36 | def setUpClass(self): # pylint: disable=bad-classmethod-argument 37 | """Set up shared inputs for tests.""" 38 | self.maxDiff = None 39 | 40 | tbx_path = os.path.join(os.path.dirname(CWD), "LargeNetworkAnalysisTools.pyt") 41 | arcpy.ImportToolbox(tbx_path) 42 | 43 | self.input_data_folder = os.path.join(CWD, "TestInput") 44 | sf_gdb = os.path.join(self.input_data_folder, "SanFrancisco.gdb") 45 | self.origins = input_data_helper.get_tract_centroids_with_store_id_fc(sf_gdb) 46 | self.destinations = os.path.join(sf_gdb, "Analysis", "Stores") 47 | self.od_pairs_table = input_data_helper.get_od_pairs_fgdb_table(sf_gdb) 48 | self.local_nd = os.path.join(sf_gdb, "Transportation", "Streets_ND") 49 | tms = arcpy.nax.GetTravelModes(self.local_nd) 50 | self.local_tm_name = "Driving Time" 51 | self.local_tm_time = tms[self.local_tm_name] 52 | self.portal_nd = portal_credentials.PORTAL_URL # Must be arcgis.com for test to work 53 | self.portal_tm = portal_credentials.PORTAL_TRAVEL_MODE 54 | 55 | arcpy.SignInToPortal(self.portal_nd, portal_credentials.PORTAL_USERNAME, portal_credentials.PORTAL_PASSWORD) 56 | 57 | # Create a unique output directory and gdb for this test 58 | self.scratch_folder = os.path.join( 59 | CWD, "TestOutput", 60 | "Output_SolveLargeAnalysisWithKnownPairs_Tool_" + datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) 61 | os.makedirs(self.scratch_folder) 62 | self.output_gdb = os.path.join(self.scratch_folder, "outputs.gdb") 63 | arcpy.management.CreateFileGDB(os.path.dirname(self.output_gdb), os.path.basename(self.output_gdb)) 64 | 65 | # Copy some data to the output gdb to serve as barriers. Do not use tutorial data directly as input because 66 | # the tests will write network location fields to it, and we don't want to modify the user's original data. 67 | self.barriers = os.path.join(self.output_gdb, "Barriers") 68 | arcpy.management.Copy(os.path.join(sf_gdb, "Analysis", "CentralDepots"), self.barriers) 69 | 70 | def test_run_tool_one_to_one(self): 71 | """Test that the tool runs with all inputs for the one-to-one pair type.""" 72 | # Run tool 73 | out_routes = os.path.join(self.output_gdb, "OutRoutesLocal_OneToOne") 74 | arcpy.LargeNetworkAnalysisTools.SolveLargeAnalysisWithKnownPairs( # pylint: disable=no-member 75 | self.origins, 76 | "ObjectID", 77 | self.destinations, 78 | "NAME", 79 | helpers.PAIR_TYPES[0], 80 | "StoreID", 81 | None, 82 | None, 83 | None, 84 | self.local_nd, 85 | self.local_tm_time, 86 | "Minutes", 87 | "Miles", 88 | 50, # chunk size 89 | 4, # max processes 90 | out_routes, 91 | datetime.datetime(2022, 3, 29, 16, 45, 0), # time of day 92 | self.barriers, # barriers 93 | True, # precalculate network locations 94 | True, # Sort origins 95 | False # Reverse direction of travel 96 | ) 97 | # Check results 98 | self.assertTrue(arcpy.Exists(out_routes)) 99 | 100 | def test_run_tool_many_to_many(self): 101 | """Test that the tool runs with all inputs for the many-to-many pair type.""" 102 | # Run tool 103 | out_routes = os.path.join(self.output_gdb, "OutRoutesLocal_ManyToMany") 104 | arcpy.LargeNetworkAnalysisTools.SolveLargeAnalysisWithKnownPairs( # pylint: disable=no-member 105 | self.origins, 106 | "ID", 107 | self.destinations, 108 | "NAME", 109 | helpers.PAIR_TYPES[1], 110 | None, 111 | self.od_pairs_table, 112 | "OriginID", 113 | "DestinationID", 114 | self.local_nd, 115 | self.local_tm_time, 116 | "Minutes", 117 | "Miles", 118 | 20, # chunk size 119 | 4, # max processes 120 | out_routes, 121 | datetime.datetime(2022, 3, 29, 16, 45, 0), # time of day 122 | None, # barriers 123 | True, # precalculate network locations 124 | False, # Sort origins 125 | False # Reverse direction of travel 126 | ) 127 | # Check results 128 | self.assertTrue(arcpy.Exists(out_routes)) 129 | 130 | def test_run_tool_service(self): 131 | """Test that the tool runs with a service as a network data source. Use reverse order.""" 132 | out_routes = os.path.join(self.output_gdb, "OutRoutesService") 133 | arcpy.LargeNetworkAnalysisTools.SolveLargeAnalysisWithKnownPairs( # pylint: disable=no-member 134 | self.origins, 135 | "ID", 136 | self.destinations, 137 | "NAME", 138 | helpers.PAIR_TYPES[0], 139 | "StoreID", 140 | None, 141 | None, 142 | None, 143 | self.portal_nd, 144 | self.portal_tm, 145 | "Minutes", 146 | "Miles", 147 | 50, # chunk size 148 | 4, # max processes 149 | out_routes, 150 | None, # time of day 151 | None, # barriers 152 | False, # precalculate network locations 153 | False, # Sort origins 154 | True # Reverse direction of travel 155 | ) 156 | # Check results 157 | self.assertTrue(arcpy.Exists(out_routes)) 158 | 159 | def test_error_max_processes(self): 160 | """Test for correct errors max processes parameter. 161 | 162 | This test primarily tests the shared cap_max_processes() function, so this is sufficient for testing all tools. 163 | """ 164 | inputs = { 165 | "Origins": self.origins, 166 | "Origin_Unique_ID_Field": "ID", 167 | "Destinations": self.destinations, 168 | "Destination_Unique_ID_Field": "NAME", 169 | "OD_Pair_Type": helpers.PAIR_TYPES[0], 170 | "Assigned_Destination_Field": "StoreID", 171 | "Network_Data_Source": self.local_nd, 172 | "Travel_Mode": self.local_tm_name, 173 | "Output_Routes": os.path.join(self.output_gdb, "Junk") 174 | } 175 | 176 | for bad_max_processes in [-2, 0]: 177 | with self.subTest(Max_Processes=bad_max_processes): 178 | with self.assertRaises(arcpy.ExecuteError) as ex: 179 | bad_inputs = deepcopy(inputs) 180 | bad_inputs["Max_Processes"] = bad_max_processes 181 | arcpy.LargeNetworkAnalysisTools.SolveLargeAnalysisWithKnownPairs( # pylint: disable=no-member 182 | **bad_inputs 183 | ) 184 | expected_message = "The maximum number of parallel processes must be positive." 185 | actual_messages = str(ex.exception).strip().split("\n") 186 | self.assertIn(expected_message, actual_messages) 187 | 188 | bad_max_processes = 5000 189 | with self.subTest(Max_Processes=bad_max_processes): 190 | with self.assertRaises(arcpy.ExecuteError) as ex: 191 | bad_inputs = deepcopy(inputs) 192 | bad_inputs["Max_Processes"] = bad_max_processes 193 | arcpy.LargeNetworkAnalysisTools.SolveLargeAnalysisWithKnownPairs( # pylint: disable=no-member 194 | **bad_inputs 195 | ) 196 | expected_message = ( 197 | f"The maximum number of parallel processes cannot exceed {helpers.MAX_ALLOWED_MAX_PROCESSES:} due " 198 | "to limitations imposed by Python's concurrent.futures module." 199 | ) 200 | actual_messages = str(ex.exception).strip().split("\n") 201 | self.assertIn(expected_message, actual_messages) 202 | 203 | bad_max_processes = 5 204 | with self.subTest(Max_Processes=bad_max_processes, Network_Data_Source=self.portal_nd): 205 | with self.assertRaises(arcpy.ExecuteError) as ex: 206 | bad_inputs = deepcopy(inputs) 207 | bad_inputs["Max_Processes"] = bad_max_processes 208 | bad_inputs["Network_Data_Source"] = self.portal_nd 209 | bad_inputs["Travel_Mode"] = self.portal_tm 210 | arcpy.LargeNetworkAnalysisTools.SolveLargeAnalysisWithKnownPairs( # pylint: disable=no-member 211 | **bad_inputs 212 | ) 213 | expected_message = ( 214 | f"The maximum number of parallel processes cannot exceed {helpers.MAX_AGOL_PROCESSES} when the " 215 | "ArcGIS Online service is used as the network data source." 216 | ) 217 | actual_messages = str(ex.exception).strip().split("\n") 218 | self.assertIn(expected_message, actual_messages) 219 | 220 | 221 | if __name__ == '__main__': 222 | unittest.main() 223 | -------------------------------------------------------------------------------- /unittests/test_SolveLargeODCostMatrix_tool.py: -------------------------------------------------------------------------------- 1 | """Unit tests for the SolveLargeODCostMatrix script tool. The test cases focus 2 | on making sure the tool parameters work correctly. 3 | 4 | Copyright 2024 Esri 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | """ 15 | # pylint: disable=import-error, invalid-name 16 | 17 | import sys 18 | import os 19 | import datetime 20 | from glob import glob 21 | import unittest 22 | import arcpy 23 | import portal_credentials 24 | import input_data_helper 25 | 26 | CWD = os.path.dirname(os.path.abspath(__file__)) 27 | sys.path.append(os.path.dirname(CWD)) 28 | import helpers # noqa: E402, pylint: disable=wrong-import-position 29 | 30 | 31 | class TestSolveLargeODCostMatrixTool(unittest.TestCase): 32 | """Test cases for the SolveLargeODCostMatrix script tool.""" 33 | 34 | @classmethod 35 | def setUpClass(self): # pylint: disable=bad-classmethod-argument 36 | self.maxDiff = None 37 | 38 | tbx_path = os.path.join(os.path.dirname(CWD), "LargeNetworkAnalysisTools.pyt") 39 | arcpy.ImportToolbox(tbx_path) 40 | 41 | self.input_data_folder = os.path.join(CWD, "TestInput") 42 | self.sf_gdb = os.path.join(self.input_data_folder, "SanFrancisco.gdb") 43 | self.origins = os.path.join(self.sf_gdb, "Analysis", "TractCentroids") 44 | self.destinations = os.path.join(self.sf_gdb, "Analysis", "Hospitals") 45 | self.local_nd = os.path.join(self.sf_gdb, "Transportation", "Streets_ND") 46 | tms = arcpy.nax.GetTravelModes(self.local_nd) 47 | self.local_tm_time = tms["Driving Time"] 48 | self.local_tm_dist = tms["Driving Distance"] 49 | self.portal_nd = portal_credentials.PORTAL_URL # Must be arcgis.com for test to work 50 | self.portal_tm = portal_credentials.PORTAL_TRAVEL_MODE 51 | 52 | arcpy.SignInToPortal(self.portal_nd, portal_credentials.PORTAL_USERNAME, portal_credentials.PORTAL_PASSWORD) 53 | 54 | # Create a unique output directory and gdb for this test 55 | self.scratch_folder = os.path.join( 56 | CWD, "TestOutput", 57 | "Output_SolveLargeODCostMatrix_Tool_" + datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) 58 | os.makedirs(self.scratch_folder) 59 | self.output_gdb = os.path.join(self.scratch_folder, "outputs.gdb") 60 | arcpy.management.CreateFileGDB(os.path.dirname(self.output_gdb), os.path.basename(self.output_gdb)) 61 | 62 | def test_run_tool_time_units_feature_class(self): 63 | """Test that the tool runs with a time-based travel mode. Write output to a feature class.""" 64 | # Run tool 65 | out_od_lines = os.path.join(self.output_gdb, "Time_ODLines") 66 | out_origins = os.path.join(self.output_gdb, "Time_Origins") 67 | out_destinations = os.path.join(self.output_gdb, "Time_Destinations") 68 | arcpy.LargeNetworkAnalysisTools.SolveLargeODCostMatrix( # pylint: disable=no-member 69 | self.origins, 70 | self.destinations, 71 | self.local_nd, 72 | self.local_tm_time, 73 | "Minutes", 74 | "Miles", 75 | 50, # chunk size 76 | 4, # max processes 77 | out_origins, 78 | out_destinations, 79 | "Feature class", 80 | out_od_lines, 81 | None, 82 | 15, # cutoff 83 | 1, # number of destinations 84 | datetime.datetime(2022, 3, 29, 16, 45, 0), # time of day 85 | None, # barriers 86 | True, # precalculate network locations 87 | True # Spatially sort inputs 88 | ) 89 | # Check results 90 | self.assertTrue(arcpy.Exists(out_od_lines)) 91 | self.assertTrue(arcpy.Exists(out_origins)) 92 | self.assertTrue(arcpy.Exists(out_destinations)) 93 | 94 | def test_run_tool_distance_units_csv_barriers(self): 95 | """Test that the tool runs with a distance-based travel mode. Write output to CSVs. Also use barriers.""" 96 | # Copy some data to the output gdb to serve as barriers. Do not use tutorial data directly as input because 97 | # the tests will write network location fields to it, and we don't want to modify the user's original data. 98 | barriers = os.path.join(self.output_gdb, "Barriers") 99 | arcpy.management.Copy(os.path.join(self.sf_gdb, "Analysis", "CentralDepots"), barriers) 100 | 101 | # Run tool 102 | out_origins = os.path.join(self.output_gdb, "Dist_Origins") 103 | out_destinations = os.path.join(self.output_gdb, "Dist_Destinations") 104 | out_folder = os.path.join(self.scratch_folder, "DistUnits_CSVs") 105 | arcpy.LargeNetworkAnalysisTools.SolveLargeODCostMatrix( # pylint: disable=no-member 106 | self.origins, 107 | self.destinations, 108 | self.local_nd, 109 | self.local_tm_dist, 110 | "Minutes", 111 | "Miles", 112 | 50, # chunk size 113 | 4, # max processes 114 | out_origins, 115 | out_destinations, 116 | "CSV files", 117 | None, 118 | out_folder, 119 | 5, # cutoff 120 | 2, # number of destinations 121 | None, # time of day 122 | barriers, # barriers 123 | True, # precalculate network locations 124 | True # Spatially sort inputs 125 | ) 126 | # Check results 127 | self.assertTrue(arcpy.Exists(out_origins)) 128 | self.assertTrue(arcpy.Exists(out_destinations)) 129 | csv_files = glob(os.path.join(out_folder, "*.csv")) 130 | self.assertGreater(len(csv_files), 0) 131 | 132 | @unittest.skipIf( 133 | helpers.arcgis_version < "2.9", "Arrow table output is not available in versions of Pro prior to 2.9.") 134 | def test_run_tool_time_units_arrow(self): 135 | """Test the tool with Apache Arrow outputs.""" 136 | # Run tool 137 | out_origins = os.path.join(self.output_gdb, "Arrow_Origins") 138 | out_destinations = os.path.join(self.output_gdb, "Arrow_Destinations") 139 | out_folder = os.path.join(self.scratch_folder, "ArrowOutputs") 140 | arcpy.LargeNetworkAnalysisTools.SolveLargeODCostMatrix( # pylint: disable=no-member 141 | self.origins, 142 | self.destinations, 143 | self.local_nd, 144 | self.local_tm_dist, 145 | "Minutes", 146 | "Miles", 147 | 50, # chunk size 148 | 4, # max processes 149 | out_origins, 150 | out_destinations, 151 | "Apache Arrow files", 152 | None, 153 | out_folder, 154 | 5, # cutoff 155 | 2, # number of destinations 156 | None, # time of day 157 | None, # barriers 158 | True, # precalculate network locations 159 | True # Spatially sort inputs 160 | ) 161 | # Check results 162 | self.assertTrue(arcpy.Exists(out_origins)) 163 | self.assertTrue(arcpy.Exists(out_destinations)) 164 | csv_files = glob(os.path.join(out_folder, "*.arrow")) 165 | self.assertGreater(len(csv_files), 0) 166 | 167 | def test_run_tool_service(self): 168 | """Test that the tool runs with a service as a network data source.""" 169 | out_od_lines = os.path.join(self.output_gdb, "Service_ODLines") 170 | out_origins = os.path.join(self.output_gdb, "Service_Origins") 171 | out_destinations = os.path.join(self.output_gdb, "Service_Destinations") 172 | arcpy.LargeNetworkAnalysisTools.SolveLargeODCostMatrix( # pylint: disable=no-member 173 | self.origins, 174 | self.destinations, 175 | self.portal_nd, 176 | self.portal_tm, 177 | "Minutes", 178 | "Miles", 179 | 50, # chunk size 180 | 4, # max processes 181 | out_origins, 182 | out_destinations, 183 | "Feature class", 184 | out_od_lines, 185 | None, 186 | 15, # cutoff 187 | 1, # number of destinations 188 | None, # time of day 189 | None, # barriers 190 | False, # precalculate network locations 191 | True # Spatially sort inputs 192 | ) 193 | # Check results 194 | self.assertTrue(arcpy.Exists(out_od_lines)) 195 | self.assertTrue(arcpy.Exists(out_origins)) 196 | self.assertTrue(arcpy.Exists(out_destinations)) 197 | 198 | def test_run_tool_per_origin_cutoff(self): 199 | """Test that the tool correctly uses the Cutoff field in the input origins layer.""" 200 | # Run tool 201 | origins = input_data_helper.get_tract_centroids_with_cutoff(self.sf_gdb) 202 | out_od_lines = os.path.join(self.output_gdb, "PerOriginCutoff_ODLines") 203 | out_origins = os.path.join(self.output_gdb, "PerOriginCutoff_Origins") 204 | out_destinations = os.path.join(self.output_gdb, "PerOriginCutoff_Destinations") 205 | arcpy.LargeNetworkAnalysisTools.SolveLargeODCostMatrix( # pylint: disable=no-member 206 | origins, 207 | os.path.join(self.sf_gdb, "Analysis", "FireStations"), 208 | self.local_nd, 209 | self.local_tm_time, 210 | "Minutes", 211 | "Miles", 212 | 50, # chunk size 213 | 4, # max processes 214 | out_origins, 215 | out_destinations, 216 | "Feature class", 217 | out_od_lines, 218 | None, 219 | 1, # cutoff - tiny cutoff that is overridden for one origin 220 | None, # number of destinations 221 | None, # time of day 222 | None, # barriers 223 | True, # precalculate network locations 224 | True # Spatially sort inputs 225 | ) 226 | # Check results 227 | self.assertTrue(arcpy.Exists(out_od_lines)) 228 | self.assertTrue(arcpy.Exists(out_origins)) 229 | self.assertTrue(arcpy.Exists(out_destinations)) 230 | self.assertEqual(63, int(arcpy.management.GetCount(out_od_lines).getOutput(0)), "Incorrect number of OD lines") 231 | num_dests = 0 232 | for row in arcpy.da.SearchCursor(out_od_lines, ["OriginName", "Total_Time"]): 233 | if row[0] == "060816029.00": 234 | num_dests += 1 235 | self.assertLessEqual(row[1], 15, "Travel time is out of bounds for origin with its own cutoff") 236 | else: 237 | self.assertLessEqual(row[1], 1, "Travel time is out of bounds for origin with with default cutoff") 238 | self.assertEqual(13, num_dests, "Incorrect number of destinations found for origin with its own cutoff.") 239 | 240 | def test_run_tool_per_origin_dest_count(self): 241 | """Test that the tool correctly uses the TargetDestinationCount field in the input origins layer.""" 242 | # Run tool 243 | origins = input_data_helper.get_stores_with_dest_count(self.sf_gdb) 244 | out_od_lines = os.path.join(self.output_gdb, "PerOriginDestCount_ODLines") 245 | out_origins = os.path.join(self.output_gdb, "PerOriginDestCount_Origins") 246 | out_destinations = os.path.join(self.output_gdb, "PerOriginDestCount_Destinations") 247 | arcpy.LargeNetworkAnalysisTools.SolveLargeODCostMatrix( # pylint: disable=no-member 248 | origins, 249 | self.destinations, 250 | self.local_nd, 251 | self.local_tm_time, 252 | "Minutes", 253 | "Miles", 254 | 10, # chunk size 255 | 4, # max processes 256 | out_origins, 257 | out_destinations, 258 | "Feature class", 259 | out_od_lines, 260 | None, 261 | None, # cutoff 262 | 1, # number of destinations - overridden for one origin 263 | None, # time of day 264 | None, # barriers 265 | True, # precalculate network locations 266 | True # Spatially sort inputs 267 | ) 268 | # Check results 269 | self.assertTrue(arcpy.Exists(out_od_lines)) 270 | self.assertTrue(arcpy.Exists(out_origins)) 271 | self.assertTrue(arcpy.Exists(out_destinations)) 272 | self.assertEqual(28, int(arcpy.management.GetCount(out_od_lines).getOutput(0)), "Incorrect number of OD lines") 273 | # Check Store_1, which should have 3 destinations 274 | num_rows = 0 275 | prev_time = 0 276 | for row in arcpy.da.SearchCursor(out_od_lines, [ "Total_Time", "DestinationRank"], "OriginName = 'Store_1'"): 277 | num_rows += 1 278 | self.assertEqual(num_rows, row[1], "Incorrect DestinationRank value for Store_1") 279 | self.assertGreater(row[0], prev_time, "Total_Time value for Store_1 isn't increasing") 280 | prev_time = row[0] 281 | self.assertEqual(3, num_rows, "Incorrect number of destinations found for Store_1") 282 | # Check Store_2, which should have 2 destinations 283 | num_rows = 0 284 | prev_time = 0 285 | for row in arcpy.da.SearchCursor(out_od_lines, [ "Total_Time", "DestinationRank"], "OriginName = 'Store_2'"): 286 | num_rows += 1 287 | self.assertEqual(num_rows, row[1], "Incorrect DestinationRank value for Store_2") 288 | self.assertGreater(row[0], prev_time, "Total_Time value for Store_2 isn't increasing") 289 | prev_time = row[0] 290 | self.assertEqual(2, num_rows, "Incorrect number of destinations found for Store_2") 291 | 292 | def test_error_required_output_od_lines(self): 293 | """Test for correct error when output format is Feature class and output OD Lines not specified.""" 294 | with self.assertRaises(arcpy.ExecuteError) as ex: 295 | # Run tool 296 | out_origins = os.path.join(self.output_gdb, "Err_Origins") 297 | out_destinations = os.path.join(self.output_gdb, "Err_Destinations") 298 | arcpy.LargeNetworkAnalysisTools.SolveLargeODCostMatrix( # pylint: disable=no-member 299 | self.origins, 300 | self.destinations, 301 | self.local_nd, 302 | self.local_tm_time, 303 | "Minutes", 304 | "Miles", 305 | 50, # chunk size 306 | 4, # max processes 307 | out_origins, 308 | out_destinations, 309 | "Feature class", 310 | None, 311 | "Junk", 312 | 15, # cutoff 313 | 1, # number of destinations 314 | None, # time of day 315 | None, # barriers 316 | True, # precalculate network locations 317 | True # Spatially sort inputs 318 | ) 319 | expected_messages = [ 320 | "Failed to execute. Parameters are not valid.", 321 | "ERROR 000735: Output OD Lines Feature Class: Value is required", 322 | "Failed to execute (SolveLargeODCostMatrix)." 323 | ] 324 | actual_messages = str(ex.exception).strip().split("\n") 325 | self.assertEqual(expected_messages, actual_messages) 326 | 327 | def test_error_required_output_folder(self): 328 | """Test for correct error when output format is CSV files and output folder not specified.""" 329 | with self.assertRaises(arcpy.ExecuteError) as ex: 330 | # Run tool 331 | out_origins = os.path.join(self.output_gdb, "Err_Origins") 332 | out_destinations = os.path.join(self.output_gdb, "Err_Destinations") 333 | arcpy.LargeNetworkAnalysisTools.SolveLargeODCostMatrix( # pylint: disable=no-member 334 | self.origins, 335 | self.destinations, 336 | self.local_nd, 337 | self.local_tm_time, 338 | "Minutes", 339 | "Miles", 340 | 50, # chunk size 341 | 4, # max processes 342 | out_origins, 343 | out_destinations, 344 | "CSV files", 345 | "Junk", 346 | None, 347 | 15, # cutoff 348 | 1, # number of destinations 349 | None, # time of day 350 | None, # barriers 351 | True, # precalculate network locations 352 | True # Spatially sort inputs 353 | ) 354 | expected_messages = [ 355 | "Failed to execute. Parameters are not valid.", 356 | "ERROR 000735: Output Folder: Value is required", 357 | "Failed to execute (SolveLargeODCostMatrix)." 358 | ] 359 | actual_messages = str(ex.exception).strip().split("\n") 360 | self.assertEqual(expected_messages, actual_messages) 361 | 362 | def test_error_output_folder_exists(self): 363 | """Test for correct error when output folder already exists.""" 364 | out_folder = os.path.join(self.scratch_folder, "ExistingOutFolder") 365 | os.makedirs(out_folder) 366 | with self.assertRaises(arcpy.ExecuteError) as ex: 367 | # Run tool 368 | out_origins = os.path.join(self.output_gdb, "Err_Origins") 369 | out_destinations = os.path.join(self.output_gdb, "Err_Destinations") 370 | arcpy.LargeNetworkAnalysisTools.SolveLargeODCostMatrix( # pylint: disable=no-member 371 | self.origins, 372 | self.destinations, 373 | self.local_nd, 374 | self.local_tm_time, 375 | "Minutes", 376 | "Miles", 377 | 50, # chunk size 378 | 4, # max processes 379 | out_origins, 380 | out_destinations, 381 | "CSV files", 382 | "Junk", 383 | out_folder, 384 | 15, # cutoff 385 | 1, # number of destinations 386 | None, # time of day 387 | None, # barriers 388 | True, # precalculate network locations 389 | True # Spatially sort inputs 390 | ) 391 | expected_messages = [ 392 | "Failed to execute. Parameters are not valid.", 393 | f"ERROR 000012: {out_folder} already exists", 394 | "Failed to execute (SolveLargeODCostMatrix)." 395 | ] 396 | actual_messages = str(ex.exception).strip().split("\n") 397 | self.assertEqual(expected_messages, actual_messages) 398 | 399 | @unittest.skipIf( 400 | helpers.arcgis_version < "2.9", "Arrow table output is not available in versions of Pro prior to 2.9.") 401 | def test_error_arrow_service(self): 402 | """Test for correct error when the network data source is a service and requesting Arrow output.""" 403 | with self.assertRaises(arcpy.ExecuteError) as ex: 404 | # Run tool 405 | out_od_folder = os.path.join(self.scratch_folder, "Err") 406 | out_origins = os.path.join(self.output_gdb, "Err_Origins") 407 | out_destinations = os.path.join(self.output_gdb, "Err_Destinations") 408 | arcpy.LargeNetworkAnalysisTools.SolveLargeODCostMatrix( # pylint: disable=no-member 409 | self.origins, 410 | self.destinations, 411 | self.portal_nd, 412 | self.portal_tm, 413 | "Minutes", 414 | "Miles", 415 | 50, # chunk size 416 | 4, # max processes 417 | out_origins, 418 | out_destinations, 419 | "Apache Arrow files", 420 | None, 421 | out_od_folder, 422 | 15, # cutoff 423 | 1, # number of destinations 424 | None, # time of day 425 | None, # barriers 426 | True, # precalculate network locations 427 | True # Spatially sort inputs 428 | ) 429 | expected_messages = [ 430 | "Failed to execute. Parameters are not valid.", 431 | "Apache Arrow files output format is not available when a service is used as the network data source.", 432 | "Failed to execute (SolveLargeODCostMatrix)." 433 | ] 434 | actual_messages = str(ex.exception).strip().split("\n") 435 | self.assertEqual(expected_messages, actual_messages) 436 | 437 | 438 | if __name__ == '__main__': 439 | unittest.main() 440 | -------------------------------------------------------------------------------- /unittests/test_helpers.py: -------------------------------------------------------------------------------- 1 | """Unit tests for the helpers.py module. 2 | 3 | Copyright 2023 Esri 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | """ 14 | import sys 15 | import os 16 | import datetime 17 | import logging 18 | import unittest 19 | import arcpy 20 | import portal_credentials # Contains log-in for an ArcGIS Online account to use as a test portal 21 | 22 | CWD = os.path.dirname(os.path.abspath(__file__)) 23 | sys.path.append(os.path.dirname(CWD)) 24 | import helpers # noqa: E402, pylint: disable=wrong-import-position 25 | from od_config import OD_PROPS # noqa: E402, pylint: disable=wrong-import-position 26 | from rt_config import RT_PROPS # noqa: E402, pylint: disable=wrong-import-position 27 | 28 | 29 | class TestHelpers(unittest.TestCase): 30 | """Test cases for the helpers module.""" 31 | 32 | @classmethod 33 | def setUpClass(self): # pylint: disable=bad-classmethod-argument 34 | """Set up shared test properties.""" 35 | self.maxDiff = None 36 | 37 | self.input_data_folder = os.path.join(CWD, "TestInput") 38 | self.sf_gdb = os.path.join(self.input_data_folder, "SanFrancisco.gdb") 39 | self.local_nd = os.path.join(self.sf_gdb, "Transportation", "Streets_ND") 40 | self.portal_nd = portal_credentials.PORTAL_URL 41 | 42 | arcpy.SignInToPortal(self.portal_nd, portal_credentials.PORTAL_USERNAME, portal_credentials.PORTAL_PASSWORD) 43 | 44 | self.scratch_folder = os.path.join( 45 | CWD, "TestOutput", "Output_Helpers_" + datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) 46 | os.makedirs(self.scratch_folder) 47 | self.output_gdb = os.path.join(self.scratch_folder, "outputs.gdb") 48 | arcpy.management.CreateFileGDB(os.path.dirname(self.output_gdb), os.path.basename(self.output_gdb)) 49 | 50 | def test_is_nds_service(self): 51 | """Test the is_nds_service function.""" 52 | self.assertTrue(helpers.is_nds_service(self.portal_nd)) 53 | self.assertFalse(helpers.is_nds_service(self.local_nd)) 54 | 55 | def test_get_tool_limits_and_is_agol(self): 56 | """Test the _get_tool_limits_and_is_agol function for a portal network data source.""" 57 | services = [ 58 | ("asyncODCostMatrix", "GenerateOriginDestinationCostMatrix"), 59 | ("asyncRoute", "FindRoutes") 60 | ] 61 | for service in services: 62 | with self.subTest(service=service): 63 | service_limits, is_agol = helpers.get_tool_limits_and_is_agol( 64 | self.portal_nd, service[0], service[1]) 65 | self.assertIsInstance(service_limits, dict) 66 | self.assertIsInstance(is_agol, bool) 67 | if service[0] == "asyncODCostMatrix": 68 | self.assertIn("maximumDestinations", service_limits) 69 | self.assertIn("maximumOrigins", service_limits) 70 | elif service[0] == "asyncRoute": 71 | self.assertIn("maximumStops", service_limits) 72 | if "arcgis.com" in self.portal_nd: 73 | # Note: If testing with some other portal, this test would need to be updated. 74 | self.assertTrue(is_agol) 75 | 76 | def test_update_agol_max_processes(self): 77 | """Test the update_agol_max_processes function.""" 78 | self.assertEqual(helpers.MAX_AGOL_PROCESSES, helpers.update_agol_max_processes(5000)) 79 | 80 | def test_convert_time_units_str_to_enum(self): 81 | """Test the convert_time_units_str_to_enum function.""" 82 | # Test all valid units 83 | valid_units = helpers.TIME_UNITS 84 | for unit in valid_units: 85 | enum_unit = helpers.convert_time_units_str_to_enum(unit) 86 | self.assertIsInstance(enum_unit, arcpy.nax.TimeUnits) 87 | self.assertEqual(unit.lower(), enum_unit.name.lower()) 88 | # Test for correct error with invalid units 89 | bad_unit = "BadUnit" 90 | with self.assertRaises(ValueError) as ex: 91 | helpers.convert_time_units_str_to_enum(bad_unit) 92 | self.assertEqual(f"Invalid time units: {bad_unit}", str(ex.exception)) 93 | 94 | def test_convert_distance_units_str_to_enum(self): 95 | """Test the convert_distance_units_str_to_enum function.""" 96 | # Test all valid units 97 | valid_units = helpers.DISTANCE_UNITS 98 | for unit in valid_units: 99 | enum_unit = helpers.convert_distance_units_str_to_enum(unit) 100 | self.assertIsInstance(enum_unit, arcpy.nax.DistanceUnits) 101 | self.assertEqual(unit.lower(), enum_unit.name.lower()) 102 | # Test for correct error with invalid units 103 | bad_unit = "BadUnit" 104 | with self.assertRaises(ValueError) as ex: 105 | helpers.convert_distance_units_str_to_enum(bad_unit) 106 | self.assertEqual(f"Invalid distance units: {bad_unit}", str(ex.exception)) 107 | 108 | def test_convert_output_format_str_to_enum(self): 109 | """Test the convert_output_format_str_to_enum function.""" 110 | # Test all valid formats 111 | valid_formats = helpers.OUTPUT_FORMATS 112 | for fm in valid_formats: 113 | enum_format = helpers.convert_output_format_str_to_enum(fm) 114 | self.assertIsInstance(enum_format, helpers.OutputFormat) 115 | # Test for correct error with an invalid format type 116 | bad_format = "BadFormat" 117 | with self.assertRaises(ValueError) as ex: 118 | helpers.convert_output_format_str_to_enum(bad_format) 119 | self.assertEqual(f"Invalid output format: {bad_format}", str(ex.exception)) 120 | 121 | def test_convert_pair_type_str_to_enum(self): 122 | """Test the convert_pair_type_str_to_enum function.""" 123 | # Test all valid pair types 124 | valid_pair_types = helpers.PAIR_TYPES 125 | for fm in valid_pair_types: 126 | enum_format = helpers.convert_pair_type_str_to_enum(fm) 127 | self.assertIsInstance(enum_format, helpers.PreassignedODPairType) 128 | # Test for correct error with an invalid format type 129 | bad_format = "BadFormat" 130 | with self.assertRaises(ValueError) as ex: 131 | helpers.convert_pair_type_str_to_enum(bad_format) 132 | self.assertEqual(f"Invalid OD pair assignment type: {bad_format}", str(ex.exception)) 133 | 134 | def test_validate_input_feature_class(self): 135 | """Test the validate_input_feature_class function.""" 136 | # Test when the input feature class does note exist. 137 | input_fc = os.path.join(self.sf_gdb, "DoesNotExist") 138 | with self.subTest(feature_class=input_fc): 139 | with self.assertRaises(ValueError) as ex: 140 | helpers.validate_input_feature_class(input_fc) 141 | self.assertEqual(f"Input dataset {input_fc} does not exist.", str(ex.exception)) 142 | 143 | # Test when the input feature class is empty 144 | input_fc = os.path.join(self.output_gdb, "EmptyFC") 145 | with self.subTest(feature_class=input_fc): 146 | arcpy.management.CreateFeatureclass(self.output_gdb, os.path.basename(input_fc)) 147 | with self.assertRaises(ValueError) as ex: 148 | helpers.validate_input_feature_class(input_fc) 149 | self.assertEqual(f"Input dataset {input_fc} has no rows.", str(ex.exception)) 150 | 151 | def test_are_input_layers_the_same(self): 152 | """Test the are_input_layers_the_same function.""" 153 | fc1 = os.path.join(self.sf_gdb, "Analysis", "TractCentroids") 154 | fc2 = os.path.join(self.sf_gdb, "Analysis", "Hospitals") 155 | lyr1_name = "Layer1" 156 | lyr2_name = "Layer2" 157 | lyr1_obj = arcpy.management.MakeFeatureLayer(fc1, lyr1_name) 158 | lyr1_obj_again = arcpy.management.MakeFeatureLayer(fc1, "Layer1 again") 159 | lyr2_obj = arcpy.management.MakeFeatureLayer(fc2, lyr2_name) 160 | lyr1_file = os.path.join(self.scratch_folder, "lyr1.lyrx") 161 | lyr2_file = os.path.join(self.scratch_folder, "lyr2.lyrx") 162 | arcpy.management.SaveToLayerFile(lyr1_obj, lyr1_file) 163 | arcpy.management.SaveToLayerFile(lyr2_obj, lyr2_file) 164 | fset_1 = arcpy.FeatureSet(fc1) 165 | fset_2 = arcpy.FeatureSet(fc2) 166 | 167 | # Feature class catalog path inputs 168 | self.assertFalse(helpers.are_input_layers_the_same(fc1, fc2)) 169 | self.assertTrue(helpers.are_input_layers_the_same(fc1, fc1)) 170 | # Layer inputs 171 | self.assertFalse(helpers.are_input_layers_the_same(lyr1_name, lyr2_name)) 172 | self.assertTrue(helpers.are_input_layers_the_same(lyr1_name, lyr1_name)) 173 | self.assertFalse(helpers.are_input_layers_the_same(lyr1_obj, lyr2_obj)) 174 | self.assertTrue(helpers.are_input_layers_the_same(lyr1_obj, lyr1_obj)) 175 | self.assertFalse(helpers.are_input_layers_the_same(lyr1_obj, lyr1_obj_again)) 176 | self.assertFalse(helpers.are_input_layers_the_same(lyr1_file, lyr2_file)) 177 | self.assertTrue(helpers.are_input_layers_the_same(lyr1_file, lyr1_file)) 178 | # Feature set inputs 179 | self.assertFalse(helpers.are_input_layers_the_same(fset_1, fset_2)) 180 | self.assertTrue(helpers.are_input_layers_the_same(fset_1, fset_1)) 181 | 182 | def test_validate_network_data_source(self): 183 | """Test the validate_network_data_source function.""" 184 | # Check that it returns the catalog path of a network dataset layer 185 | nd_layer = arcpy.na.MakeNetworkDatasetLayer(self.local_nd).getOutput(0) 186 | with self.subTest(network_data_source=nd_layer): 187 | self.assertEqual(self.local_nd, helpers.validate_network_data_source(nd_layer)) 188 | # Check that it returns a portal URL with a trailing slash if it initially lacked one 189 | portal_url = self.portal_nd.strip(r"/") 190 | with self.subTest(network_data_source=portal_url): 191 | self.assertEqual(portal_url + r"/", helpers.validate_network_data_source(portal_url)) 192 | # Check for ValueError if the network dataset doesn't exist 193 | bad_network = os.path.join(self.sf_gdb, "Transportation", "DoesNotExist") 194 | with self.subTest(network_data_source=bad_network): 195 | with self.assertRaises(ValueError) as ex: 196 | helpers.validate_network_data_source(bad_network) 197 | self.assertEqual(str(ex.exception), f"Input network dataset {bad_network} does not exist.") 198 | 199 | def test_get_locatable_network_source_names(self): 200 | """Test the get_locatable_network_source_names funtion.""" 201 | self.assertEqual( 202 | ["Streets", "Streets_ND_Junctions"], 203 | helpers.get_locatable_network_source_names(self.local_nd) 204 | ) 205 | 206 | def test_get_default_locatable_network_source_names(self): 207 | """Test the get_default_locatable_network_source_names funtion.""" 208 | self.assertEqual( 209 | ["Streets"], 210 | helpers.get_default_locatable_network_source_names(self.local_nd) 211 | ) 212 | 213 | def test_get_locate_settings_from_config_file(self): 214 | """Test the get_locate_settings_from_config_file function.""" 215 | # Test searchTolerance and searchQuery without searchSources 216 | config_props = { 217 | "searchQuery": [["Streets", "ObjectID <> 1"], ["Streets_ND_Junctions", ""]], 218 | "searchTolerance": 1000, 219 | "searchToleranceUnits": arcpy.nax.DistanceUnits.Feet 220 | } 221 | search_tolerance, search_criteria, search_query = helpers.get_locate_settings_from_config_file( 222 | config_props, self.local_nd) 223 | self.assertEqual("1000 Feet", search_tolerance, "Incorrect search tolerance.") 224 | self.assertEqual( 225 | "", search_criteria, 226 | "Search criteria should be an empty string when searchSources is not used.") 227 | self.assertEqual("Streets 'ObjectID <> 1';Streets_ND_Junctions #", search_query, "Incorrect search query.") 228 | 229 | # Test searchSources 230 | config_props = { 231 | "searchSources": [["Streets", "ObjectID <> 1"]], 232 | "searchTolerance": 1000, 233 | } 234 | search_tolerance, search_criteria, search_query = helpers.get_locate_settings_from_config_file( 235 | config_props, self.local_nd) 236 | self.assertEqual( 237 | "", search_tolerance, 238 | "Search tolerance should be an empty string when both searchTolerance and searchToleranceUnits are not set." 239 | ) 240 | self.assertEqual( 241 | "Streets SHAPE;Streets_ND_Junctions NONE", search_criteria, 242 | "Incorrect search criteria.") 243 | self.assertEqual("Streets 'ObjectID <> 1'", search_query, "Incorrect search query.") 244 | 245 | def test_get_oid_ranges_for_input(self): 246 | """Test the get_oid_ranges_for_input function.""" 247 | ranges = helpers.get_oid_ranges_for_input(os.path.join(self.sf_gdb, "Analysis", "TractCentroids"), 50) 248 | self.assertEqual([[1, 50], [51, 100], [101, 150], [151, 200], [201, 208]], ranges) 249 | 250 | def test_parse_std_and_write_to_gp_ui(self): 251 | """Test the parse_std_and_write_to_gp_ui function.""" 252 | # There is nothing much to test here except that nothing terrible happens. 253 | msgs = [ 254 | f"CRITICAL{helpers.MSG_STR_SPLITTER}Critical message", 255 | f"ERROR{helpers.MSG_STR_SPLITTER}Error message", 256 | f"WARNING{helpers.MSG_STR_SPLITTER}Warning message", 257 | f"INFO{helpers.MSG_STR_SPLITTER}Info message", 258 | f"DEBUG{helpers.MSG_STR_SPLITTER}Debug message", 259 | "Poorly-formatted message 1", 260 | f"Poorly-formatted{helpers.MSG_STR_SPLITTER}message 2" 261 | ] 262 | for msg in msgs: 263 | with self.subTest(msg=msg): 264 | helpers.parse_std_and_write_to_gp_ui(msg) 265 | 266 | def test_run_gp_tool(self): 267 | """Test the run_gp_tool function.""" 268 | # Set up a logger to use with the function 269 | logger = logging.getLogger(__name__) # pylint:disable=invalid-name 270 | # Test for handled tool execute error (create fgdb in invalid folder) 271 | with self.assertRaises(arcpy.ExecuteError): 272 | helpers.run_gp_tool( 273 | logger, 274 | arcpy.management.CreateFileGDB, 275 | [self.scratch_folder + "DoesNotExist"], 276 | {"out_name": "outputs.gdb"} 277 | ) 278 | # Test for handled non-arcpy error when calling function 279 | with self.assertRaises(TypeError): 280 | helpers.run_gp_tool(logger, "BadTool", [self.scratch_folder]) 281 | # Valid call to tool with simple function 282 | helpers.run_gp_tool( 283 | logger, arcpy.management.CreateFileGDB, [self.scratch_folder], {"out_name": "testRunTool.gdb"}) 284 | 285 | 286 | if __name__ == '__main__': 287 | unittest.main() 288 | -------------------------------------------------------------------------------- /unittests/test_parallel_calculate_locations.py: -------------------------------------------------------------------------------- 1 | """Unit tests for the parallel_calculate_locations.py module. 2 | 3 | Copyright 2023 Esri 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | """ 14 | # pylint: disable=import-error, protected-access, invalid-name 15 | 16 | import sys 17 | import os 18 | import datetime 19 | import subprocess 20 | import unittest 21 | import arcpy 22 | 23 | CWD = os.path.dirname(os.path.abspath(__file__)) 24 | sys.path.append(os.path.dirname(CWD)) 25 | import parallel_calculate_locations # noqa: E402, pylint: disable=wrong-import-position 26 | from helpers import configure_global_logger, teardown_logger 27 | 28 | 29 | class TestParallelCalculateLocations(unittest.TestCase): 30 | """Test cases for the parallel_calculate_locations module.""" 31 | 32 | @classmethod 33 | def setUpClass(self): # pylint: disable=bad-classmethod-argument 34 | """Set up shared test properties.""" 35 | self.maxDiff = None 36 | 37 | self.input_data_folder = os.path.join(CWD, "TestInput") 38 | sf_gdb = os.path.join(self.input_data_folder, "SanFrancisco.gdb") 39 | self.input_fc = os.path.join(sf_gdb, "Analysis", "Stores") 40 | self.local_nd = os.path.join(sf_gdb, "Transportation", "Streets_ND") 41 | self.local_tm_time = "Driving Time" 42 | 43 | # Create a unique output directory and gdb for this test 44 | self.output_folder = os.path.join( 45 | CWD, "TestOutput", "Output_ParallelCalcLocs_" + datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) 46 | os.makedirs(self.output_folder) 47 | self.output_gdb = os.path.join(self.output_folder, "outputs.gdb") 48 | arcpy.management.CreateFileGDB(os.path.dirname(self.output_gdb), os.path.basename(self.output_gdb)) 49 | 50 | def check_precalculated_locations(self, fc, check_has_values): 51 | """Check precalculated locations.""" 52 | loc_fields = {"SourceID", "SourceOID", "PosAlong", "SideOfEdge"} 53 | actual_fields = set([f.name for f in arcpy.ListFields(fc)]) 54 | self.assertTrue(loc_fields.issubset(actual_fields), "Network location fields not added") 55 | if check_has_values: 56 | for row in arcpy.da.SearchCursor(fc, list(loc_fields)): # pylint: disable=no-member 57 | for val in row: 58 | self.assertIsNotNone(val) 59 | 60 | def test_LocationCalculator_subset_inputs(self): 61 | """Test the _subset_inputs method of the LocationCalculator class.""" 62 | inputs = { 63 | "input_fc": self.input_fc, 64 | "network_data_source": self.local_nd, 65 | "travel_mode": self.local_tm_time, 66 | "scratch_folder": self.output_folder 67 | } 68 | location_calculator = parallel_calculate_locations.LocationCalculator(**inputs) 69 | location_calculator._subset_inputs([9, 14]) 70 | self.assertTrue(arcpy.Exists(location_calculator.out_fc), "Subset fc does not exist.") 71 | self.assertEqual( 72 | 6, int(arcpy.management.GetCount(location_calculator.out_fc).getOutput(0)), 73 | "Subset feature class has the wrong number of rows." 74 | ) 75 | 76 | def test_LocationCalculator_calculate_locations(self): 77 | """Test the calculate_locations method of the LocationCalculator class. 78 | 79 | Use all optional Calculate Locations tool settings. 80 | """ 81 | fc_to_precalculate = os.path.join(self.output_gdb, "PrecalcFC_LocationCalculator") 82 | arcpy.management.Copy(self.input_fc, fc_to_precalculate) 83 | inputs = { 84 | "input_fc": fc_to_precalculate, 85 | "network_data_source": self.local_nd, 86 | "travel_mode": self.local_tm_time, 87 | "scratch_folder": self.output_folder, 88 | "search_tolerance": "1000 Feet", 89 | "search_criteria": [["Streets", "SHAPE"], ["Streets_ND_Junctions", "SHAPE"]], 90 | "search_query": [["Streets", "ObjectID <> 1"], ["Streets_ND_Junctions", ""]] 91 | } 92 | location_calculator = parallel_calculate_locations.LocationCalculator(**inputs) 93 | oid_range = [9, 14] 94 | location_calculator.calculate_locations(oid_range) 95 | self.assertTrue(arcpy.Exists(location_calculator.out_fc), "Subset fc does not exist.") 96 | self.assertEqual( 97 | 6, int(arcpy.management.GetCount(location_calculator.out_fc).getOutput(0)), 98 | "Subset feature class has the wrong number of rows." 99 | ) 100 | self.check_precalculated_locations(location_calculator.out_fc, check_has_values=False) 101 | self.assertEqual( 102 | location_calculator.out_fc, location_calculator.job_result["outputFC"], 103 | "outputFC property of job_result was not set correctly." 104 | ) 105 | self.assertEqual( 106 | tuple(oid_range), location_calculator.job_result["oidRange"], 107 | "oidRange property of job_result was not set correctly." 108 | ) 109 | 110 | def test_ParallelLocationCalculator(self): 111 | """Test the ParallelLocationCalculator class.""" 112 | # The input feature class should not be overwritten by this tool, but copy it first just in case. 113 | fc_to_precalculate = os.path.join(self.output_gdb, "PrecalcFC_Parallel") 114 | arcpy.management.Copy(self.input_fc, fc_to_precalculate) 115 | out_fc = os.path.join(self.output_gdb, "PrecalcFC_Parallel_out") 116 | logger = configure_global_logger(parallel_calculate_locations.LOG_LEVEL) 117 | inputs = { 118 | "logger": logger, 119 | "input_features": fc_to_precalculate, 120 | "output_features": out_fc, 121 | "chunk_size": 6, 122 | "max_processes": 4, 123 | "network_data_source": self.local_nd, 124 | "travel_mode": self.local_tm_time, 125 | "search_tolerance": "1000 Feet", 126 | "search_criteria": [["Streets", "SHAPE"], ["Streets_ND_Junctions", "SHAPE"]], 127 | "search_query": [["Streets", "ObjectID <> 1"], ["Streets_ND_Junctions", ""]] 128 | } 129 | try: 130 | parallel_calculator = parallel_calculate_locations.ParallelLocationCalculator(**inputs) 131 | parallel_calculator.calc_locs_in_parallel() 132 | self.assertTrue(arcpy.Exists(out_fc), "Output fc does not exist.") 133 | self.assertEqual( 134 | int(arcpy.management.GetCount(self.input_fc).getOutput(0)), 135 | int(arcpy.management.GetCount(out_fc).getOutput(0)), 136 | "Output feature class doesn't have the same number of rows as the original input." 137 | ) 138 | self.check_precalculated_locations(out_fc, check_has_values=True) 139 | finally: 140 | teardown_logger(logger) 141 | 142 | def test_cli(self): 143 | """Test the command line interface.""" 144 | # The input feature class should not be overwritten by this tool, but copy it first just in case. 145 | fc_to_precalculate = os.path.join(self.output_gdb, "PrecalcFC_CLI") 146 | arcpy.management.Copy(self.input_fc, fc_to_precalculate) 147 | out_fc = os.path.join(self.output_gdb, "PrecalcFC_CLI_out") 148 | inputs = [ 149 | os.path.join(sys.exec_prefix, "python.exe"), 150 | os.path.join(os.path.dirname(CWD), "parallel_calculate_locations.py"), 151 | "--input-features", fc_to_precalculate, 152 | "--output-features", out_fc, 153 | "--network-data-source", self.local_nd, 154 | "--chunk-size", "6", 155 | "--max-processes", "4", 156 | "--travel-mode", self.local_tm_time, 157 | "--search-tolerance", "1000 Feet", 158 | "--search-criteria", "Streets SHAPE;Streets_ND_Junctions NONE", 159 | "--search-query", "Streets 'OBJECTID <> 1'" 160 | ] 161 | result = subprocess.run(inputs, check=True) 162 | self.assertEqual(result.returncode, 0) 163 | self.assertTrue(arcpy.Exists(out_fc)) 164 | 165 | 166 | if __name__ == '__main__': 167 | unittest.main() 168 | -------------------------------------------------------------------------------- /unittests/test_parallel_odcm.py: -------------------------------------------------------------------------------- 1 | """Unit tests for the parallel_odcm.py module. 2 | 3 | Copyright 2024 Esri 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | """ 14 | # pylint: disable=import-error, protected-access, invalid-name 15 | 16 | import sys 17 | import os 18 | import datetime 19 | import unittest 20 | import pandas as pd 21 | from copy import deepcopy 22 | from glob import glob 23 | import arcpy 24 | 25 | CWD = os.path.dirname(os.path.abspath(__file__)) 26 | sys.path.append(os.path.dirname(CWD)) 27 | import parallel_odcm # noqa: E402, pylint: disable=wrong-import-position 28 | import helpers # noqa: E402, pylint: disable=wrong-import-position 29 | import portal_credentials # noqa: E402, pylint: disable=wrong-import-position 30 | 31 | TEST_ARROW = False 32 | if helpers.arcgis_version >= "2.9": 33 | # The pyarrow module was not included in earlier versions of Pro, and the toArrowTable method was 34 | # added to the ODCostMatrix object at Pro 2.9. Do not attempt to test this output format in 35 | # earlier versions of Pro. 36 | TEST_ARROW = True 37 | import pyarrow as pa 38 | 39 | 40 | class TestParallelODCM(unittest.TestCase): 41 | """Test cases for the parallel_odcm module.""" 42 | 43 | @classmethod 44 | def setUpClass(self): # pylint: disable=bad-classmethod-argument 45 | """Set up shared test properties.""" 46 | self.maxDiff = None 47 | 48 | self.input_data_folder = os.path.join(CWD, "TestInput") 49 | sf_gdb = os.path.join(self.input_data_folder, "SanFrancisco.gdb") 50 | self.origins = os.path.join(sf_gdb, "Analysis", "TractCentroids") 51 | self.destinations = os.path.join(sf_gdb, "Analysis", "Hospitals") 52 | self.local_nd = os.path.join(sf_gdb, "Transportation", "Streets_ND") 53 | self.local_tm_time = "Driving Time" 54 | self.local_tm_dist = "Driving Distance" 55 | self.portal_nd = portal_credentials.PORTAL_URL 56 | self.portal_tm = portal_credentials.PORTAL_TRAVEL_MODE 57 | 58 | arcpy.SignInToPortal(self.portal_nd, portal_credentials.PORTAL_USERNAME, portal_credentials.PORTAL_PASSWORD) 59 | 60 | # Create a unique output directory and gdb for this test 61 | self.scratch_folder = os.path.join( 62 | CWD, "TestOutput", "Output_ParallelODCM_" + datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) 63 | os.makedirs(self.scratch_folder) 64 | self.output_gdb = os.path.join(self.scratch_folder, "outputs.gdb") 65 | arcpy.management.CreateFileGDB(os.path.dirname(self.output_gdb), os.path.basename(self.output_gdb)) 66 | 67 | self.od_args = { 68 | "origins": self.origins, 69 | "destinations": self.destinations, 70 | "output_format": helpers.OutputFormat.featureclass, 71 | "output_od_location": os.path.join(self.output_gdb, "TestOutput"), 72 | "network_data_source": self.local_nd, 73 | "travel_mode": self.local_tm_dist, 74 | "time_units": arcpy.nax.TimeUnits.Minutes, 75 | "distance_units": arcpy.nax.DistanceUnits.Miles, 76 | "cutoff": 2, 77 | "num_destinations": 1, 78 | "time_of_day": None, 79 | "scratch_folder": self.scratch_folder, 80 | "barriers": [] 81 | } 82 | 83 | self.logger = helpers.configure_global_logger(parallel_odcm.LOG_LEVEL) 84 | 85 | self.parallel_od_class_args = { 86 | "logger": self.logger, 87 | "origins": self.origins, 88 | "destinations": self.destinations, 89 | "network_data_source": self.local_nd, 90 | "travel_mode": self.local_tm_dist, 91 | "output_format": "Feature class", 92 | "output_od_location": os.path.join(self.output_gdb, "TestOutput"), 93 | "max_origins": 1000, 94 | "max_destinations": 1000, 95 | "max_processes": 4, 96 | "time_units": "Minutes", 97 | "distance_units": "Miles", 98 | "cutoff": 2, 99 | "num_destinations": 1, 100 | "time_of_day": "20220329 16:45", 101 | "barriers": [] 102 | } 103 | 104 | @classmethod 105 | def tearDownClass(self): 106 | """Deconstruct the logger when tests are finished.""" 107 | helpers.teardown_logger(self.logger) 108 | 109 | def test_ODCostMatrix_hour_to_time_units(self): 110 | """Test the _hour_to_time_units method of the ODCostMatrix class.""" 111 | # Sanity test to make sure the method works for valid units 112 | od_inputs = deepcopy(self.od_args) 113 | od_inputs["time_units"] = arcpy.nax.TimeUnits.Seconds 114 | od = parallel_odcm.ODCostMatrix(**od_inputs) 115 | self.assertEqual(3600, od._hour_to_time_units()) 116 | 117 | def test_ODCostMatrix_mile_to_dist_units(self): 118 | """Test the _mile_to_dist_units method of the ODCostMatrix class.""" 119 | # Sanity test to make sure the method works for valid units 120 | od_inputs = deepcopy(self.od_args) 121 | od_inputs["distance_units"] = arcpy.nax.DistanceUnits.Kilometers 122 | od = parallel_odcm.ODCostMatrix(**od_inputs) 123 | self.assertEqual(1.60934, od._mile_to_dist_units()) 124 | 125 | def test_ODCostMatrix_convert_time_cutoff_to_distance(self): 126 | """Test the _convert_time_cutoff_to_distance method of the ODCostMatrix class.""" 127 | # We start with a 20-minute cutoff. The method converts this to a reasonable distance in units of miles. 128 | od_inputs = deepcopy(self.od_args) 129 | od_inputs["travel_mode"] = self.local_tm_time 130 | od = parallel_odcm.ODCostMatrix(**od_inputs) 131 | self.assertAlmostEqual(28, od._convert_time_cutoff_to_distance(20), 1) 132 | 133 | def test_ODCostMatrix_select_inputs(self): 134 | """Test the _select_inputs method of the ODCostMatrix class.""" 135 | od = parallel_odcm.ODCostMatrix(**self.od_args) 136 | origin_criteria = [1, 2] # Encompasses 2 rows in the southwest corner 137 | 138 | with arcpy.EnvManager(overwriteOutput=True): 139 | # Test when a subset of destinations meets the cutoff criteria 140 | dest_criteria = [8, 12] # Encompasses 5 rows. Two are close to the origins. 141 | od._select_inputs(origin_criteria, dest_criteria) 142 | self.assertEqual(2, int(arcpy.management.GetCount(od.input_origins_layer_obj).getOutput(0))) 143 | # Only two destinations fall within the distance threshold 144 | self.assertEqual(2, int(arcpy.management.GetCount(od.input_destinations_layer_obj).getOutput(0))) 145 | 146 | # Test when none of the destinations are within the threshold 147 | dest_criteria = [14, 17] # Encompasses 4 locations in the far northeast corner 148 | od._select_inputs(origin_criteria, dest_criteria) 149 | self.assertEqual(2, int(arcpy.management.GetCount(od.input_origins_layer_obj).getOutput(0))) 150 | self.assertIsNone( 151 | od.input_destinations_layer_obj, 152 | "Destinations layer should be None since no destinations fall within the straight-line cutoff of origins." 153 | ) 154 | 155 | def test_ODCostMatrix_solve_featureclass(self): 156 | """Test the solve method of the ODCostMatrix class with feature class output.""" 157 | # Initialize an ODCostMatrix analysis object 158 | od = parallel_odcm.ODCostMatrix(**self.od_args) 159 | # Solve a chunk 160 | origin_criteria = [1, 2] # Encompasses 2 rows 161 | dest_criteria = [8, 12] # Encompasses 5 rows 162 | od.solve(origin_criteria, dest_criteria) 163 | # Check results 164 | self.assertIsInstance(od.job_result, dict) 165 | self.assertTrue(od.job_result["solveSucceeded"], "OD solve failed") 166 | self.assertTrue(arcpy.Exists(od.job_result["outputLines"]), "OD line output does not exist.") 167 | self.assertEqual(2, int(arcpy.management.GetCount(od.job_result["outputLines"]).getOutput(0))) 168 | 169 | def test_ODCostMatrix_solve_csv(self): 170 | """Test the solve method of the ODCostMatrix class with csv output.""" 171 | # Initialize an ODCostMatrix analysis object 172 | out_folder = os.path.join(self.scratch_folder, "ODCostMatrix_CSV") 173 | os.mkdir(out_folder) 174 | od_inputs = deepcopy(self.od_args) 175 | od_inputs["output_format"] = helpers.OutputFormat.csv 176 | od_inputs["output_od_location"] = out_folder 177 | od = parallel_odcm.ODCostMatrix(**od_inputs) 178 | # Solve a chunk 179 | origin_criteria = [1, 2] # Encompasses 2 rows 180 | dest_criteria = [8, 12] # Encompasses 5 rows 181 | od.solve(origin_criteria, dest_criteria) 182 | # Check results 183 | self.assertIsInstance(od.job_result, dict) 184 | self.assertTrue(od.job_result["solveSucceeded"], "OD solve failed") 185 | expected_out_file = os.path.join( 186 | out_folder, 187 | f"ODLines_O_{origin_criteria[0]}_{origin_criteria[1]}_D_{dest_criteria[0]}_{dest_criteria[1]}.csv" 188 | ) 189 | self.assertTrue(os.path.exists(od.job_result["outputLines"]), "OD line CSV file output does not exist.") 190 | self.assertEqual(expected_out_file, od.job_result["outputLines"], "OD line CSV file has the wrong filepath.") 191 | row_count = pd.read_csv(expected_out_file).shape[0] 192 | self.assertEqual(2, row_count, "OD line CSV file has an incorrect number of rows.") 193 | 194 | @unittest.skipIf(not TEST_ARROW, "Arrow table output is not available in versions of Pro prior to 2.9.") 195 | def test_ODCostMatrix_solve_arrow(self): 196 | """Test the solve method of the ODCostMatrix class with Arrow output.""" 197 | # Initialize an ODCostMatrix analysis object 198 | out_folder = os.path.join(self.scratch_folder, "ODCostMatrix_Arrow") 199 | os.mkdir(out_folder) 200 | od_inputs = deepcopy(self.od_args) 201 | od_inputs["output_format"] = helpers.OutputFormat.arrow 202 | od_inputs["output_od_location"] = out_folder 203 | od = parallel_odcm.ODCostMatrix(**od_inputs) 204 | # Solve a chunk 205 | origin_criteria = [1, 2] # Encompasses 2 rows 206 | dest_criteria = [8, 12] # Encompasses 5 rows 207 | od.solve(origin_criteria, dest_criteria) 208 | # Check results 209 | self.assertIsInstance(od.job_result, dict) 210 | self.assertTrue(od.job_result["solveSucceeded"], "OD solve failed") 211 | expected_out_file = os.path.join( 212 | out_folder, 213 | f"ODLines_O_{origin_criteria[0]}_{origin_criteria[1]}_D_{dest_criteria[0]}_{dest_criteria[1]}.arrow" 214 | ) 215 | self.assertTrue(os.path.exists(od.job_result["outputLines"]), "OD line Arrow file output does not exist.") 216 | self.assertEqual(expected_out_file, od.job_result["outputLines"], "OD line Arrow file has the wrong filepath.") 217 | with pa.memory_map(expected_out_file, 'r') as source: 218 | batch_reader = pa.ipc.RecordBatchFileReader(source) 219 | arrow_table = batch_reader.read_all() 220 | self.assertEqual(2, arrow_table.num_rows, "OD line Arrow file has an incorrect number of rows.") 221 | 222 | def test_ODCostMatrix_solve_service_featureclass(self): 223 | """Test the solve method of the ODCostMatrix class using a service with feature class output. 224 | 225 | When a service is used, the code does special extra steps to ensure the original origin and 226 | destination OIDs are transferred instead of reset starting with 1 for each chunk. This test 227 | specifically validates this special behavior. 228 | """ 229 | # Initialize an ODCostMatrix analysis object 230 | od_inputs = { 231 | "origins": self.origins, 232 | "destinations": self.destinations, 233 | "output_format": helpers.OutputFormat.featureclass, 234 | "output_od_location": os.path.join(self.output_gdb, "TestOutputService"), 235 | "network_data_source": self.portal_nd, 236 | "travel_mode": self.portal_tm, 237 | "time_units": arcpy.nax.TimeUnits.Minutes, 238 | "distance_units": arcpy.nax.DistanceUnits.Miles, 239 | "cutoff": 10, 240 | "num_destinations": None, 241 | "time_of_day": None, 242 | "scratch_folder": self.scratch_folder, 243 | "barriers": [] 244 | } 245 | od = parallel_odcm.ODCostMatrix(**od_inputs) 246 | # Solve a chunk 247 | origin_criteria = [45, 50] 248 | dest_criteria = [8, 12] 249 | od.solve(origin_criteria, dest_criteria) 250 | self.assertTrue(od.job_result["solveSucceeded"], "OD solve failed") 251 | self.assertTrue(arcpy.Exists(od.job_result["outputLines"]), "OD line output does not exist.") 252 | # Ensure the OriginOID and DestinationOID fields in the output have the correct numerical ranges 253 | fields_to_check = ["OriginOID", "DestinationOID"] 254 | for row in arcpy.da.SearchCursor(od.job_result["outputLines"], fields_to_check): 255 | self.assertTrue(origin_criteria[0] <= row[0] <= origin_criteria[1], f"{fields_to_check[0]} out of range.") 256 | self.assertTrue(dest_criteria[0] <= row[1] <= dest_criteria[1], f"{fields_to_check[1]} out of range.") 257 | 258 | def test_ODCostMatrix_solve_service_csv(self): 259 | """Test the solve method of the ODCostMatrix class using a service with csv output. 260 | 261 | When a service is used, the code does special extra steps to ensure the original origin and 262 | destination OIDs are transferred instead of reset starting with 1 for each chunk. This test 263 | specifically validates this special behavior. 264 | """ 265 | out_folder = os.path.join(self.scratch_folder, "ODCostMatrix_CSV_service") 266 | os.mkdir(out_folder) 267 | # Initialize an ODCostMatrix analysis object 268 | od_inputs = { 269 | "origins": self.origins, 270 | "destinations": self.destinations, 271 | "output_format": helpers.OutputFormat.csv, 272 | "output_od_location": out_folder, 273 | "network_data_source": self.portal_nd, 274 | "travel_mode": self.portal_tm, 275 | "time_units": arcpy.nax.TimeUnits.Minutes, 276 | "distance_units": arcpy.nax.DistanceUnits.Miles, 277 | "cutoff": 10, 278 | "num_destinations": None, 279 | "time_of_day": None, 280 | "scratch_folder": self.scratch_folder, 281 | "barriers": [] 282 | } 283 | od = parallel_odcm.ODCostMatrix(**od_inputs) 284 | # Solve a chunk 285 | origin_criteria = [45, 50] 286 | dest_criteria = [8, 12] 287 | od.solve(origin_criteria, dest_criteria) 288 | self.assertTrue(od.job_result["solveSucceeded"], "OD solve failed") 289 | expected_out_file = os.path.join( 290 | out_folder, 291 | f"ODLines_O_{origin_criteria[0]}_{origin_criteria[1]}_D_{dest_criteria[0]}_{dest_criteria[1]}.csv" 292 | ) 293 | self.assertTrue(os.path.exists(od.job_result["outputLines"]), "OD line CSV file output does not exist.") 294 | self.assertEqual(expected_out_file, od.job_result["outputLines"], "OD line CSV file has the wrong filepath.") 295 | # Ensure the OriginOID and DestinationOID fields in the output have the correct numerical ranges 296 | df = pd.read_csv(expected_out_file) 297 | self.assertTrue(df["OriginOID"].min() >= origin_criteria[0], "OriginOID out of range.") 298 | self.assertTrue(df["OriginOID"].max() <= origin_criteria[1], "OriginOID out of range.") 299 | self.assertTrue(df["DestinationOID"].min() >= dest_criteria[0], "DestinationOID out of range.") 300 | self.assertTrue(df["DestinationOID"].max() <= dest_criteria[1], "DestinationOID out of range.") 301 | 302 | def test_solve_od_cost_matrix(self): 303 | """Test the solve_od_cost_matrix function.""" 304 | result = parallel_odcm.solve_od_cost_matrix([[1, 2], [8, 12]], self.od_args) 305 | # Check results 306 | self.assertIsInstance(result, dict) 307 | self.assertTrue(os.path.exists(result["logFile"]), "Log file does not exist.") 308 | self.assertTrue(result["solveSucceeded"], "OD solve failed") 309 | self.assertTrue(arcpy.Exists(result["outputLines"]), "OD line output does not exist.") 310 | self.assertEqual(2, int(arcpy.management.GetCount(result["outputLines"]).getOutput(0))) 311 | 312 | def test_ParallelODCalculator_validate_od_settings(self): 313 | """Test the _validate_od_settings function.""" 314 | # Test that with good inputs, we return the correct optimized field name 315 | od_calculator = parallel_odcm.ParallelODCalculator(**self.parallel_od_class_args) 316 | optimized_cost_field = od_calculator._validate_od_settings() 317 | self.assertEqual("Total_Distance", optimized_cost_field) 318 | # Test completely invalid travel mode 319 | od_inputs = deepcopy(self.parallel_od_class_args) 320 | od_inputs["travel_mode"] = "InvalidTM" 321 | od_calculator = parallel_odcm.ParallelODCalculator(**od_inputs) 322 | error_type = ValueError if helpers.arcgis_version >= "3.1" else RuntimeError 323 | with self.assertRaises(error_type): 324 | od_calculator._validate_od_settings() 325 | 326 | def test_ParallelODCalculator_solve_od_in_parallel_featureclass(self): 327 | """Test the solve_od_in_parallel function. Output to feature class.""" 328 | out_od_lines = os.path.join(self.output_gdb, "Out_OD_Lines") 329 | inputs = { 330 | "logger": self.logger, 331 | "origins": self.origins, 332 | "destinations": self.destinations, 333 | "network_data_source": self.local_nd, 334 | "travel_mode": self.local_tm_time, 335 | "output_format": "Feature class", 336 | "output_od_location": out_od_lines, 337 | "max_origins": 20, 338 | "max_destinations": 20, 339 | "max_processes": 4, 340 | "time_units": "Minutes", 341 | "distance_units": "Miles", 342 | "cutoff": 30, 343 | "num_destinations": 2, 344 | "time_of_day": None, 345 | "barriers": [] 346 | } 347 | 348 | # Run parallel process. This calculates the OD and also post-processes the results 349 | od_calculator = parallel_odcm.ParallelODCalculator(**inputs) 350 | od_calculator.solve_od_in_parallel() 351 | 352 | # Check results 353 | self.assertTrue(arcpy.Exists(out_od_lines)) 354 | # With 2 destinations for each origin, expect 414 rows in the output 355 | # Note: 1 origin finds no destinations, and that's why we don't have 416. 356 | self.assertEqual(414, int(arcpy.management.GetCount(out_od_lines).getOutput(0))) 357 | 358 | def test_ParallelODCalculator_solve_od_in_parallel_featureclass_no_dest_limit(self): 359 | """Test the solve_od_in_parallel function. Output to feature class. No destination limit. 360 | 361 | A different codepath is used when post-processing OD Line feature classes if there is no destination limit. 362 | """ 363 | out_od_lines = os.path.join(self.output_gdb, "Out_OD_Lines_NoLimit") 364 | inputs = { 365 | "logger": self.logger, 366 | "origins": self.origins, 367 | "destinations": self.destinations, 368 | "network_data_source": self.local_nd, 369 | "travel_mode": self.local_tm_time, 370 | "output_format": "Feature class", 371 | "output_od_location": out_od_lines, 372 | "max_origins": 20, 373 | "max_destinations": 20, 374 | "max_processes": 4, 375 | "time_units": "Minutes", 376 | "distance_units": "Miles", 377 | "cutoff": 30, 378 | "num_destinations": None, 379 | "time_of_day": None, 380 | "barriers": [] 381 | } 382 | 383 | # Run parallel process. This calculates the OD and also post-processes the results 384 | od_calculator = parallel_odcm.ParallelODCalculator(**inputs) 385 | od_calculator.solve_od_in_parallel() 386 | 387 | # Check results 388 | self.assertTrue(arcpy.Exists(out_od_lines)) 389 | self.assertEqual(4545, int(arcpy.management.GetCount(out_od_lines).getOutput(0))) 390 | 391 | def test_ParallelODCalculator_solve_od_in_parallel_csv(self): 392 | """Test the solve_od_in_parallel function. Output to CSV.""" 393 | out_folder = os.path.join(self.scratch_folder, "ParallelODCalculator_CSV") 394 | os.mkdir(out_folder) 395 | inputs = { 396 | "logger": self.logger, 397 | "origins": self.origins, 398 | "destinations": self.destinations, 399 | "network_data_source": self.local_nd, 400 | "travel_mode": self.local_tm_time, 401 | "output_format": "CSV files", 402 | "output_od_location": out_folder, 403 | "max_origins": 20, 404 | "max_destinations": 20, 405 | "max_processes": 4, 406 | "time_units": "Minutes", 407 | "distance_units": "Miles", 408 | "cutoff": 30, 409 | "num_destinations": 2, 410 | "time_of_day": None, 411 | "barriers": [] 412 | } 413 | 414 | # Run parallel process. This calculates the OD and also post-processes the results 415 | od_calculator = parallel_odcm.ParallelODCalculator(**inputs) 416 | od_calculator.solve_od_in_parallel() 417 | 418 | # Check results 419 | csv_files = glob(os.path.join(out_folder, "*.csv")) 420 | self.assertEqual(11, len(csv_files), "Incorrect number of CSV files produced.") 421 | df = pd.concat(map(pd.read_csv, csv_files), ignore_index=True) 422 | # With 2 destinations for each origin, expect 414 rows in the output 423 | # Note: 1 origin finds no destinations, and that's why we don't have 416. 424 | self.assertEqual(414, df.shape[0], "Incorrect number of rows in combined output CSV files.") 425 | 426 | @unittest.skipIf(not TEST_ARROW, "Arrow table output is not available in versions of Pro prior to 2.9.") 427 | def test_ParallelODCalculator_solve_od_in_parallel_arrow(self): 428 | """Test the solve_od_in_parallel function. Output to Arrow files.""" 429 | out_folder = os.path.join(self.scratch_folder, "ParallelODCalculator_Arrow") 430 | os.mkdir(out_folder) 431 | inputs = { 432 | "logger": self.logger, 433 | "origins": self.origins, 434 | "destinations": self.destinations, 435 | "network_data_source": self.local_nd, 436 | "travel_mode": self.local_tm_time, 437 | "output_format": "Apache Arrow files", 438 | "output_od_location": out_folder, 439 | "max_origins": 20, 440 | "max_destinations": 20, 441 | "max_processes": 4, 442 | "time_units": "Minutes", 443 | "distance_units": "Miles", 444 | "cutoff": 30, 445 | "num_destinations": 2, 446 | "time_of_day": None, 447 | "barriers": [] 448 | } 449 | 450 | # Run parallel process. This calculates the OD and also post-processes the results 451 | od_calculator = parallel_odcm.ParallelODCalculator(**inputs) 452 | od_calculator.solve_od_in_parallel() 453 | 454 | # Check results 455 | arrow_files = glob(os.path.join(out_folder, "*.arrow")) 456 | self.assertEqual(11, len(arrow_files), "Incorrect number of CSV files produced.") 457 | arrow_dfs = [] 458 | for arrow_file in arrow_files: 459 | with pa.memory_map(arrow_file, 'r') as source: 460 | batch_reader = pa.ipc.RecordBatchFileReader(source) 461 | chunk_table = batch_reader.read_all() 462 | arrow_dfs.append(chunk_table.to_pandas(split_blocks=True)) 463 | df = pd.concat(arrow_dfs, ignore_index=True) 464 | # With 2 destinations for each origin, expect 414 rows in the output 465 | # Note: 1 origin finds no destinations, and that's why we don't have 416. 466 | self.assertEqual(414, df.shape[0], "Incorrect number of rows in combined output Arrow files.") 467 | 468 | 469 | if __name__ == '__main__': 470 | unittest.main() 471 | -------------------------------------------------------------------------------- /unittests/test_parallel_route_pairs.py: -------------------------------------------------------------------------------- 1 | """Unit tests for the parallel_route_pairs.py module. 2 | 3 | Copyright 2023 Esri 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | """ 14 | # pylint: disable=import-error, protected-access, invalid-name 15 | 16 | import sys 17 | import os 18 | import datetime 19 | import subprocess 20 | import unittest 21 | from copy import deepcopy 22 | import pandas as pd 23 | import arcpy 24 | import portal_credentials 25 | import input_data_helper 26 | 27 | CWD = os.path.dirname(os.path.abspath(__file__)) 28 | sys.path.append(os.path.dirname(CWD)) 29 | import parallel_route_pairs # noqa: E402, pylint: disable=wrong-import-position 30 | from helpers import arcgis_version, PreassignedODPairType, configure_global_logger, teardown_logger 31 | 32 | 33 | class TestParallelRoutePairs(unittest.TestCase): 34 | """Test cases for the parallel_route_pairs module.""" 35 | 36 | @classmethod 37 | def setUpClass(self): # pylint: disable=bad-classmethod-argument 38 | """Set up shared test properties.""" 39 | self.maxDiff = None 40 | 41 | self.input_data_folder = os.path.join(CWD, "TestInput") 42 | sf_gdb = os.path.join(self.input_data_folder, "SanFrancisco.gdb") 43 | self.origins = input_data_helper.get_tract_centroids_with_store_id_fc(sf_gdb) 44 | self.destinations = os.path.join(sf_gdb, "Analysis", "Stores") 45 | self.od_pair_table = input_data_helper.get_od_pair_csv(self.input_data_folder) 46 | self.local_nd = os.path.join(sf_gdb, "Transportation", "Streets_ND") 47 | self.local_tm_time = "Driving Time" 48 | self.local_tm_dist = "Driving Distance" 49 | self.portal_nd = portal_credentials.PORTAL_URL 50 | self.portal_tm = portal_credentials.PORTAL_TRAVEL_MODE 51 | 52 | arcpy.SignInToPortal(self.portal_nd, portal_credentials.PORTAL_USERNAME, portal_credentials.PORTAL_PASSWORD) 53 | 54 | # Create a unique output directory and gdb for this test 55 | self.output_folder = os.path.join( 56 | CWD, "TestOutput", "Output_ParallelRoutePairs_" + datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) 57 | os.makedirs(self.output_folder) 58 | self.output_gdb = os.path.join(self.output_folder, "outputs.gdb") 59 | arcpy.management.CreateFileGDB(os.path.dirname(self.output_gdb), os.path.basename(self.output_gdb)) 60 | self.logger = configure_global_logger(parallel_route_pairs.LOG_LEVEL) 61 | 62 | self.route_args_one_to_one = { 63 | "pair_type": PreassignedODPairType.one_to_one, 64 | "origins": self.origins, 65 | "origin_id_field": "ID", 66 | "destinations": self.destinations, 67 | "dest_id_field": "NAME", 68 | "network_data_source": self.local_nd, 69 | "travel_mode": self.local_tm_dist, 70 | "time_units": arcpy.nax.TimeUnits.Minutes, 71 | "distance_units": arcpy.nax.DistanceUnits.Miles, 72 | "time_of_day": None, 73 | "reverse_direction": False, 74 | "scratch_folder": self.output_folder, 75 | "assigned_dest_field": "StoreID", 76 | "od_pair_table": None, 77 | "origin_transfer_fields": [], 78 | "destination_transfer_fields": [], 79 | "barriers": [] 80 | } 81 | self.route_args_many_to_many = { 82 | "pair_type": PreassignedODPairType.many_to_many, 83 | "origins": self.origins, 84 | "origin_id_field": "ID", 85 | "destinations": self.destinations, 86 | "dest_id_field": "NAME", 87 | "network_data_source": self.local_nd, 88 | "travel_mode": self.local_tm_dist, 89 | "time_units": arcpy.nax.TimeUnits.Minutes, 90 | "distance_units": arcpy.nax.DistanceUnits.Miles, 91 | "time_of_day": None, 92 | "reverse_direction": False, 93 | "scratch_folder": self.output_folder, 94 | "assigned_dest_field": None, 95 | "od_pair_table": self.od_pair_table, 96 | "origin_transfer_fields": [], 97 | "destination_transfer_fields": [], 98 | "barriers": [] 99 | } 100 | self.parallel_rt_class_args_one_to_one = { 101 | "logger": self.logger, 102 | "pair_type_str": "one_to_one", 103 | "origins": self.origins, 104 | "origin_id_field": "ID", 105 | "destinations": self.destinations, 106 | "dest_id_field": "NAME", 107 | "network_data_source": self.local_nd, 108 | "travel_mode": self.local_tm_dist, 109 | "time_units": "Minutes", 110 | "distance_units": "Miles", 111 | "max_routes": 15, 112 | "max_processes": 4, 113 | "out_routes": os.path.join(self.output_gdb, "OutRoutes_OneToOne"), 114 | "reverse_direction": False, 115 | "scratch_folder": self.output_folder, # Should be set within test if real output will be written 116 | "assigned_dest_field": "StoreID", 117 | "od_pair_table": None, 118 | "time_of_day": "20220329 16:45", 119 | "barriers": "" 120 | } 121 | self.parallel_rt_class_args_many_to_many = { 122 | "logger": self.logger, 123 | "pair_type_str": "many_to_many", 124 | "origins": self.origins, 125 | "origin_id_field": "ID", 126 | "destinations": self.destinations, 127 | "dest_id_field": "NAME", 128 | "network_data_source": self.local_nd, 129 | "travel_mode": self.local_tm_dist, 130 | "time_units": "Minutes", 131 | "distance_units": "Miles", 132 | "max_routes": 15, 133 | "max_processes": 4, 134 | "out_routes": os.path.join(self.output_gdb, "OutRoutes_ManyToMany"), 135 | "reverse_direction": False, 136 | "scratch_folder": self.output_folder, # Should be set within test if real output will be written 137 | "assigned_dest_field": None, 138 | "od_pair_table": self.od_pair_table, 139 | "time_of_day": "20220329 16:45", 140 | "barriers": "" 141 | } 142 | 143 | @classmethod 144 | def tearDownClass(self): 145 | """Deconstruct the logger when tests are finished.""" 146 | teardown_logger(self.logger) 147 | 148 | def test_Route_get_od_pairs_for_chunk(self): 149 | """Test the _get_od_pairs_for_chunk method of the Route class.""" 150 | rt = parallel_route_pairs.Route(**self.route_args_many_to_many) 151 | chunk_size = 10 152 | chunk_num = 3 153 | # Get the third chunk in the OD pairs file 154 | chunk_definition = [chunk_num, chunk_size] 155 | rt._get_od_pairs_for_chunk(chunk_definition) 156 | # Make sure we have the right number of OD pairs 157 | self.assertEqual(chunk_size, len(rt.od_pairs)) 158 | # Verify that the solver's OD pairs are the right ones. 159 | df_od_pairs = pd.read_csv( 160 | self.od_pair_table, 161 | header=None, 162 | dtype=str 163 | ) 164 | chunk_start = chunk_num * chunk_size 165 | expected_od_pairs = df_od_pairs.loc[chunk_start:chunk_start + chunk_size - 1].values.tolist() 166 | self.assertEqual(expected_od_pairs, rt.od_pairs) 167 | 168 | def test_Route_select_inputs_one_to_one(self): 169 | """Test the _select_inputs_one_to_one method of the Route class.""" 170 | rt = parallel_route_pairs.Route(**self.route_args_one_to_one) 171 | origin_criteria = [4, 9] # Encompasses 6 rows 172 | rt._select_inputs_one_to_one(origin_criteria) 173 | self.assertEqual(6, int(arcpy.management.GetCount(rt.input_origins_layer_obj).getOutput(0))) 174 | 175 | def test_Route_select_inputs_many_to_many_str(self): 176 | """Test the _select_inputs_many_to_many method of the Route class using string-type ID fields.""" 177 | rt = parallel_route_pairs.Route(**self.route_args_many_to_many) 178 | rt.od_pairs = [ # 3 unique origins, 5 unique destinations 179 | ["06075060700", "Store_19"], 180 | ["06081601400", "Store_7"], 181 | ["06081601400", "Store_15"], 182 | ["06081601400", "Store_2"], 183 | ["06075023001", "Store_15"], 184 | ["06075023001", "Store_25"] 185 | ] 186 | rt._select_inputs_many_to_many() 187 | self.assertEqual(3, int(arcpy.management.GetCount(rt.input_origins_layer_obj).getOutput(0))) 188 | self.assertEqual(5, int(arcpy.management.GetCount(rt.input_dests_layer_obj).getOutput(0))) 189 | 190 | def test_Route_select_inputs_many_to_many_num(self): 191 | """Test the _select_inputs_many_to_many method of the Route class using numerical ID fields.""" 192 | rt = parallel_route_pairs.Route(**self.route_args_many_to_many) 193 | rt.origin_id_field = "ObjectID" 194 | rt.dest_id_field = "ObjectID" 195 | rt.od_pairs = [ # 3 unique origins, 5 unique destinations 196 | [2, 16], 197 | [4, 19], 198 | [4, 5], 199 | [4, 10], 200 | [7, 5], 201 | [7, 25] 202 | ] 203 | rt._select_inputs_many_to_many() 204 | self.assertEqual(3, int(arcpy.management.GetCount(rt.input_origins_layer_obj).getOutput(0))) 205 | self.assertEqual(5, int(arcpy.management.GetCount(rt.input_dests_layer_obj).getOutput(0))) 206 | 207 | def test_Route_solve_one_to_one(self): 208 | """Test the solve method of the Route class with feature class output for the one-to-one pair type.""" 209 | # Initialize an Route analysis object 210 | rt = parallel_route_pairs.Route(**self.route_args_one_to_one) 211 | # Solve a chunk 212 | origin_criteria = [2, 12] # 11 rows 213 | rt.solve(origin_criteria) 214 | # Check results 215 | self.assertIsInstance(rt.job_result, dict) 216 | self.assertTrue(rt.job_result["solveSucceeded"], "Route solve failed") 217 | self.assertTrue(arcpy.Exists(rt.job_result["outputRoutes"]), "Route output does not exist.") 218 | # Expect 9 rows because two of the StoreID values are bad and are skipped 219 | self.assertEqual(9, int(arcpy.management.GetCount(rt.job_result["outputRoutes"]).getOutput(0))) 220 | # Make sure the ID fields have been added and populated 221 | route_fields = [f.name for f in arcpy.ListFields(rt.job_result["outputRoutes"])] 222 | self.assertIn("OriginUniqueID", route_fields, "Routes output missing OriginUniqueID field.") 223 | self.assertIn("DestinationUniqueID", route_fields, "Routes output missing DestinationUniqueID field.") 224 | for row in arcpy.da.SearchCursor( # pylint: disable=no-member 225 | rt.job_result["outputRoutes"], ["OriginUniqueID", "DestinationUniqueID"] 226 | ): 227 | self.assertIsNotNone(row[0], "Null OriginUniqueID field value in output routes.") 228 | self.assertIsNotNone(row[1], "Null DestinationUniqueID field value in output routes.") 229 | 230 | def test_Route_solve_many_to_many(self): 231 | """Test the solve method of the Route class with feature class output for the many-to-many pair type.""" 232 | # Initialize an Route analysis object 233 | rt = parallel_route_pairs.Route(**self.route_args_many_to_many) 234 | # Solve a chunk 235 | chunk_size = 10 236 | chunk_num = 2 # Corresponds to OD pairs 2-20 237 | chunk_definition = [chunk_num, chunk_size] 238 | rt.solve(chunk_definition) 239 | # Check results 240 | self.assertIsInstance(rt.job_result, dict) 241 | self.assertTrue(rt.job_result["solveSucceeded"], "Route solve failed") 242 | self.assertTrue(arcpy.Exists(rt.job_result["outputRoutes"]), "Route output does not exist.") 243 | # Check for correct number of routes 244 | self.assertEqual(chunk_size, int(arcpy.management.GetCount(rt.job_result["outputRoutes"]).getOutput(0))) 245 | # Make sure the ID fields have been added and populated 246 | route_fields = [f.name for f in arcpy.ListFields(rt.job_result["outputRoutes"])] 247 | self.assertIn("OriginUniqueID", route_fields, "Routes output missing OriginUniqueID field.") 248 | self.assertIn("DestinationUniqueID", route_fields, "Routes output missing DestinationUniqueID field.") 249 | df_od_pairs = pd.read_csv( 250 | self.od_pair_table, 251 | header=None, 252 | skiprows=chunk_size*chunk_num, 253 | nrows=chunk_size, 254 | dtype=str 255 | ) 256 | expected_origin_ids = df_od_pairs[0].unique().tolist() 257 | expected_dest_ids = df_od_pairs[1].unique().tolist() 258 | for row in arcpy.da.SearchCursor( # pylint: disable=no-member 259 | rt.job_result["outputRoutes"], ["OriginUniqueID", "DestinationUniqueID"] 260 | ): 261 | self.assertIsNotNone(row[0], "Null OriginUniqueID field value in output routes.") 262 | self.assertIn(row[0], expected_origin_ids, "OriginUniqueID does not match expected list of IDs.") 263 | self.assertIsNotNone(row[1], "Null DestinationUniqueID field value in output routes.") 264 | self.assertIn(row[1], expected_dest_ids, "DestinationUniqueID does not match expected list of IDs.") 265 | 266 | def test_Route_solve_service_one_to_one(self): 267 | """Test the solve method of the Route class with feature class output using a service.""" 268 | # Initialize an Route analysis object 269 | route_args = deepcopy(self.route_args_one_to_one) 270 | route_args["network_data_source"] = self.portal_nd 271 | route_args["travel_mode"] = self.portal_tm 272 | rt = parallel_route_pairs.Route(**route_args) 273 | # Solve a chunk 274 | origin_criteria = [2, 12] # 11 rows 275 | rt.solve(origin_criteria) 276 | # Check results 277 | self.assertIsInstance(rt.job_result, dict) 278 | self.assertTrue(rt.job_result["solveSucceeded"], "Route solve failed") 279 | self.assertTrue(arcpy.Exists(rt.job_result["outputRoutes"]), "Route output does not exist.") 280 | # Expect 9 rows because two of the StoreID values are bad and are skipped 281 | self.assertEqual(9, int(arcpy.management.GetCount(rt.job_result["outputRoutes"]).getOutput(0))) 282 | # Make sure the ID fields have been added and populated 283 | route_fields = [f.name for f in arcpy.ListFields(rt.job_result["outputRoutes"])] 284 | self.assertIn("OriginUniqueID", route_fields, "Routes output missing OriginUniqueID field.") 285 | self.assertIn("DestinationUniqueID", route_fields, "Routes output missing DestinationUniqueID field.") 286 | for row in arcpy.da.SearchCursor( # pylint: disable=no-member 287 | rt.job_result["outputRoutes"], ["OriginUniqueID", "DestinationUniqueID"] 288 | ): 289 | self.assertIsNotNone(row[0], "Null OriginUniqueID field value in output routes.") 290 | self.assertIsNotNone(row[1], "Null DestinationUniqueID field value in output routes.") 291 | 292 | def test_ParallelRoutePairCalculator_validate_route_settings(self): 293 | """Test the _validate_route_settings function.""" 294 | # Test that with good inputs, we return the correct optimized field name 295 | rt_calculator = parallel_route_pairs.ParallelRoutePairCalculator(**self.parallel_rt_class_args_one_to_one) 296 | rt_calculator._validate_route_settings() 297 | # Test completely invalid travel mode 298 | rt_inputs = deepcopy(self.parallel_rt_class_args_one_to_one) 299 | rt_inputs["travel_mode"] = "InvalidTM" 300 | rt_calculator = parallel_route_pairs.ParallelRoutePairCalculator(**rt_inputs) 301 | error_type = ValueError if arcgis_version >= "3.1" else RuntimeError 302 | with self.assertRaises(error_type): 303 | rt_calculator._validate_route_settings() 304 | 305 | def test_ParallelRoutePairCalculator_solve_route_in_parallel_one_to_one(self): 306 | """Test the solve_od_in_parallel function with the one-to-one pair type. Output to feature class.""" 307 | out_routes = os.path.join(self.output_gdb, "Out_Combined_Routes_OneToOne") 308 | scratch_folder = os.path.join(self.output_folder, "Out_Combined_Routes_OneToOne") 309 | os.mkdir(scratch_folder) 310 | rt_inputs = deepcopy(self.parallel_rt_class_args_one_to_one) 311 | rt_inputs["out_routes"] = out_routes 312 | rt_inputs["scratch_folder"] = scratch_folder 313 | 314 | # Run parallel process. This calculates the OD and also post-processes the results 315 | rt_calculator = parallel_route_pairs.ParallelRoutePairCalculator(**rt_inputs) 316 | rt_calculator.solve_route_in_parallel() 317 | 318 | # Check results 319 | self.assertTrue(arcpy.Exists(out_routes)) 320 | # There are 208 tract centroids, but three of them have null or invalid assigned destinations 321 | self.assertEqual(205, int(arcpy.management.GetCount(out_routes).getOutput(0))) 322 | 323 | def test_ParallelRoutePairCalculator_solve_route_in_parallel_many_to_many(self): 324 | """Test the solve_od_in_parallel function with the many-to-many pair type. Output to feature class.""" 325 | out_routes = os.path.join(self.output_gdb, "Out_Combined_Routes_ManyToMany") 326 | scratch_folder = os.path.join(self.output_folder, "Out_Combined_Routes_ManyToMany") 327 | os.mkdir(scratch_folder) 328 | rt_inputs = deepcopy(self.parallel_rt_class_args_many_to_many) 329 | rt_inputs["out_routes"] = out_routes 330 | rt_inputs["scratch_folder"] = scratch_folder 331 | 332 | # Run parallel process. This calculates the OD and also post-processes the results 333 | rt_calculator = parallel_route_pairs.ParallelRoutePairCalculator(**rt_inputs) 334 | rt_calculator.solve_route_in_parallel() 335 | 336 | # Check results 337 | self.assertTrue(arcpy.Exists(out_routes)) 338 | # The CSV file has 63 lines, so we should get 63 routes. 339 | self.assertEqual(63, int(arcpy.management.GetCount(out_routes).getOutput(0))) 340 | 341 | def test_cli_one_to_one(self): 342 | """Test the command line interface of and launch_parallel_rt_pairs function for the one-to-one pair type.""" 343 | out_folder = os.path.join(self.output_folder, "CLI_Output_OneToOne") 344 | os.mkdir(out_folder) 345 | out_routes = os.path.join(self.output_gdb, "OutCLIRoutes_OneToOne") 346 | rt_inputs = [ 347 | os.path.join(sys.exec_prefix, "python.exe"), 348 | os.path.join(os.path.dirname(CWD), "parallel_route_pairs.py"), 349 | "--pair-type", "one_to_one", 350 | "--origins", self.origins, 351 | "--origins-id-field", "ID", 352 | "--destinations", self.destinations, 353 | "--destinations-id-field", "NAME", 354 | "--network-data-source", self.local_nd, 355 | "--travel-mode", self.local_tm_dist, 356 | "--time-units", "Minutes", 357 | "--distance-units", "Miles", 358 | "--max-routes", "15", 359 | "--max-processes", "4", 360 | "--out-routes", out_routes, 361 | "--reverse-direction", "false", 362 | "--scratch-folder", out_folder, 363 | "--assigned-dest-field", "StoreID", 364 | "--time-of-day", "20220329 16:45" 365 | ] 366 | result = subprocess.run(rt_inputs, check=True) 367 | self.assertEqual(result.returncode, 0) 368 | self.assertTrue(arcpy.Exists(out_routes)) 369 | 370 | def test_cli_many_to_many(self): 371 | """Test the command line interface of and launch_parallel_rt_pairs function for the many-to-many.""" 372 | out_folder = os.path.join(self.output_folder, "CLI_Output_ManyToMany") 373 | os.mkdir(out_folder) 374 | out_routes = os.path.join(self.output_gdb, "OutCLIRoutes_ManyToMany") 375 | rt_inputs = [ 376 | os.path.join(sys.exec_prefix, "python.exe"), 377 | os.path.join(os.path.dirname(CWD), "parallel_route_pairs.py"), 378 | "--pair-type", "many_to_many", 379 | "--origins", self.origins, 380 | "--origins-id-field", "ID", 381 | "--destinations", self.destinations, 382 | "--destinations-id-field", "NAME", 383 | "--network-data-source", self.local_nd, 384 | "--travel-mode", self.local_tm_dist, 385 | "--time-units", "Minutes", 386 | "--distance-units", "Miles", 387 | "--max-routes", "15", 388 | "--max-processes", "4", 389 | "--out-routes", out_routes, 390 | "--reverse-direction", "false", 391 | "--scratch-folder", out_folder, 392 | "--od-pair-table", self.od_pair_table, 393 | "--time-of-day", "20220329 16:45" 394 | ] 395 | result = subprocess.run(rt_inputs, check=True) 396 | self.assertEqual(result.returncode, 0) 397 | self.assertTrue(arcpy.Exists(out_routes)) 398 | 399 | 400 | if __name__ == '__main__': 401 | unittest.main() 402 | -------------------------------------------------------------------------------- /unittests/test_solve_large_odcm.py: -------------------------------------------------------------------------------- 1 | """Unit tests for the solve_large_odcm.py module. 2 | 3 | Copyright 2023 Esri 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | """ 14 | # pylint: disable=import-error, protected-access, invalid-name 15 | 16 | import sys 17 | import os 18 | import datetime 19 | import subprocess 20 | import unittest 21 | from copy import deepcopy 22 | from glob import glob 23 | import arcpy 24 | import portal_credentials # Contains log-in for an ArcGIS Online account to use as a test portal 25 | 26 | CWD = os.path.dirname(os.path.abspath(__file__)) 27 | sys.path.append(os.path.dirname(CWD)) 28 | import solve_large_odcm # noqa: E402, pylint: disable=wrong-import-position 29 | import helpers # noqa: E402, pylint: disable=wrong-import-position 30 | from helpers import arcgis_version, MAX_ALLOWED_MAX_PROCESSES # noqa: E402, pylint: disable=wrong-import-position 31 | 32 | 33 | class TestSolveLargeODCM(unittest.TestCase): 34 | """Test cases for the solve_large_odcm module.""" 35 | 36 | @classmethod 37 | def setUpClass(self): # pylint: disable=bad-classmethod-argument 38 | self.maxDiff = None 39 | 40 | self.input_data_folder = os.path.join(CWD, "TestInput") 41 | self.sf_gdb = os.path.join(self.input_data_folder, "SanFrancisco.gdb") 42 | self.origins = os.path.join(self.sf_gdb, "Analysis", "TractCentroids") 43 | self.destinations = os.path.join(self.sf_gdb, "Analysis", "Hospitals") 44 | self.local_nd = os.path.join(self.sf_gdb, "Transportation", "Streets_ND") 45 | self.local_tm_time = "Driving Time" 46 | self.local_tm_dist = "Driving Distance" 47 | self.portal_nd = portal_credentials.PORTAL_URL 48 | self.portal_tm = portal_credentials.PORTAL_TRAVEL_MODE 49 | self.time_of_day_str = "20220329 16:45" 50 | 51 | arcpy.SignInToPortal(self.portal_nd, portal_credentials.PORTAL_USERNAME, portal_credentials.PORTAL_PASSWORD) 52 | 53 | # Create a unique output directory and gdb for this test 54 | self.scratch_folder = os.path.join( 55 | CWD, "TestOutput", "Output_SolveLargeODCM_" + datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) 56 | os.makedirs(self.scratch_folder) 57 | self.output_gdb = os.path.join(self.scratch_folder, "outputs.gdb") 58 | arcpy.management.CreateFileGDB(os.path.dirname(self.output_gdb), os.path.basename(self.output_gdb)) 59 | 60 | # Copy some data to the output gdb to serve as barriers. Do not use tutorial data directly as input because 61 | # the tests will write network location fields to it, and we don't want to modify the user's original data. 62 | self.barriers = os.path.join(self.output_gdb, "Barriers") 63 | arcpy.management.Copy(os.path.join(self.sf_gdb, "Analysis", "CentralDepots"), self.barriers) 64 | 65 | self.od_args = { 66 | "origins": self.origins, 67 | "destinations": self.destinations, 68 | "network_data_source": self.local_nd, 69 | "travel_mode": self.local_tm_dist, 70 | "output_origins": os.path.join(self.output_gdb, "OutOrigins"), 71 | "output_destinations": os.path.join(self.output_gdb, "OutDestinations"), 72 | "chunk_size": 20, 73 | "max_processes": 4, 74 | "time_units": "Minutes", 75 | "distance_units": "Miles", 76 | "output_format": "Feature class", 77 | "output_od_lines": os.path.join(self.output_gdb, "OutODLines"), 78 | "output_data_folder": None, 79 | "cutoff": 2, 80 | "num_destinations": 1, 81 | "time_of_day": None, 82 | "precalculate_network_locations": True, 83 | "sort_inputs": True, 84 | "barriers": [self.barriers] 85 | } 86 | 87 | def test_validate_inputs(self): 88 | """Test the validate_inputs function.""" 89 | does_not_exist = os.path.join(self.sf_gdb, "Analysis", "DoesNotExist") 90 | invalid_inputs = [ 91 | ("chunk_size", -5, ValueError, "Chunk size must be greater than 0."), 92 | ("max_processes", 0, ValueError, "Maximum allowed parallel processes must be greater than 0."), 93 | ("max_processes", 5000, ValueError, ( 94 | f"The maximum allowed parallel processes cannot exceed {MAX_ALLOWED_MAX_PROCESSES:} due " 95 | "to limitations imposed by Python's concurrent.futures module." 96 | )), 97 | ("time_units", "BadUnits", ValueError, "Invalid time units: BadUnits"), 98 | ("distance_units", "BadUnits", ValueError, "Invalid distance units: BadUnits"), 99 | ("origins", does_not_exist, ValueError, f"Input dataset {does_not_exist} does not exist."), 100 | ("destinations", does_not_exist, ValueError, f"Input dataset {does_not_exist} does not exist."), 101 | ("barriers", [does_not_exist], ValueError, f"Input dataset {does_not_exist} does not exist."), 102 | ("network_data_source", does_not_exist, ValueError, 103 | f"Input network dataset {does_not_exist} does not exist."), 104 | ("travel_mode", "BadTM", ValueError if arcgis_version >= "3.1" else RuntimeError, ""), 105 | ("time_of_day", "3/29/2022 4:45 PM", ValueError, ""), 106 | ("time_of_day", "BadDateTime", ValueError, ""), 107 | ("cutoff", 0, ValueError, "Impedance cutoff must be greater than 0."), 108 | ("cutoff", -5, ValueError, "Impedance cutoff must be greater than 0."), 109 | ("num_destinations", 0, ValueError, "Number of destinations to find must be greater than 0."), 110 | ("num_destinations", -5, ValueError, "Number of destinations to find must be greater than 0."), 111 | ] 112 | for invalid_input in invalid_inputs: 113 | property_name, value, error_type, expected_message = invalid_input 114 | with self.subTest( 115 | property_name=property_name, value=value, error_type=error_type, expected_message=expected_message 116 | ): 117 | inputs = deepcopy(self.od_args) 118 | inputs[property_name] = value 119 | od_solver = solve_large_odcm.ODCostMatrixSolver(**inputs) 120 | with self.assertRaises(error_type) as ex: 121 | od_solver._validate_inputs() 122 | if expected_message: 123 | self.assertEqual(expected_message, str(ex.exception)) 124 | 125 | # Check validation of missing output feature class or folder location depending on output format 126 | for output_format in helpers.OUTPUT_FORMATS: 127 | if output_format == "Feature class": 128 | output_od_lines = None 129 | output_data_folder = "Stuff" 130 | else: 131 | output_od_lines = "Stuff" 132 | output_data_folder = None 133 | with self.subTest(output_format=output_format): 134 | inputs = deepcopy(self.od_args) 135 | inputs["output_format"] = output_format 136 | inputs["output_od_lines"] = output_od_lines 137 | inputs["output_data_folder"] = output_data_folder 138 | od_solver = solve_large_odcm.ODCostMatrixSolver(**inputs) 139 | with self.assertRaises(ValueError): 140 | od_solver._validate_inputs() 141 | 142 | # Check validation when the network data source is a service and Arrow output is requested 143 | # Arrow output from services is not yet supported. 144 | output_format = "Apache Arrow files" 145 | with self.subTest(output_format=output_format, network_data_source=self.portal_nd): 146 | inputs = deepcopy(self.od_args) 147 | inputs["output_format"] = output_format 148 | inputs["output_data_folder"] = "Stuff" 149 | inputs["network_data_source"] = self.portal_nd 150 | od_solver = solve_large_odcm.ODCostMatrixSolver(**inputs) 151 | with self.assertRaises(ValueError): 152 | od_solver._validate_inputs() 153 | 154 | def test_update_max_inputs_for_service(self): 155 | """Test the update_max_inputs_for_service function.""" 156 | max_origins = 1500 157 | max_destinations = 700 158 | 159 | inputs = deepcopy(self.od_args) 160 | inputs["network_data_source"] = self.portal_nd 161 | inputs["travel_mode"] = self.portal_tm 162 | od_solver = solve_large_odcm.ODCostMatrixSolver(**inputs) 163 | od_solver.max_origins = max_origins 164 | od_solver.max_destinations = max_destinations 165 | tool_limits = { 166 | 'forceHierarchyBeyondDistance': 50.0, 167 | 'forceHierarchyBeyondDistanceUnits': 168 | 'Miles', 169 | 'maximumDestinations': 1000.0, 170 | 'maximumFeaturesAffectedByLineBarriers': 500.0, 171 | 'maximumFeaturesAffectedByPointBarriers': 250.0, 172 | 'maximumFeaturesAffectedByPolygonBarriers': 2000.0, 173 | 'maximumGeodesicDistanceUnitsWhenWalking': 'Miles', 174 | 'maximumGeodesicDistanceWhenWalking': 27.0, 175 | 'maximumOrigins': 1000.0 176 | } 177 | od_solver.service_limits = tool_limits 178 | od_solver._update_max_inputs_for_service() 179 | self.assertEqual(tool_limits["maximumOrigins"], od_solver.max_origins) 180 | self.assertEqual(max_destinations, od_solver.max_destinations) 181 | 182 | # Test when there are no limits 183 | tool_limits = { 184 | 'forceHierarchyBeyondDistance': 50.0, 185 | 'forceHierarchyBeyondDistanceUnits': 186 | 'Miles', 187 | 'maximumDestinations': None, 188 | 'maximumFeaturesAffectedByLineBarriers': 500.0, 189 | 'maximumFeaturesAffectedByPointBarriers': 250.0, 190 | 'maximumFeaturesAffectedByPolygonBarriers': 2000.0, 191 | 'maximumGeodesicDistanceUnitsWhenWalking': 'Miles', 192 | 'maximumGeodesicDistanceWhenWalking': 27.0, 193 | 'maximumOrigins': None 194 | } 195 | od_solver.max_origins = max_origins 196 | od_solver.max_destinations = max_destinations 197 | od_solver.service_limits = tool_limits 198 | od_solver._update_max_inputs_for_service() 199 | self.assertEqual(max_origins, od_solver.max_origins) 200 | self.assertEqual(max_destinations, od_solver.max_destinations) 201 | 202 | def test_add_tracked_oid_field(self): 203 | """Test the _add_tracked_oid_field method.""" 204 | od_solver = solve_large_odcm.ODCostMatrixSolver(**self.od_args) 205 | fc_to_modify = os.path.join(self.output_gdb, "TrackedID") 206 | arcpy.management.Copy(self.destinations, fc_to_modify) 207 | od_solver._add_tracked_oid_field(fc_to_modify, "DestinationOID") 208 | self.assertTrue(arcpy.Exists(fc_to_modify)) 209 | self.assertIn("DestinationOID", [f.name for f in arcpy.ListFields(fc_to_modify)]) 210 | 211 | def test_spatially_sort_input(self): 212 | """Test the _spatially_sort_input method.""" 213 | od_solver = solve_large_odcm.ODCostMatrixSolver(**self.od_args) 214 | fc_to_sort = os.path.join(self.output_gdb, "Sorted") 215 | arcpy.management.Copy(self.destinations, fc_to_sort) 216 | od_solver._spatially_sort_input(fc_to_sort) 217 | self.assertTrue(arcpy.Exists(fc_to_sort)) 218 | 219 | def test_solve_large_od_cost_matrix_featureclass(self): 220 | """Test the full solve OD Cost Matrix workflow with feature class output.""" 221 | od_solver = solve_large_odcm.ODCostMatrixSolver(**self.od_args) 222 | od_solver.solve_large_od_cost_matrix() 223 | self.assertTrue(arcpy.Exists(self.od_args["output_od_lines"])) 224 | self.assertTrue(arcpy.Exists(self.od_args["output_origins"])) 225 | self.assertTrue(arcpy.Exists(self.od_args["output_destinations"])) 226 | 227 | def test_solve_large_od_cost_matrix_same_inputs_csv(self): 228 | """Test the full solve OD Cost Matrix workflow when origins and destinations are the same. Use CSV outputs.""" 229 | out_folder = os.path.join(self.scratch_folder, "FullWorkflow_CSV_SameInputs") 230 | os.mkdir(out_folder) 231 | od_args = { 232 | "origins": self.destinations, 233 | "destinations": self.destinations, 234 | "network_data_source": self.local_nd, 235 | "travel_mode": self.local_tm_dist, 236 | "output_origins": os.path.join(self.output_gdb, "OutOriginsSame"), 237 | "output_destinations": os.path.join(self.output_gdb, "OutDestinationsSame"), 238 | "chunk_size": 50, 239 | "max_processes": 4, 240 | "time_units": "Minutes", 241 | "distance_units": "Miles", 242 | "output_format": "CSV files", 243 | "output_od_lines": None, 244 | "output_data_folder": out_folder, 245 | "cutoff": 2, 246 | "num_destinations": 2, 247 | "time_of_day": self.time_of_day_str, 248 | "precalculate_network_locations": True, 249 | "sort_inputs": True, 250 | "barriers": "" 251 | } 252 | od_solver = solve_large_odcm.ODCostMatrixSolver(**od_args) 253 | od_solver.solve_large_od_cost_matrix() 254 | self.assertTrue(arcpy.Exists(od_args["output_origins"])) 255 | self.assertTrue(arcpy.Exists(od_args["output_destinations"])) 256 | csv_files = glob(os.path.join(out_folder, "*.csv")) 257 | self.assertGreater(len(csv_files), 0) 258 | 259 | def test_solve_large_od_cost_matrix_arrow(self): 260 | """Test the full solve OD Cost Matrix workflow with Arrow table outputs""" 261 | out_folder = os.path.join(self.scratch_folder, "FullWorkflow_Arrow") 262 | os.mkdir(out_folder) 263 | od_args = { 264 | "origins": self.origins, 265 | "destinations": self.destinations, 266 | "network_data_source": self.local_nd, 267 | "travel_mode": self.local_tm_dist, 268 | "output_origins": os.path.join(self.output_gdb, "OutOriginsArrow"), 269 | "output_destinations": os.path.join(self.output_gdb, "OutDestinationsArrow"), 270 | "chunk_size": 50, 271 | "max_processes": 4, 272 | "time_units": "Minutes", 273 | "distance_units": "Miles", 274 | "output_format": "Apache Arrow files", 275 | "output_od_lines": None, 276 | "output_data_folder": out_folder, 277 | "cutoff": 2, 278 | "num_destinations": 2, 279 | "time_of_day": None, 280 | "precalculate_network_locations": True, 281 | "sort_inputs": True, 282 | "barriers": "" 283 | } 284 | od_solver = solve_large_odcm.ODCostMatrixSolver(**od_args) 285 | od_solver.solve_large_od_cost_matrix() 286 | self.assertTrue(arcpy.Exists(od_args["output_origins"])) 287 | self.assertTrue(arcpy.Exists(od_args["output_destinations"])) 288 | arrow_files = glob(os.path.join(out_folder, "*.arrow")) 289 | self.assertGreater(len(arrow_files), 0) 290 | 291 | def test_solve_large_od_cost_matrix_no_sorting(self): 292 | """Test the full solve OD Cost Matrix workflow without spatially sorting inputs. Use CSV outputs.""" 293 | out_folder = os.path.join(self.scratch_folder, "FullWorkflow_CSV_NoSort") 294 | os.mkdir(out_folder) 295 | od_args = { 296 | "origins": self.origins, 297 | "destinations": self.destinations, 298 | "network_data_source": self.local_nd, 299 | "travel_mode": self.local_tm_dist, 300 | "output_origins": os.path.join(self.output_gdb, "OutUnsortedOrigins"), 301 | "output_destinations": os.path.join(self.output_gdb, "OutUnsortedDestinations"), 302 | "chunk_size": 50, 303 | "max_processes": 4, 304 | "time_units": "Minutes", 305 | "distance_units": "Miles", 306 | "output_format": "CSV files", 307 | "output_od_lines": None, 308 | "output_data_folder": out_folder, 309 | "cutoff": 2, 310 | "num_destinations": 2, 311 | "time_of_day": self.time_of_day_str, 312 | "precalculate_network_locations": True, 313 | "sort_inputs": False, 314 | "barriers": "" 315 | } 316 | od_solver = solve_large_odcm.ODCostMatrixSolver(**od_args) 317 | od_solver.solve_large_od_cost_matrix() 318 | self.assertTrue(arcpy.Exists(od_args["output_origins"])) 319 | self.assertTrue(arcpy.Exists(od_args["output_destinations"])) 320 | csv_files = glob(os.path.join(out_folder, "*.csv")) 321 | self.assertGreater(len(csv_files), 0) 322 | 323 | def test_cli(self): 324 | """Test the command line interface of solve_large_odcm.""" 325 | out_folder = os.path.join(self.scratch_folder, "CLI_CSV_Output") 326 | os.mkdir(out_folder) 327 | odcm_inputs = [ 328 | os.path.join(sys.exec_prefix, "python.exe"), 329 | os.path.join(os.path.dirname(CWD), "solve_large_odcm.py"), 330 | "--origins", self.origins, 331 | "--destinations", self.destinations, 332 | "--output-origins", os.path.join(self.output_gdb, "OutOriginsCLI"), 333 | "--output-destinations", os.path.join(self.output_gdb, "OutDestinationsCLI"), 334 | "--network-data-source", self.local_nd, 335 | "--travel-mode", self.local_tm_time, 336 | "--time-units", "Minutes", 337 | "--distance-units", "Miles", 338 | "--output-format", "CSV files", 339 | "--output-data-folder", out_folder, 340 | "--chunk-size", "50", 341 | "--max-processes", "4", 342 | "--cutoff", "10", 343 | "--num-destinations", "1", 344 | "--time-of-day", self.time_of_day_str, 345 | "--precalculate-network-locations", "true", 346 | "--sort-inputs", "true", 347 | "--barriers", self.barriers 348 | ] 349 | result = subprocess.run(odcm_inputs) 350 | self.assertEqual(result.returncode, 0) 351 | 352 | 353 | if __name__ == '__main__': 354 | unittest.main() 355 | -------------------------------------------------------------------------------- /unittests/test_solve_large_route_pair_analysis.py: -------------------------------------------------------------------------------- 1 | """Unit tests for the solve_large_route_pair_analysis.py module. 2 | 3 | Copyright 2023 Esri 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | """ 14 | # pylint: disable=import-error, protected-access, invalid-name 15 | 16 | import sys 17 | import os 18 | import datetime 19 | import subprocess 20 | import unittest 21 | from copy import deepcopy 22 | import arcpy 23 | import portal_credentials # Contains log-in for an ArcGIS Online account to use as a test portal 24 | import input_data_helper 25 | 26 | CWD = os.path.dirname(os.path.abspath(__file__)) 27 | sys.path.append(os.path.dirname(CWD)) 28 | import solve_large_route_pair_analysis # noqa: E402, pylint: disable=wrong-import-position 29 | from helpers import arcgis_version, PreassignedODPairType, \ 30 | MAX_ALLOWED_MAX_PROCESSES # noqa: E402, pylint: disable=wrong-import-position 31 | 32 | 33 | class TestSolveLargeRoutePairAnalysis(unittest.TestCase): 34 | """Test cases for the solve_large_route_pair_analysis module.""" 35 | 36 | @classmethod 37 | def setUpClass(self): # pylint: disable=bad-classmethod-argument 38 | self.maxDiff = None 39 | 40 | self.input_data_folder = os.path.join(CWD, "TestInput") 41 | self.sf_gdb = os.path.join(self.input_data_folder, "SanFrancisco.gdb") 42 | self.origins = input_data_helper.get_tract_centroids_with_store_id_fc(self.sf_gdb) 43 | self.destinations = os.path.join(self.sf_gdb, "Analysis", "Stores") 44 | self.od_pairs_table = input_data_helper.get_od_pairs_fgdb_table(self.sf_gdb) 45 | self.local_nd = os.path.join(self.sf_gdb, "Transportation", "Streets_ND") 46 | self.local_tm_time = "Driving Time" 47 | self.local_tm_dist = "Driving Distance" 48 | self.portal_nd = portal_credentials.PORTAL_URL 49 | self.portal_tm = portal_credentials.PORTAL_TRAVEL_MODE 50 | self.time_of_day_str = "20220329 16:45" 51 | 52 | arcpy.SignInToPortal(self.portal_nd, portal_credentials.PORTAL_USERNAME, portal_credentials.PORTAL_PASSWORD) 53 | 54 | # Create a unique output directory and gdb for this test 55 | self.scratch_folder = os.path.join( 56 | CWD, "TestOutput", "Output_SolveLargeRoutePair_" + datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) 57 | os.makedirs(self.scratch_folder) 58 | self.output_gdb = os.path.join(self.scratch_folder, "outputs.gdb") 59 | arcpy.management.CreateFileGDB(os.path.dirname(self.output_gdb), os.path.basename(self.output_gdb)) 60 | 61 | # Copy some data to the output gdb to serve as barriers. Do not use tutorial data directly as input because 62 | # the tests will write network location fields to it, and we don't want to modify the user's original data. 63 | self.barriers = os.path.join(self.output_gdb, "Barriers") 64 | arcpy.management.Copy(os.path.join(self.sf_gdb, "Analysis", "CentralDepots"), self.barriers) 65 | 66 | self.rt_args_one_to_one = { 67 | "origins": self.origins, 68 | "origin_id_field": "ID", 69 | "destinations": self.destinations, 70 | "dest_id_field": "NAME", 71 | "pair_type": PreassignedODPairType.one_to_one, 72 | "network_data_source": self.local_nd, 73 | "travel_mode": self.local_tm_dist, 74 | "time_units": "Minutes", 75 | "distance_units": "Miles", 76 | "chunk_size": 15, 77 | "max_processes": 4, 78 | "output_routes": os.path.join(self.output_gdb, "OutRoutes_OneToOne"), 79 | "assigned_dest_field": "StoreID", 80 | "time_of_day": self.time_of_day_str, 81 | "barriers": "", 82 | "precalculate_network_locations": True, 83 | "sort_origins": True, 84 | "reverse_direction": False 85 | } 86 | self.rt_args_many_to_many = { 87 | "origins": self.origins, 88 | "origin_id_field": "ID", 89 | "destinations": self.destinations, 90 | "dest_id_field": "NAME", 91 | "pair_type": PreassignedODPairType.many_to_many, 92 | "network_data_source": self.local_nd, 93 | "travel_mode": self.local_tm_dist, 94 | "time_units": "Minutes", 95 | "distance_units": "Miles", 96 | "chunk_size": 15, 97 | "max_processes": 4, 98 | "output_routes": os.path.join(self.output_gdb, "OutRoutes_ManyToMany"), 99 | "pair_table": self.od_pairs_table, 100 | "pair_table_origin_id_field": "OriginID", 101 | "pair_table_dest_id_field": "DestinationID", 102 | "time_of_day": self.time_of_day_str, 103 | "barriers": "", 104 | "precalculate_network_locations": True, 105 | "sort_origins": False, 106 | "reverse_direction": False 107 | } 108 | 109 | def test_validate_inputs_one_to_one(self): 110 | """Test the validate_inputs function for all generic/shared cases and one-to-one pair type specific cases.""" 111 | does_not_exist = os.path.join(self.sf_gdb, "Analysis", "DoesNotExist") 112 | invalid_inputs = [ 113 | ("chunk_size", -5, ValueError, "Chunk size must be greater than 0."), 114 | ("max_processes", 0, ValueError, "Maximum allowed parallel processes must be greater than 0."), 115 | ("max_processes", 5000, ValueError, ( 116 | f"The maximum allowed parallel processes cannot exceed {MAX_ALLOWED_MAX_PROCESSES:} due " 117 | "to limitations imposed by Python's concurrent.futures module." 118 | )), 119 | ("time_units", "BadUnits", ValueError, "Invalid time units: BadUnits"), 120 | ("distance_units", "BadUnits", ValueError, "Invalid distance units: BadUnits"), 121 | ("origins", does_not_exist, ValueError, f"Input dataset {does_not_exist} does not exist."), 122 | ("destinations", does_not_exist, ValueError, f"Input dataset {does_not_exist} does not exist."), 123 | ("barriers", [does_not_exist], ValueError, f"Input dataset {does_not_exist} does not exist."), 124 | ("network_data_source", does_not_exist, ValueError, 125 | f"Input network dataset {does_not_exist} does not exist."), 126 | ("travel_mode", "BadTM", ValueError if arcgis_version >= "3.1" else RuntimeError, ""), 127 | ("time_of_day", "3/29/2022 4:45 PM", ValueError, ""), 128 | ("time_of_day", "BadDateTime", ValueError, ""), 129 | ("pair_type", "BadPairType", ValueError, "Invalid preassigned OD pair type: BadPairType"), 130 | ("origin_id_field", "BadField", ValueError, 131 | f"Unique ID field BadField does not exist in dataset {self.origins}."), 132 | ("origin_id_field", "STATE_NAME", ValueError, 133 | f"Non-unique values were found in the unique ID field STATE_NAME in {self.origins}."), 134 | ("dest_id_field", "BadField", ValueError, 135 | f"Unique ID field BadField does not exist in dataset {self.destinations}."), 136 | ("dest_id_field", "ServiceTime", ValueError, 137 | f"Non-unique values were found in the unique ID field ServiceTime in {self.destinations}."), 138 | ("assigned_dest_field", "BadField", ValueError, 139 | f"Assigned destination field BadField does not exist in Origins dataset {self.origins}."), 140 | ("assigned_dest_field", "STATE_NAME", ValueError, 141 | (f"All origins in the Origins dataset {self.origins} have invalid values in the assigned " 142 | f"destination field STATE_NAME that do not correspond to values in the " 143 | f"destinations unique ID field NAME in {self.destinations}. Ensure that you " 144 | "have chosen the correct datasets and fields and that the field types match.")), 145 | ("assigned_dest_field", None, ValueError, 146 | "Assigned destination field is required when preassigned OD pair type is " 147 | f"{PreassignedODPairType.one_to_one.name}") 148 | ] 149 | for invalid_input in invalid_inputs: 150 | property_name, value, error_type, expected_message = invalid_input 151 | with self.subTest( 152 | property_name=property_name, value=value, error_type=error_type, expected_message=expected_message 153 | ): 154 | inputs = deepcopy(self.rt_args_one_to_one) 155 | inputs[property_name] = value 156 | rt_solver = solve_large_route_pair_analysis.RoutePairSolver(**inputs) 157 | with self.assertRaises(error_type) as ex: 158 | rt_solver._validate_inputs() 159 | if expected_message: 160 | self.assertEqual(expected_message, str(ex.exception)) 161 | 162 | def test_validate_inputs_many_to_many(self): 163 | """Test the validate_inputs function for many-to-many pair type specific cases.""" 164 | does_not_exist = os.path.join(self.sf_gdb, "DoesNotExist") 165 | invalid_inputs = [ 166 | ("pair_table", None, ValueError, 167 | "Origin-destination pair table is required when preassigned OD pair type is " 168 | f"{PreassignedODPairType.many_to_many.name}"), 169 | ("pair_table", does_not_exist, ValueError, f"Input dataset {does_not_exist} does not exist."), 170 | ("pair_table_origin_id_field", None, ValueError, 171 | "Origin-destination pair table Origin ID field is required when preassigned OD pair type is " 172 | f"{PreassignedODPairType.many_to_many.name}"), 173 | ("pair_table_origin_id_field", "BadFieldName", ValueError, 174 | ("Origin-destination pair table Origin ID field BadFieldName does not exist in " 175 | f"{self.rt_args_many_to_many['pair_table']}.")), 176 | ("pair_table_dest_id_field", None, ValueError, 177 | "Origin-destination pair table Destination ID field is required when preassigned OD pair type is " 178 | f"{PreassignedODPairType.many_to_many.name}"), 179 | ("pair_table_dest_id_field", "BadFieldName", ValueError, 180 | ("Origin-destination pair table Destination ID field BadFieldName does not exist in " 181 | f"{self.rt_args_many_to_many['pair_table']}.")) 182 | ] 183 | for invalid_input in invalid_inputs: 184 | property_name, value, error_type, expected_message = invalid_input 185 | with self.subTest( 186 | property_name=property_name, value=value, error_type=error_type, expected_message=expected_message 187 | ): 188 | inputs = deepcopy(self.rt_args_many_to_many) 189 | inputs[property_name] = value 190 | rt_solver = solve_large_route_pair_analysis.RoutePairSolver(**inputs) 191 | with self.assertRaises(error_type) as ex: 192 | rt_solver._validate_inputs() 193 | if expected_message: 194 | self.assertEqual(expected_message, str(ex.exception)) 195 | 196 | def test_pair_table_errors_no_matching_ods(self): 197 | """Test for correct error when the pair table's origin or destination IDs don't match the input tables.""" 198 | inputs = deepcopy(self.rt_args_many_to_many) 199 | inputs["origin_id_field"] = "OBJECTID" 200 | rt_solver = solve_large_route_pair_analysis.RoutePairSolver(**inputs) 201 | rt_solver._validate_inputs() 202 | with self.assertRaises(ValueError) as ex: 203 | rt_solver._preprocess_inputs() 204 | self.assertIn( 205 | "All origin-destination pairs in the preassigned origin-destination pair table", 206 | str(ex.exception) 207 | ) 208 | 209 | def test_pair_table_drop_irrelevant_data(self): 210 | """Test that irrelevant data is dropped from the origins, destinations, and pair table.""" 211 | # Construct an OD pair table such that we can check for eliminated duplicate and irrelevant data 212 | in_od_pairs = [ 213 | ["06075013000", "Store_12"], 214 | ["06075013000", "Store_12"], # Duplicate of the last row. Should be eliminated 215 | ["06075013000", "Store_25"], 216 | ["06081602500", "Store_25"], 217 | ["06075030400", "Store_7"], 218 | ["06075030400", "Store_5"], 219 | ] # 3 unique origins; 4 unique destinations; 5 unique OD pairs 220 | od_pair_table = os.path.join("memory", "ODPairsDuplicates") 221 | arcpy.management.CreateTable("memory", "ODPairsDuplicates", template=self.od_pairs_table) 222 | with arcpy.da.InsertCursor( # pylint: disable=no-member 223 | od_pair_table, 224 | [ 225 | self.rt_args_many_to_many["pair_table_origin_id_field"], 226 | self.rt_args_many_to_many["pair_table_dest_id_field"] 227 | ] 228 | ) as cur: 229 | for od_pair in in_od_pairs: 230 | cur.insertRow(od_pair) 231 | 232 | # Validate and preprocess the data 233 | inputs = deepcopy(self.rt_args_many_to_many) 234 | inputs["pair_table"] = od_pair_table 235 | rt_solver = solve_large_route_pair_analysis.RoutePairSolver(**inputs) 236 | rt_solver._validate_inputs() 237 | rt_solver._preprocess_inputs() 238 | 239 | # Verify that the outputs generated by preprocessing have the correct number of rows 240 | with open(rt_solver.output_pair_table, "r", encoding="utf-8") as f: 241 | output_od_pairs = f.readlines() 242 | self.assertEqual(5, len(output_od_pairs), "Incorrect number of rows in output preprocessed OD pairs CSV.") 243 | self.assertEqual( 244 | 3, int(arcpy.management.GetCount(rt_solver.output_origins).getOutput(0)), 245 | "Incorrect number of rows in output preprocessed origins." 246 | ) 247 | self.assertEqual( 248 | 4, int(arcpy.management.GetCount(rt_solver.output_destinations).getOutput(0)), 249 | "Incorrect number of rows in output preprocessed destinations." 250 | ) 251 | 252 | def test_update_max_inputs_for_service(self): 253 | """Test the update_max_inputs_for_service function.""" 254 | max_routes = 20000000 255 | inputs = deepcopy(self.rt_args_one_to_one) 256 | inputs["network_data_source"] = self.portal_nd 257 | inputs["travel_mode"] = self.portal_tm 258 | inputs["chunk_size"] = max_routes 259 | rt_solver = solve_large_route_pair_analysis.RoutePairSolver(**inputs) 260 | tool_limits = { 261 | 'forceHierarchyBeyondDistance': 50.0, 262 | 'forceHierarchyBeyondDistanceUnits': 'Miles', 263 | 'maximumFeaturesAffectedByLineBarriers': 500.0, 264 | 'maximumFeaturesAffectedByPointBarriers': 250.0, 265 | 'maximumFeaturesAffectedByPolygonBarriers': 2000.0, 266 | 'maximumGeodesicDistanceUnitsWhenWalking': 'Miles', 267 | 'maximumGeodesicDistanceWhenWalking': 27.0, 268 | 'maximumStops': 10000.0, 269 | 'maximumStopsPerRoute': 150.0 270 | } 271 | rt_solver.service_limits = tool_limits 272 | rt_solver._update_max_inputs_for_service() 273 | self.assertEqual(tool_limits["maximumStops"] / 2, rt_solver.chunk_size) 274 | 275 | # Test when there are no limits 276 | tool_limits = { 277 | 'forceHierarchyBeyondDistance': 50.0, 278 | 'forceHierarchyBeyondDistanceUnits': 'Miles', 279 | 'maximumFeaturesAffectedByLineBarriers': 500.0, 280 | 'maximumFeaturesAffectedByPointBarriers': 250.0, 281 | 'maximumFeaturesAffectedByPolygonBarriers': 2000.0, 282 | 'maximumGeodesicDistanceUnitsWhenWalking': 'Miles', 283 | 'maximumGeodesicDistanceWhenWalking': 27.0, 284 | 'maximumStops': None, 285 | 'maximumStopsPerRoute': 150.0 286 | } 287 | rt_solver.chunk_size = max_routes 288 | rt_solver.service_limits = tool_limits 289 | rt_solver._update_max_inputs_for_service() 290 | self.assertEqual(max_routes, rt_solver.chunk_size) 291 | 292 | def test_solve_large_route_pair_analysis_one_to_one(self): 293 | """Test the full solve route pair workflow for the one-to-one pair type.""" 294 | rt_solver = solve_large_route_pair_analysis.RoutePairSolver(**self.rt_args_one_to_one) 295 | rt_solver.solve_large_route_pair_analysis() 296 | self.assertTrue(arcpy.Exists(self.rt_args_one_to_one["output_routes"])) 297 | 298 | def test_solve_large_route_pair_analysis_many_to_many(self): 299 | """Test the full solve route pair workflow for the many-to-many pair type.""" 300 | rt_solver = solve_large_route_pair_analysis.RoutePairSolver(**self.rt_args_many_to_many) 301 | rt_solver.solve_large_route_pair_analysis() 302 | self.assertTrue(arcpy.Exists(self.rt_args_many_to_many["output_routes"])) 303 | 304 | def test_solve_large_route_pair_analysis_use_oids(self): 305 | """Test the full solve route pair workflow for the one-to-one pair type using ObjectID fields as unique IDs.""" 306 | inputs = deepcopy(self.rt_args_one_to_one) 307 | inputs["origin_id_field"] = arcpy.Describe(inputs["origins"]).oidFieldName 308 | dest_oid = arcpy.Describe(inputs["destinations"]).oidFieldName 309 | inputs["dest_id_field"] = dest_oid 310 | inputs["assigned_dest_field"] = dest_oid 311 | rt_solver = solve_large_route_pair_analysis.RoutePairSolver(**inputs) 312 | rt_solver.solve_large_route_pair_analysis() 313 | self.assertTrue(arcpy.Exists(inputs["output_routes"])) 314 | 315 | def test_cli_one_to_one(self): 316 | """Test the command line interface of solve_large_route_pair_analysis for the one-to-one pair type.""" 317 | out_folder = os.path.join(self.scratch_folder, "CLI_CSV_Output_OneToOne") 318 | os.mkdir(out_folder) 319 | out_routes = os.path.join(self.output_gdb, "OutCLIRoutes_OneToOne") 320 | rt_inputs = [ 321 | os.path.join(sys.exec_prefix, "python.exe"), 322 | os.path.join(os.path.dirname(CWD), "solve_large_route_pair_analysis.py"), 323 | "--pair-type", "one_to_one", 324 | "--origins", self.origins, 325 | "--origins-id-field", "ID", 326 | "--destinations", self.destinations, 327 | "--destinations-id-field", "NAME", 328 | "--network-data-source", self.local_nd, 329 | "--travel-mode", self.local_tm_dist, 330 | "--time-units", "Minutes", 331 | "--distance-units", "Miles", 332 | "--max-routes", "15", 333 | "--max-processes", "4", 334 | "--out-routes", out_routes, 335 | "--assigned-dest-field", "StoreID", 336 | "--time-of-day", self.time_of_day_str, 337 | "--precalculate-network-locations", "true", 338 | "--sort-origins", "true", 339 | "--reverse-direction", "false" 340 | ] 341 | result = subprocess.run(rt_inputs, check=True) 342 | self.assertEqual(result.returncode, 0) 343 | self.assertTrue(arcpy.Exists(out_routes)) 344 | 345 | def test_cli_many_to_many(self): 346 | """Test the command line interface of solve_large_route_pair_analysis for the many-to-many pair type.""" 347 | out_folder = os.path.join(self.scratch_folder, "CLI_CSV_Output_ManyToMany") 348 | os.mkdir(out_folder) 349 | out_routes = os.path.join(self.output_gdb, "OutCLIRoutes_ManyToMany") 350 | rt_inputs = [ 351 | os.path.join(sys.exec_prefix, "python.exe"), 352 | os.path.join(os.path.dirname(CWD), "solve_large_route_pair_analysis.py"), 353 | "--pair-type", "many_to_many", 354 | "--origins", self.origins, 355 | "--origins-id-field", "ID", 356 | "--destinations", self.destinations, 357 | "--destinations-id-field", "NAME", 358 | "--network-data-source", self.local_nd, 359 | "--travel-mode", self.local_tm_dist, 360 | "--time-units", "Minutes", 361 | "--distance-units", "Miles", 362 | "--max-routes", "15", 363 | "--max-processes", "4", 364 | "--out-routes", out_routes, 365 | "--od-pair-table", self.od_pairs_table, 366 | "--od-pair-table-origin-id", "OriginID", 367 | "--od-pair-table-dest-id", "DestinationID", 368 | "--time-of-day", self.time_of_day_str, 369 | "--precalculate-network-locations", "true", 370 | "--sort-origins", "false", 371 | "--reverse-direction", "false" 372 | ] 373 | result = subprocess.run(rt_inputs, check=True) 374 | self.assertEqual(result.returncode, 0) 375 | self.assertTrue(arcpy.Exists(out_routes)) 376 | 377 | 378 | if __name__ == '__main__': 379 | unittest.main() 380 | -------------------------------------------------------------------------------- /unittests/unittests_README.txt: -------------------------------------------------------------------------------- 1 | The unit tests use data and from the SanFrancisco.gdb geodatabase from the ArcGIS Pro Network Analyst tutorial data. Download the data from https://links.esri.com/NetworkAnalyst/TutorialData/Pro. Extract the zip file and put SanFrancisco.gdb here in a folder called "TestInput". 2 | 3 | The tests also require valid values for a test portal with network analysis services in the portal_credentials.py file. --------------------------------------------------------------------------------