├── .gitignore ├── services.txt ├── mixin.js ├── LICENSE.txt ├── package.json ├── readme.md └── index.js /.gitignore: -------------------------------------------------------------------------------- 1 | #ignore node modules 2 | node_modules 3 | output 4 | .DS_Store 5 | _services.txt 6 | -------------------------------------------------------------------------------- /services.txt: -------------------------------------------------------------------------------- 1 | http://maps.lacity.org/lahub/rest/services/Recreation_and_Parks_Department/MapServer/4|parks|100 2 | -------------------------------------------------------------------------------- /mixin.js: -------------------------------------------------------------------------------- 1 | // @author: Joshua Tanner 2 | // mixin.js 3 | // mixin supplied object params with an 4 | // exisiting object 5 | 6 | var mixin = function (newObj, oldObj) { 7 | if(typeof newObj !== 'object') { 8 | throw 'Please provide an object'; 9 | } 10 | var result = oldObj || {}; // can be empty obj 11 | for ( var key in newObj ) { 12 | if( newObj.hasOwnProperty(key) ) { 13 | result[key] = newObj[key]; 14 | } 15 | } 16 | return result; 17 | }; 18 | 19 | module.exports = mixin; 20 | 21 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Joshua Tanner 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "agsout", 3 | "engines": { 4 | "node": ">=6.0 <7.0" 5 | }, 6 | "homepage": "https://github.com/tannerjt/AGStoShapefile", 7 | "version": "1.0.4", 8 | "description": "Converts ArcGIS Server Map Service to Shapefile", 9 | "main": "index.js", 10 | "scripts": { 11 | "test": "echo \"Error: no test specified\" && exit 1" 12 | }, 13 | "repository": { 14 | "type" : "git", 15 | "url" : "https://github.com/tannerjt/AGStoShapefile" 16 | }, 17 | "bin": { 18 | "agsout": "./index.js" 19 | }, 20 | "keywords": [ 21 | "ArcGIS", 22 | "Server", 23 | "JSON", 24 | "Shapefile", 25 | "Open", 26 | "Download" 27 | ], 28 | "author": "Joshua Tanner", 29 | "license": "ISC", 30 | "dependencies": { 31 | "JSONStream": "^1.3.5", 32 | "combined-stream": "^1.0.7", 33 | "commander": "^2.19.0", 34 | "esri2geo": "^0.1.3", 35 | "geojson-stream": "0.0.1", 36 | "lodash": "^4.17.11", 37 | "merge2": "^1.2.3", 38 | "ogr2ogr": "^0.5.1", 39 | "query-string": "^3.0.0", 40 | "request": "^2.88.0", 41 | "request-promise": "^4.2.2", 42 | "rimraf": "^2.6.2", 43 | "terraformer-arcgis-parser": "^1.1.0" 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # AGStoShapefile by [TannerGeo](http://tannergeo.com) 2 | 3 | A command line tool for backing up ArcGIS Server REST Services to file. 4 | 5 | AGStoShapefile is a node.js script that will convert Esri map services (Geoservices REST API) to GeoJSON and optionally Shapefile formats. This script will export all features and is not limited to max feature limits. AGStoShapefile can be used to cache your map services, store data for offline viewing, or used to build applications using a more simple GeJSON format. 6 | 7 | # Dependencies 8 | 9 | If you wish to download files as Shapefile, you will need to install the following: 10 | 11 | 1. You will need to install [node.js](https://nodejs.org/en/) with NPM. This script only runs on node versions 6.x. 12 | 2. Install and setup GDAL [Windows](http://sandbox.idre.ucla.edu/sandbox/tutorials/installing-gdal-for-windows) - [Mac/Linux](https://tilemill-project.github.io/tilemill/docs/guides/gdal/) 13 | 14 | # Instructions 15 | 16 | + Create a services.txt to include the services you wish to query 17 | 18 | *You can install via NPM* 19 | 20 | ``` 21 | npm install agsout -g 22 | ``` 23 | 24 | *Or, optionally, download and install from local* 25 | 26 | ``` 27 | npm install . -g 28 | ``` 29 | 30 | *Run the script* 31 | ``` 32 | agsout --help 33 | agsout -s ./services.txt -o ./backupdir -S 34 | #-s location of services text file 35 | #-o directory to backup services 36 | #-S optional shapefile output (requires gdal) 37 | ``` 38 | 39 | *Arguments* 40 | 41 | This command line script accepts 3 Arguments: 42 | 43 | + `-s` -> Location to services.txt file (**see below for example**) 44 | + `-o` -> Location to output directory to put files 45 | + `-S` -> *OPTIONAL* Output shapefile, will output geojson as well by default 46 | 47 | *for services.txt - use format [service_endpoint]|[title]|[throttle in ms]. Take note of the PIPE (|) symbol and new line.* 48 | ``` 49 | //example services.txt file 50 | http://test.service/arcigs/rest/flooding/MapServer/0|Flooding_Layer|0 51 | http://test.service/arcigs/rest/flooding/MapServer/1|Earthquake_Layer|5000 52 | http://test.service/arcigs/rest/flooding/MapServer/2|Tornado_Layer| 53 | ``` 54 | 55 | The throttle is helpful for very large extracts where servers may reject too many requests. 56 | The throttle number is in milliseconds. 57 | 58 | 59 | ## help 60 | 61 | Please contact [TannerGeo](http://tannergeo.com) for questions or assistance. 62 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | // @Author: Joshua Tanner 4 | // @Date: 1/15/2018 5 | // @Description: Easy way to convert ArcGIS Server service to GeoJSON 6 | // and shapefile format. Good for backup solution. 7 | // @services.txt format :: serviceLayerURL|layerName|throttle(ms) 8 | // @githubURL : https://github.com/tannerjt/AGStoShapefile 9 | 10 | // Node Modules 11 | const fs = require('fs'); 12 | const rp = require('request-promise'); 13 | const request = require('request'); 14 | const _ = require('lodash'); 15 | const TerraformerArcGIS = require('terraformer-arcgis-parser'); 16 | const geojsonStream = require('geojson-stream'); 17 | const JSONStream = require('JSONStream'); 18 | const CombinedStream = require('combined-stream'); 19 | const queryString = require('query-string'); 20 | const merge2 = require('merge2'); 21 | const rimraf = require('rimraf'); 22 | const ogr2ogr = require('ogr2ogr'); 23 | // ./mixin.js 24 | // merge user query params with default 25 | const mixin = require('./mixin'); 26 | var program = require('commander'); 27 | 28 | program 29 | .version('1.0.2') 30 | .option('-o, --outdir [directory]', 'Output directory') 31 | .option('-s, --services [path to txt file]', 'Text file containing service list to extract') 32 | .option('-S, --shapefile', 'Optional export to shapefile') 33 | .parse(process.argv); 34 | 35 | const serviceFile = program.services || 'services.txt'; 36 | var outDir = program.outdir || './output/'; 37 | // Remove trailing '/' 38 | outDir = outDir.replace(/\/$/, ''); 39 | 40 | fs.readFile(serviceFile, function (err, data) { 41 | if (err) { 42 | throw err; 43 | } 44 | 45 | data.toString().split('\n').forEach(function (service) { 46 | var service = service.split('|'); 47 | if(service[0].split('').length == 0) return; 48 | var baseUrl = getBaseUrl(service[0].trim()); 49 | var reqQS = { 50 | where: '1=1', 51 | returnIdsOnly: true, 52 | f: 'json' 53 | }; 54 | var userQS = getUrlVars(service[0].trim()); 55 | // mix one obj with another 56 | var qs = mixin(userQS, reqQS); 57 | qs = queryString.stringify(qs); 58 | var url = decodeURIComponent(getBaseUrl(baseUrl) + '/query/?' + qs); 59 | 60 | var throttle = 0; 61 | if(service.length > 2) { 62 | throttle = +service[2]; 63 | } 64 | 65 | rp({ 66 | url : url, 67 | method : 'GET', 68 | json : true 69 | }).then((body) => { 70 | requestService(service[0].trim(), service[1].trim(), body.objectIds, throttle); 71 | }).catch((err) => { 72 | console.log(err); 73 | }); 74 | }) 75 | }); 76 | 77 | // Resquest JSON from AGS 78 | function requestService(serviceUrl, serviceName, objectIds, throttle) { 79 | objectIds.sort(); 80 | const requests = Math.ceil(objectIds.length / 100); 81 | var completedRequests = 0; 82 | console.log(`Number of features for service ${serviceName}:`, objectIds.length); 83 | console.log(`Getting chunks of 100 features, will make ${requests} total requests`); 84 | 85 | for(let i = 0; i < Math.ceil(objectIds.length / 100); i++) { 86 | var ids = []; 87 | if ( ((i + 1) * 100) < objectIds.length ) { 88 | ids = objectIds.slice(i * 100, (i * 100) + 100); 89 | } else { 90 | ids = objectIds.slice(i * 100, objectIds[objectIds.length]); 91 | } 92 | 93 | // we need these query params 94 | const reqQS = { 95 | objectIds : ids.join(','), 96 | geometryType : 'esriGeometryEnvelope', 97 | returnGeometry : true, 98 | returnIdsOnly : false, 99 | outFields : '*', 100 | outSR : '4326', 101 | f : 'json' 102 | }; 103 | // user provided query params 104 | const userQS = getUrlVars(serviceUrl); 105 | // mix one obj with another 106 | var qs = mixin(userQS, reqQS); 107 | qs = queryString.stringify(qs); 108 | const url = decodeURIComponent(getBaseUrl(serviceUrl) + '/query/?' + qs); 109 | 110 | 111 | const partialsDir = `${outDir}/${serviceName}/partials`; 112 | 113 | if(i == 0) { 114 | // first pass, setup folders 115 | if(!fs.existsSync(`${outDir}`)) { 116 | fs.mkdirSync(`${outDir}`) 117 | } 118 | 119 | if(!fs.existsSync(`${outDir}/${serviceName}`)) { 120 | fs.mkdirSync(`${outDir}/${serviceName}`); 121 | } 122 | 123 | if (!fs.existsSync(partialsDir)){ 124 | fs.mkdirSync(partialsDir); 125 | } else { 126 | rimraf.sync(partialsDir); 127 | fs.mkdirSync(partialsDir); 128 | } 129 | } 130 | 131 | const featureStream = JSONStream.parse('features.*', convert); 132 | const outFile = fs.createWriteStream(`${partialsDir}/${i}.json`); 133 | 134 | const options = { 135 | url: url, 136 | method: 'GET', 137 | json: true, 138 | }; 139 | 140 | // timeout for throttle 141 | setTimeout(() => { 142 | request(options) 143 | .pipe(featureStream) 144 | .pipe(geojsonStream.stringify()) 145 | .pipe(outFile) 146 | .on('finish', () => { 147 | completedRequests += 1; 148 | console.log(`Completed ${completedRequests} of ${requests} requests for ${serviceName}`); 149 | if(requests == completedRequests) { 150 | mergeFiles(); 151 | } 152 | }) 153 | .on('error', (err) => { 154 | console.log(err); 155 | }); 156 | }, i * throttle); 157 | 158 | function convert (feature) { 159 | if(!feature.geometry) { 160 | console.log("Feature Missing Geometry and is Omitted: ", JSON.stringify(feature)) 161 | return null; 162 | } 163 | const gj = { 164 | type: 'Feature', 165 | properties: feature.attributes, 166 | geometry: TerraformerArcGIS.parse(feature.geometry) 167 | } 168 | return gj 169 | } 170 | 171 | function mergeFiles() { 172 | console.log(`Finished extracting chunks for ${serviceName}, merging files...`) 173 | fs.readdir(partialsDir, (err, files) => { 174 | const finalFilePath = `${outDir}/${serviceName}/${serviceName}_${Date.now()}.geojson` 175 | const finalFile = fs.createWriteStream(finalFilePath); 176 | 177 | let streams = CombinedStream.create(); 178 | _.each(files, (file) => { 179 | streams.append((next) => { 180 | next( 181 | fs.createReadStream(`${partialsDir}/${file}`) 182 | .pipe(JSONStream.parse('features.*')) 183 | .on('error', (err) => { 184 | console.log(err); 185 | }) 186 | ); 187 | }) 188 | }); 189 | 190 | streams 191 | .pipe(geojsonStream.stringify()) 192 | .pipe(finalFile) 193 | .on('finish', () => { 194 | rimraf(partialsDir, () => { 195 | console.log(`${serviceName} is complete`); 196 | console.log(`File Location: ${finalFilePath}`); 197 | if(program.shapefile) { 198 | makeShape(finalFilePath); 199 | } 200 | }); 201 | }) 202 | .on('error', (err) => { 203 | console.log(err); 204 | }) 205 | }); 206 | } 207 | 208 | function makeShape(geojsonPath) { 209 | console.log(`Generating shapefile for ${serviceName}`) 210 | // todo: make optional with flag 211 | const shpPath = `${outDir}/${serviceName}/${serviceName}_${Date.now()}.zip`; 212 | const shpFile = fs.createWriteStream(shpPath); 213 | var shapefile = ogr2ogr(geojsonPath) 214 | .format('ESRI Shapefile') 215 | .options(['-nln', serviceName]) 216 | .timeout(120000) 217 | .skipfailures() 218 | .stream(); 219 | shapefile.pipe(shpFile); 220 | } 221 | 222 | }; 223 | } 224 | 225 | 226 | //http://stackoverflow.com/questions/4656843/jquery-get-querystring-from-url 227 | function getUrlVars(url) { 228 | var vars = {}, hash; 229 | var hashes = url.slice(url.indexOf('?') + 1).split('&'); 230 | for(var i = 0; i < hashes.length; i++) 231 | { 232 | hash = hashes[i].split('='); 233 | vars[hash[0].toString()] = hash[1]; 234 | } 235 | return vars; 236 | } 237 | 238 | // get base url for query 239 | function getBaseUrl(url) { 240 | // remove any query params 241 | var url = url.split("?")[0]; 242 | if((/\/$/ig).test(url)) { 243 | url = url.substring(0, url.length - 1); 244 | } 245 | return url; 246 | } 247 | --------------------------------------------------------------------------------