├── .npmignore ├── .npmrc ├── .gitignore ├── release ├── test ├── fixtures │ ├── ImageServer-3_json_Download.json │ ├── rewind │ │ ├── rev.input.geojson │ │ ├── flip.input.geojson │ │ ├── near-zero.input.geojson │ │ ├── multipolygon.input.geojson │ │ ├── featuregood.input.geojson │ │ ├── near-zero.output.geojson │ │ ├── flip.output.geojson │ │ ├── rev.output.geojson │ │ ├── featuregood.output.geojson │ │ ├── geomcollection.input.geojson │ │ ├── multipolygon.output.geojson │ │ ├── collection.input.geojson │ │ ├── geomcollection.output.geojson │ │ └── collection.output.geojson │ ├── ImageServer-Download-1_json_Download.json │ ├── ImageServer-Download-2_json_Download.json │ ├── ImageServer-2_json_Download.json │ ├── ImageServer-1_json_Download.json │ ├── pass-imageserver.json │ ├── ImageServer_json_noDownload.json │ └── ImageServer_json_Download.json ├── schema.test.ts ├── rewind.test.ts ├── mapserver.test.ts ├── featureserver.test.ts ├── server.ts └── geometry.test.ts ├── eslint.config.js ├── tsconfig.json ├── .github └── workflows │ ├── test.yml │ └── release.yml ├── lib ├── fetch.ts ├── rewind.ts ├── schema.ts ├── discovery.ts ├── rings2geojson.ts └── geometry.ts ├── LICENSE ├── package.json ├── README.md ├── cli.ts ├── CHANGELOG.md └── index.ts /.npmignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | -------------------------------------------------------------------------------- /.npmrc: -------------------------------------------------------------------------------- 1 | //registry.npmjs.org/:_authToken=${NPM_TOKEN} 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | lib-cov 2 | *.geojson 3 | *.seed 4 | *.log 5 | *.csv 6 | *.dat 7 | *.out 8 | *.pid 9 | *.gz 10 | 11 | pids 12 | logs 13 | results 14 | 15 | npm-debug.log 16 | node_modules 17 | -------------------------------------------------------------------------------- /release: -------------------------------------------------------------------------------- 1 | if [[ -z "$1" ]]; then 2 | echo "No Version Specified: ./release " 3 | exit 1 4 | fi 5 | 6 | set -euo pipefail 7 | 8 | grep -Pzo "### ${1}(?s).*?(?=###)" CHANGELOG.md 9 | -------------------------------------------------------------------------------- /test/fixtures/ImageServer-3_json_Download.json: -------------------------------------------------------------------------------- 1 | { 2 | "error": { 3 | "code": 400, 4 | "message": "Unable to complete operation.", 5 | "details": [] 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /test/fixtures/rewind/rev.input.geojson: -------------------------------------------------------------------------------- 1 | { "type": "Polygon", 2 | "coordinates": [ 3 | [ [100.0, 0.0], [101.0, 0.0], [101.0, 1.0], [100.0, 1.0], [100.0, 0.0] ], 4 | [ [100.2, 0.2], [100.8, 0.2], [100.8, 0.8], [100.2, 0.8], [100.2, 0.2] ] 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /test/fixtures/rewind/flip.input.geojson: -------------------------------------------------------------------------------- 1 | { "type": "Polygon", 2 | "coordinates": [ 3 | [ [100.0, 0.0], [101.0, 0.0], [101.0, 1.0], [100.0, 1.0], [100.0, 0.0] ], 4 | [ [ 100.2, 0.2 ], 5 | [ 100.2, 0.8 ], 6 | [ 100.8, 0.8 ], 7 | [ 100.8, 0.2 ], 8 | [ 100.2, 0.2 ] ] 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /eslint.config.js: -------------------------------------------------------------------------------- 1 | import eslint from '@eslint/js'; 2 | import tseslint from 'typescript-eslint'; 3 | 4 | export default tseslint.config( 5 | eslint.configs.recommended, 6 | ...tseslint.configs.recommended, 7 | { 8 | "rules": { 9 | "@typescript-eslint/no-explicit-any": "warn" 10 | } 11 | } 12 | ); 13 | -------------------------------------------------------------------------------- /test/fixtures/rewind/near-zero.input.geojson: -------------------------------------------------------------------------------- 1 | { 2 | "type": "Polygon", 3 | "coordinates": [ 4 | [ 5 | [7.396768398983337, 43.72260793482001], 6 | [7.396784857564814, 43.722607191112004], 7 | [7.396784857564812, 43.722607191112004], 8 | [7.396768398983337, 43.72260793482001] 9 | ] 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /test/fixtures/rewind/multipolygon.input.geojson: -------------------------------------------------------------------------------- 1 | { "type": "MultiPolygon", 2 | "coordinates": [ 3 | [[[102.0, 2.0], [103.0, 2.0], [103.0, 3.0], [102.0, 3.0], [102.0, 2.0]]], 4 | [[[100.0, 0.0], [101.0, 0.0], [101.0, 1.0], [100.0, 1.0], [100.0, 0.0]], 5 | [[100.2, 0.2], [100.8, 0.2], [100.8, 0.8], [100.2, 0.8], [100.2, 0.2]]] 6 | ] 7 | } 8 | -------------------------------------------------------------------------------- /test/fixtures/rewind/featuregood.input.geojson: -------------------------------------------------------------------------------- 1 | { 2 | "type": "Feature", 3 | "geometry": { "type": "Polygon", 4 | "coordinates": [ 5 | [ [ 100, 0 ], 6 | [ 100, 1 ], 7 | [ 101, 1 ], 8 | [ 101, 0 ], 9 | [ 100, 0 ] ], 10 | [ [100.2, 0.2], [100.8, 0.2], [100.8, 0.8], [100.2, 0.8], [100.2, 0.2] ] 11 | ] 12 | }, 13 | "properties": { 14 | "foo": "bar" 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /test/fixtures/rewind/near-zero.output.geojson: -------------------------------------------------------------------------------- 1 | { 2 | "type": "Polygon", 3 | "coordinates": [ 4 | [ 5 | [ 6 | 7.396768398983337, 7 | 43.72260793482001 8 | ], 9 | [ 10 | 7.396784857564814, 11 | 43.722607191112004 12 | ], 13 | [ 14 | 7.396784857564812, 15 | 43.722607191112004 16 | ], 17 | [ 18 | 7.396768398983337, 19 | 43.72260793482001 20 | ] 21 | ] 22 | ] 23 | } 24 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "ts-node": { 3 | "esm": true 4 | }, 5 | "compilerOptions": { 6 | "module": "es2022", 7 | "esModuleInterop": true, 8 | "target": "es2022", 9 | "noImplicitAny": true, 10 | "moduleResolution": "node", 11 | "sourceMap": true, 12 | "outDir": "dist", 13 | "baseUrl": ".", 14 | "paths": { 15 | "*": [ 16 | "node_modules/*" 17 | ] 18 | } 19 | }, 20 | "include": [ 21 | "index.ts", 22 | "cli.ts", 23 | "lib/**/*" 24 | ] 25 | } 26 | -------------------------------------------------------------------------------- /test/fixtures/ImageServer-Download-1_json_Download.json: -------------------------------------------------------------------------------- 1 | { 2 | "rasterFiles": [ 3 | { 4 | "id": "http://localhost:3000/image.tif", 5 | "size": 621175109, 6 | "rasterIds": [ 7 | 1 8 | ] 9 | }, 10 | { 11 | "id": "http://localhost:3000/image.tif.ovr", 12 | "size": 55929330, 13 | "rasterIds": [ 14 | 1 15 | ] 16 | }, 17 | { 18 | "id": "http://localhost:3000/image.tif.aux.xml", 19 | "size": 9634, 20 | "rasterIds": [ 21 | 1 22 | ] 23 | } 24 | ] 25 | } 26 | -------------------------------------------------------------------------------- /test/fixtures/ImageServer-Download-2_json_Download.json: -------------------------------------------------------------------------------- 1 | { 2 | "rasterFiles": [ 3 | { 4 | "id": "http://localhost:3000/image.tif", 5 | "size": 621175109, 6 | "rasterIds": [ 7 | 1 8 | ] 9 | }, 10 | { 11 | "id": "http://localhost:3000/image.tif.ovr", 12 | "size": 55929330, 13 | "rasterIds": [ 14 | 1 15 | ] 16 | }, 17 | { 18 | "id": "http://localhost:3000/image.tif.aux.xml", 19 | "size": 9634, 20 | "rasterIds": [ 21 | 1 22 | ] 23 | } 24 | ] 25 | } 26 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | types: 9 | - opened 10 | - synchronize 11 | - reopened 12 | - ready_for_review 13 | 14 | jobs: 15 | test: 16 | runs-on: ubuntu-latest 17 | if: github.event.pull_request.draft == false 18 | steps: 19 | - uses: actions/checkout@v3 20 | with: 21 | ref: ${{ github.event.pull_request.head.sha }} 22 | 23 | - uses: actions/setup-node@v3 24 | with: 25 | node-version: 18 26 | registry-url: https://registry.npmjs.org/ 27 | 28 | - name: Install 29 | run: npm install 30 | 31 | - name: Test 32 | run: npm test 33 | 34 | - name: Lint 35 | run: npm run lint 36 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: NPM Release 2 | 3 | on: 4 | push: 5 | tags: 6 | - '*' 7 | 8 | jobs: 9 | build: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v3 13 | 14 | - name: Get tag 15 | id: tag 16 | uses: dawidd6/action-get-tag@v1 17 | 18 | - uses: actions/setup-node@v3 19 | with: 20 | node-version: 18 21 | registry-url: https://registry.npmjs.org/ 22 | 23 | - name: npm install 24 | run: npm install 25 | 26 | - name: npm run build 27 | run: npm run build 28 | 29 | - name: npm publish 30 | run: npm publish 31 | env: 32 | NPM_TOKEN: ${{ secrets.NPM_SECRET }} 33 | 34 | - name: Generate CHANGELOG 35 | run: ./release ${{steps.tag.outputs.tag}} > RELEASE 36 | 37 | - name: Github Release 38 | uses: softprops/action-gh-release@v2 39 | with: 40 | body_path: RELEASE 41 | -------------------------------------------------------------------------------- /test/fixtures/rewind/flip.output.geojson: -------------------------------------------------------------------------------- 1 | { 2 | "type": "Polygon", 3 | "coordinates": [ 4 | [ 5 | [ 6 | 100, 7 | 0 8 | ], 9 | [ 10 | 101, 11 | 0 12 | ], 13 | [ 14 | 101, 15 | 1 16 | ], 17 | [ 18 | 100, 19 | 1 20 | ], 21 | [ 22 | 100, 23 | 0 24 | ] 25 | ], 26 | [ 27 | [ 28 | 100.2, 29 | 0.2 30 | ], 31 | [ 32 | 100.2, 33 | 0.8 34 | ], 35 | [ 36 | 100.8, 37 | 0.8 38 | ], 39 | [ 40 | 100.8, 41 | 0.2 42 | ], 43 | [ 44 | 100.2, 45 | 0.2 46 | ] 47 | ] 48 | ] 49 | } -------------------------------------------------------------------------------- /test/fixtures/rewind/rev.output.geojson: -------------------------------------------------------------------------------- 1 | { 2 | "type": "Polygon", 3 | "coordinates": [ 4 | [ 5 | [ 6 | 100, 7 | 0 8 | ], 9 | [ 10 | 101, 11 | 0 12 | ], 13 | [ 14 | 101, 15 | 1 16 | ], 17 | [ 18 | 100, 19 | 1 20 | ], 21 | [ 22 | 100, 23 | 0 24 | ] 25 | ], 26 | [ 27 | [ 28 | 100.2, 29 | 0.2 30 | ], 31 | [ 32 | 100.2, 33 | 0.8 34 | ], 35 | [ 36 | 100.8, 37 | 0.8 38 | ], 39 | [ 40 | 100.8, 41 | 0.2 42 | ], 43 | [ 44 | 100.2, 45 | 0.2 46 | ] 47 | ] 48 | ] 49 | } -------------------------------------------------------------------------------- /lib/fetch.ts: -------------------------------------------------------------------------------- 1 | import { EsriDumpConfig } from '../index.js'; 2 | 3 | // TODO: Remove this once TypeScript has fetch definitions for core node 4 | export interface FetchRequest { 5 | method?: string; 6 | headers?: { 7 | [k: string]: string; 8 | }; 9 | } 10 | 11 | export default async function Fetch( 12 | config: EsriDumpConfig, 13 | url: URL, 14 | opts: FetchRequest = {} 15 | ): Promise { 16 | url = new URL(url); 17 | 18 | if (!config.headers) config.headers = {}; 19 | if (!config.params) config.params = {}; 20 | 21 | for (const param in config.params) url.searchParams.append(param, config.params[param]); 22 | 23 | url.searchParams.append('f', 'json'); 24 | 25 | if (!opts.headers) opts.headers = {}; 26 | Object.assign(opts.headers, config.headers); 27 | 28 | const headers: HeadersInit = new Headers(); 29 | headers.set('Accept-Encoding', 'gzip'); 30 | 31 | for (const header in opts.headers) { 32 | headers.set(header, opts.headers[header]); 33 | } 34 | 35 | return await fetch(url, { 36 | ...opts, 37 | headers 38 | }); 39 | } 40 | -------------------------------------------------------------------------------- /test/schema.test.ts: -------------------------------------------------------------------------------- 1 | import EsriDump from '../index.js'; 2 | import test from 'tape'; 3 | 4 | test('FeatureServer Schema', async (t) => { 5 | const url = 'https://sampleserver6.arcgisonline.com/arcgis/rest/services/Wildfire/FeatureServer/0'; 6 | 7 | const esri = new EsriDump(url); 8 | const schema = await esri.schema(); 9 | 10 | 11 | t.deepEquals(schema, { 12 | type: 'object', 13 | required: [], 14 | additionalProperties: false, 15 | properties: { 16 | objectid: { type: 'number' }, 17 | rotation: { type: 'integer' }, 18 | description: { type: 'string', maxLength: 75 }, 19 | eventdate: { type: 'string', format: 'date-time', maxLength: 8 }, 20 | eventtype: { type: 'integer' }, 21 | created_user: { type: 'string', maxLength: 255 }, 22 | created_date: { type: 'string', format: 'date-time', maxLength: 8 }, 23 | last_edited_user: { type: 'string', maxLength: 255 }, 24 | last_edited_date: { type: 'string', format: 'date-time', maxLength: 8 } 25 | } 26 | }); 27 | 28 | t.end(); 29 | }); 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 OpenAddresses 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /test/fixtures/rewind/featuregood.output.geojson: -------------------------------------------------------------------------------- 1 | { 2 | "type": "Feature", 3 | "geometry": { 4 | "type": "Polygon", 5 | "coordinates": [ 6 | [ 7 | [ 8 | 100, 9 | 0 10 | ], 11 | [ 12 | 101, 13 | 0 14 | ], 15 | [ 16 | 101, 17 | 1 18 | ], 19 | [ 20 | 100, 21 | 1 22 | ], 23 | [ 24 | 100, 25 | 0 26 | ] 27 | ], 28 | [ 29 | [ 30 | 100.2, 31 | 0.2 32 | ], 33 | [ 34 | 100.2, 35 | 0.8 36 | ], 37 | [ 38 | 100.8, 39 | 0.8 40 | ], 41 | [ 42 | 100.8, 43 | 0.2 44 | ], 45 | [ 46 | 100.2, 47 | 0.2 48 | ] 49 | ] 50 | ] 51 | }, 52 | "properties": { 53 | "foo": "bar" 54 | } 55 | } -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "esri-dump", 3 | "type": "module", 4 | "version": "5.5.1", 5 | "author": "Ian Dees ", 6 | "description": "Assist with pulling data out of an ESRI ArcGIS REST server into a more useful format.", 7 | "scripts": { 8 | "test": "ts-node-test test/", 9 | "build": "tsc --build", 10 | "lint": "eslint *.ts test/*.ts lib/*.ts" 11 | }, 12 | "repository": { 13 | "type": "git", 14 | "url": "git://github.com/openaddresses/esri-dump.git" 15 | }, 16 | "bin": { 17 | "esri-dump": "./dist/cli.js" 18 | }, 19 | "main": "dist/index.js", 20 | "types": "index.ts", 21 | "license": "MIT", 22 | "engines": { 23 | "node": ">=18" 24 | }, 25 | "devDependencies": { 26 | "@types/geojson": "^7946.0.10", 27 | "@types/minimist": "^1.2.2", 28 | "@types/tape": "^5.0.0", 29 | "eslint": "^9.0.0", 30 | "geojsonhint": "^2.0.0", 31 | "tape": "^5.0.0", 32 | "ts-node": "^10.9.1", 33 | "ts-node-test": "^0.4.1", 34 | "typescript": "^5.0.4", 35 | "typescript-eslint": "^8.0.0" 36 | }, 37 | "dependencies": { 38 | "@openaddresses/batch-error": "^2.2.0", 39 | "@types/json-schema": "^7.0.12", 40 | "minimist": "^1.2.8" 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /test/fixtures/rewind/geomcollection.input.geojson: -------------------------------------------------------------------------------- 1 | { 2 | "type": "GeometryCollection", 3 | "geometries": [ 4 | { 5 | "type": "Point", 6 | "coordinates": [102.0, 0.5] 7 | }, 8 | { 9 | "type": "LineString", 10 | "coordinates": [ 11 | [102.0, 0.0], 12 | [103.0, 1.0], 13 | [104.0, 0.0], 14 | [105.0, 1.0] 15 | ] 16 | }, 17 | { 18 | "type": "Polygon", 19 | "coordinates": [ 20 | [ 21 | [100.0, 0.0], 22 | [101.0, 0.0], 23 | [101.0, 1.0], 24 | [100.0, 1.0], 25 | [100.0, 0.0] 26 | ] 27 | ] 28 | }, 29 | { 30 | "type": "Polygon", 31 | "coordinates": [ 32 | [ 33 | [100.0, 0.0], 34 | [101.0, 0.0], 35 | [101.0, 1.0], 36 | [100.0, 1.0], 37 | [100.0, 0.0] 38 | ], 39 | [ 40 | [100.2, 0.2], 41 | [100.8, 0.2], 42 | [100.8, 0.8], 43 | [100.2, 0.8], 44 | [100.2, 0.2] 45 | ] 46 | ] 47 | } 48 | ] 49 | } 50 | -------------------------------------------------------------------------------- /test/fixtures/ImageServer-2_json_Download.json: -------------------------------------------------------------------------------- 1 | { 2 | "attributes": { 3 | "OBJECTID": 2, 4 | "Name": "image", 5 | "MinPS": 0, 6 | "MaxPS": 20, 7 | "LowPS": 0.5, 8 | "HighPS": 2, 9 | "Category": 1, 10 | "Tag": "Dataset", 11 | "GroupName": "", 12 | "ProductName": "", 13 | "CenterX": 2350782.578273649, 14 | "CenterY": 1955351.2145930487, 15 | "ZOrder": null, 16 | "Shape_Length": 24625.204791258868, 17 | "Shape_Area": 36542245.57806721 18 | }, 19 | "geometry": { 20 | "rings": [ 21 | [ 22 | [ 23 | 2348144.8322, 24 | 1951791.5059999991 25 | ], 26 | [ 27 | 2348433.1635, 28 | 1959107.4727999996 29 | ], 30 | [ 31 | 2353420.3403000003, 32 | 1958910.8248999994 33 | ], 34 | [ 35 | 2353132.0220999997, 36 | 1951595.0555000007 37 | ], 38 | [ 39 | 2348144.8322, 40 | 1951791.5059999991 41 | ] 42 | ] 43 | ], 44 | "spatialReference": { 45 | "wkid": 102605, 46 | "latestWkid": 102605 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /test/fixtures/ImageServer-1_json_Download.json: -------------------------------------------------------------------------------- 1 | { 2 | "attributes": { 3 | "OBJECTID": 1, 4 | "Name": "image", 5 | "MinPS": 0, 6 | "MaxPS": 20, 7 | "LowPS": 0.5, 8 | "HighPS": 2, 9 | "Category": 1, 10 | "Tag": "Dataset", 11 | "GroupName": "", 12 | "ProductName": "", 13 | "CenterX": 2335067.800079886, 14 | "CenterY": 1538781.926635325, 15 | "ZOrder": null, 16 | "Shape_Length": 25246.863514616693, 17 | "Shape_Area": 38837213.29536865 18 | }, 19 | "geometry": { 20 | "rings": [ 21 | [ 22 | [ 23 | 2332278.4853999997, 24 | 1535226.7589999996 25 | ], 26 | [ 27 | 2332549.3433999997, 28 | 1542533.8412999995 29 | ], 30 | [ 31 | 2337857.135, 32 | 1542336.9820000008 33 | ], 34 | [ 35 | 2337586.2912, 36 | 1535030.1249000002 37 | ], 38 | [ 39 | 2332278.4853999997, 40 | 1535226.7589999996 41 | ] 42 | ] 43 | ], 44 | "spatialReference": { 45 | "wkid": 102605, 46 | "latestWkid": 102605 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /test/rewind.test.ts: -------------------------------------------------------------------------------- 1 | import rewind from '../lib/rewind.js'; 2 | import fs from 'node:fs'; 3 | import path from 'node:path'; 4 | import test, {Test} from 'tape'; 5 | 6 | const base = new URL(path.parse(import.meta.url).dir).pathname; 7 | 8 | function f(_: string) { 9 | return JSON.parse(fs.readFileSync(_, 'utf8')); 10 | } 11 | 12 | function fixture(t: Test, name: string, title: string) { 13 | const result = rewind(f(name)); 14 | const outputName = name.replace('.input.', '.output.'); 15 | if (process.env.UPDATE) { 16 | fs.writeFileSync(outputName, JSON.stringify(result, null, 4)); 17 | } 18 | const expect = f(outputName); 19 | t.deepEqual(result, expect, title); 20 | } 21 | 22 | test('rewind', (t) => { 23 | fixture(t, base + '/fixtures/rewind/featuregood.input.geojson', 'feature-good'); 24 | fixture(t, base + '/fixtures/rewind/flip.input.geojson', 'flip'); 25 | fixture(t, base + '/fixtures/rewind/collection.input.geojson', 'feature-collection'); 26 | fixture(t, base + '/fixtures/rewind/geomcollection.input.geojson', 'geometry-collection'); 27 | fixture(t, base + '/fixtures/rewind/multipolygon.input.geojson', 'multipolygon'); 28 | fixture(t, base + '/fixtures/rewind/rev.input.geojson', 'rev'); 29 | fixture(t, base + '/fixtures/rewind/near-zero.input.geojson', 'near-zero'); 30 | t.end(); 31 | }); 32 | 33 | test('passthrough', (t) => { 34 | t.equal(rewind(null), null); 35 | t.end(); 36 | }); 37 | -------------------------------------------------------------------------------- /lib/rewind.ts: -------------------------------------------------------------------------------- 1 | import { 2 | GeoJSON, 3 | FeatureCollection, 4 | GeometryCollection, 5 | Feature, 6 | Polygon, 7 | MultiPolygon 8 | } from 'geojson'; 9 | 10 | export default function rewind(gj: GeoJSON, outer?: any) { 11 | const type = gj && gj.type; 12 | 13 | if (type === 'FeatureCollection') { 14 | gj = gj as FeatureCollection; 15 | for (let i = 0; i < gj.features.length; i++) rewind(gj.features[i], outer); 16 | 17 | } else if (type === 'GeometryCollection') { 18 | gj = gj as GeometryCollection; 19 | for (let i = 0; i < gj.geometries.length; i++) rewind(gj.geometries[i], outer); 20 | 21 | } else if (type === 'Feature') { 22 | gj = gj as Feature; 23 | rewind(gj.geometry, outer); 24 | 25 | } else if (type === 'Polygon') { 26 | gj = gj as Polygon; 27 | rewindRings(gj.coordinates, outer); 28 | } else if (type === 'MultiPolygon') { 29 | gj = gj as MultiPolygon; 30 | for (let i = 0; i < gj.coordinates.length; i++) rewindRings(gj.coordinates[i], outer); 31 | } 32 | 33 | return gj; 34 | } 35 | 36 | function rewindRings(rings: Array, outer: any) { 37 | if (rings.length === 0) return; 38 | 39 | rewindRing(rings[0], outer); 40 | for (let i = 1; i < rings.length; i++) { 41 | rewindRing(rings[i], !outer); 42 | } 43 | } 44 | 45 | function rewindRing(ring: Array, dir: any) { 46 | let area = 0, err = 0; 47 | for (let i = 0, len = ring.length, j = len - 1; i < len; j = i++) { 48 | const k = (ring[i][0] - ring[j][0]) * (ring[j][1] + ring[i][1]); 49 | const m = area + k; 50 | err += Math.abs(area) >= Math.abs(k) ? area - m + k : k - m + area; 51 | area = m; 52 | } 53 | if (area + err >= 0 !== !!dir) ring.reverse(); 54 | } 55 | -------------------------------------------------------------------------------- /lib/schema.ts: -------------------------------------------------------------------------------- 1 | import { JSONSchema6 } from 'json-schema'; 2 | 3 | // Ref: https://help.arcgis.com/en/sdk/10.0/java_ao_adf/api/arcgiswebservices/com/esri/arcgisws/EsriFieldType.html 4 | const Types: Map = new Map([ 5 | ['esriFieldTypeDate', { type: 'string', format: 'date-time' }], 6 | ['esriFieldTypeString', { type: 'string' }], 7 | ['esriFieldTypeDouble', { type: 'number' }], 8 | ['esriFieldTypeSingle', { type: 'number' }], 9 | ['esriFieldTypeOID', { type: 'number' }], 10 | ['esriFieldTypeInteger', { type: 'integer' }], 11 | ['esriFieldTypeSmallInteger', { type: 'integer' }], 12 | ['esriFieldTypeGeometry', { type: 'object' }], 13 | ['esriFieldTypeBlob', { type: 'object' }], 14 | ['esriFieldTypeGlobalID', { type: 'string' }], 15 | ['esriFieldTypeRaster', { type: 'object' }], 16 | ['esriFieldTypeGUID', { type: 'string' }], 17 | ['esriFieldTypeXML', { type: 'string' }], 18 | ]); 19 | 20 | export default function FieldToSchema(metadata: any): JSONSchema6 { 21 | const doc: JSONSchema6 = { 22 | type: 'object', 23 | required: [], 24 | additionalProperties: false, 25 | properties: {} 26 | } 27 | 28 | if (!metadata.fields && !Array.isArray(metadata.fields)) { 29 | return doc; 30 | } 31 | 32 | for (const field of metadata.fields) { 33 | const name = String(field.name); 34 | 35 | const type: JSONSchema6 = Types.has(field.type) ? Types.get(field.type) : { type: 'string' }; 36 | 37 | const prop: JSONSchema6 = doc.properties[name] = { 38 | ...JSON.parse(JSON.stringify(type)) 39 | } 40 | 41 | if (!isNaN(field.length) && type.type === 'string') { 42 | prop.maxLength = field.length; 43 | } 44 | } 45 | 46 | return doc; 47 | } 48 | -------------------------------------------------------------------------------- /test/fixtures/rewind/multipolygon.output.geojson: -------------------------------------------------------------------------------- 1 | { 2 | "type": "MultiPolygon", 3 | "coordinates": [ 4 | [ 5 | [ 6 | [ 7 | 102, 8 | 2 9 | ], 10 | [ 11 | 103, 12 | 2 13 | ], 14 | [ 15 | 103, 16 | 3 17 | ], 18 | [ 19 | 102, 20 | 3 21 | ], 22 | [ 23 | 102, 24 | 2 25 | ] 26 | ] 27 | ], 28 | [ 29 | [ 30 | [ 31 | 100, 32 | 0 33 | ], 34 | [ 35 | 101, 36 | 0 37 | ], 38 | [ 39 | 101, 40 | 1 41 | ], 42 | [ 43 | 100, 44 | 1 45 | ], 46 | [ 47 | 100, 48 | 0 49 | ] 50 | ], 51 | [ 52 | [ 53 | 100.2, 54 | 0.2 55 | ], 56 | [ 57 | 100.2, 58 | 0.8 59 | ], 60 | [ 61 | 100.8, 62 | 0.8 63 | ], 64 | [ 65 | 100.8, 66 | 0.2 67 | ], 68 | [ 69 | 100.2, 70 | 0.2 71 | ] 72 | ] 73 | ] 74 | ] 75 | } -------------------------------------------------------------------------------- /test/mapserver.test.ts: -------------------------------------------------------------------------------- 1 | import EsriDump from '../index.js'; 2 | import test from 'tape'; 3 | import { Feature } from 'geojson'; 4 | // @ts-expect-error No Type Defs 5 | import geojsonhint from 'geojsonhint'; 6 | 7 | test('MapServer with points geometry', (t) => { 8 | const url = 'https://sampleserver6.arcgisonline.com/arcgis/rest/services/Wildfire/MapServer/0'; 9 | const data: { 10 | type: string, 11 | features: Feature[] 12 | } = { 13 | type: 'FeatureCollection', 14 | features: [] 15 | }; 16 | 17 | const esri = new EsriDump(url); 18 | esri.fetch(); 19 | 20 | esri.on('type', (type) => { 21 | t.equals(type, 'FeatureServer', 'recognizes FeatureServer'); 22 | }); 23 | 24 | esri.on('feature', (feature) => { 25 | data.features.push(feature); 26 | }); 27 | 28 | esri.on('error', (err) => { 29 | throw err; 30 | }); 31 | 32 | esri.on('done', () => { 33 | const errors = geojsonhint.hint(data); 34 | t.ok(errors.length === 0, 'GeoJSON valid'); 35 | 36 | t.ok(data.features.length > 1); 37 | 38 | t.end(); 39 | }); 40 | }); 41 | 42 | test('MapServer with polygon geometry', (t) => { 43 | t.plan(2); 44 | 45 | const url = 'https://sampleserver6.arcgisonline.com/arcgis/rest/services/Wildfire/MapServer/2'; 46 | const data: { 47 | type: string, 48 | features: Feature[] 49 | } = { 50 | type: 'FeatureCollection', 51 | features: [] 52 | }; 53 | 54 | const esri = new EsriDump(url); 55 | esri.fetch(); 56 | 57 | esri.on('type', (type) => { 58 | t.equals(type, 'FeatureServer', 'recognizes FeatureServer'); 59 | }); 60 | 61 | esri.on('feature', (feature) => { 62 | data.features.push(feature); 63 | }); 64 | 65 | esri.on('error', (err) => { 66 | throw err; 67 | }); 68 | 69 | esri.on('done', () => { 70 | const errors = geojsonhint.hint(data); 71 | t.ok(errors.length === 0, 'GeoJSON valid'); 72 | 73 | t.ok(data.features.length > 1); 74 | 75 | t.end(); 76 | }); 77 | }); 78 | -------------------------------------------------------------------------------- /test/featureserver.test.ts: -------------------------------------------------------------------------------- 1 | import EsriDump from '../index.js'; 2 | import test from 'tape'; 3 | // @ts-expect-error No Type Defs 4 | import geojsonhint from 'geojsonhint'; 5 | import { Feature } from 'geojson'; 6 | 7 | test('FeatureServer with points geometry', (t) => { 8 | const url = 'https://sampleserver6.arcgisonline.com/arcgis/rest/services/Wildfire/FeatureServer/0'; 9 | const data: { 10 | type: string, 11 | features: Feature[] 12 | } = { 13 | type: 'FeatureCollection', 14 | features: [] 15 | }; 16 | 17 | const esri = new EsriDump(url); 18 | esri.fetch(); 19 | 20 | esri.on('type', (type) => { 21 | t.equals(type, 'FeatureServer', 'recognizes FeatureServer'); 22 | }); 23 | 24 | esri.on('feature', (feature) => { 25 | data.features.push(feature); 26 | }); 27 | 28 | esri.on('error', (err) => { 29 | throw err; 30 | }); 31 | 32 | esri.on('done', () => { 33 | const errors = geojsonhint.hint(data); 34 | t.ok(errors.length === 0, 'GeoJSON valid'); 35 | 36 | t.ok(data.features.length > 1); 37 | 38 | t.end(); 39 | }); 40 | }); 41 | 42 | test('FeatureServer with polygon geometry', (t) => { 43 | t.plan(2); 44 | 45 | const url = 'https://sampleserver6.arcgisonline.com/arcgis/rest/services/Wildfire/FeatureServer/2'; 46 | const data: { 47 | type: string, 48 | features: Feature[] 49 | } = { 50 | type: 'FeatureCollection', 51 | features: [] 52 | }; 53 | 54 | const esri = new EsriDump(url); 55 | esri.fetch(); 56 | 57 | esri.on('type', (type) => { 58 | t.equals(type, 'FeatureServer', 'recognizes FeatureServer'); 59 | }); 60 | 61 | esri.on('feature', (feature) => { 62 | data.features.push(feature); 63 | }); 64 | 65 | esri.on('error', (err) => { 66 | throw err; 67 | }); 68 | 69 | esri.on('done', () => { 70 | const errors = geojsonhint.hint(data); 71 | t.ok(errors.length === 0, 'GeoJSON valid'); 72 | 73 | t.ok(data.features.length > 1); 74 | 75 | t.end(); 76 | }); 77 | }); 78 | -------------------------------------------------------------------------------- /test/fixtures/rewind/collection.input.geojson: -------------------------------------------------------------------------------- 1 | { 2 | "type": "FeatureCollection", 3 | "features": [ 4 | { 5 | "type": "Feature", 6 | "geometry": { 7 | "type": "Point", 8 | "coordinates": [102.0, 0.5] 9 | }, 10 | "properties": { 11 | "prop0": "value0" 12 | } 13 | }, 14 | { 15 | "type": "Feature", 16 | "geometry": { 17 | "type": "LineString", 18 | "coordinates": [ 19 | [102.0, 0.0], 20 | [103.0, 1.0], 21 | [104.0, 0.0], 22 | [105.0, 1.0] 23 | ] 24 | }, 25 | "properties": { 26 | "prop0": "value0", 27 | "prop1": 0.0 28 | } 29 | }, 30 | { 31 | "type": "Feature", 32 | "geometry": { 33 | "type": "Polygon", 34 | "coordinates": [ 35 | [ 36 | [100.0, 0.0], 37 | [101.0, 0.0], 38 | [101.0, 1.0], 39 | [100.0, 1.0], 40 | [100.0, 0.0] 41 | ] 42 | ] 43 | }, 44 | "properties": { 45 | "prop0": "value0", 46 | "prop1": { 47 | "this": "that" 48 | } 49 | } 50 | }, 51 | { 52 | "type": "Feature", 53 | "geometry": { 54 | "type": "Polygon", 55 | "coordinates": [ 56 | [ 57 | [100.0, 0.0], 58 | [101.0, 0.0], 59 | [101.0, 1.0], 60 | [100.0, 1.0], 61 | [100.0, 0.0] 62 | ], 63 | [ 64 | [100.2, 0.2], 65 | [100.8, 0.2], 66 | [100.8, 0.8], 67 | [100.2, 0.8], 68 | [100.2, 0.2] 69 | ] 70 | ] 71 | }, 72 | "properties": { 73 | "prop0": "value0", 74 | "prop1": { 75 | "this": "that" 76 | } 77 | } 78 | } 79 | ] 80 | } 81 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

esri-dump

2 | 3 |

A Node module to assist with pulling data out of an ESRI ArcGIS REST server into GeoJSON or ImageryURLs

4 | 5 | Based On [PyEsriDump](http://github.com/openaddresses/pyesridump) by [@iandees](https://github.com/iandees) 6 | 7 | ## Install 8 | 9 | ```sh 10 | npm install -g esri-dump 11 | ``` 12 | 13 | ## API 14 | 15 | exposes a function, which if you give it a url, will return a stream of the geojson features. 16 | 17 | ```js 18 | import EsriDump from 'esri-dump'; 19 | const esri = new EsriDump(url); 20 | 21 | const featureCollection = { 22 | type: 'FeatureCollection', 23 | features: [] 24 | } 25 | 26 | esri.fetch(); 27 | 28 | esri.on('type', (type) => { 29 | //Emitted before any data events 30 | //emits one of 31 | // - `MapServer' 32 | // - `FeatureServer' 33 | }); 34 | 35 | esri.on('feature', (feature) => { 36 | featureCollection.features.push(feature); 37 | }); 38 | 39 | esri.on('done', () => { 40 | doSomething(null, featureCollection) 41 | }); 42 | 43 | esri.on('error', (err) => { 44 | doSomething(err); 45 | }); 46 | ``` 47 | 48 | ## Command Line 49 | 50 | Streams a geojson feature collection to stdout 51 | 52 | ```sh 53 | esri-dump fetch http://services2.bhamaps.com/arcgis/rest/services/AGS_jackson_co_il_taxmap/MapServer/0 > output.geojson 54 | ``` 55 | 56 | ## Data Output 57 | 58 | ### FeatureServer and MapServer 59 | 60 | Output from an ESRI `FeatureServer` or an ESRI `MapServer` is returned as GeoJSON as in the example below. 61 | 62 | ```json 63 | { 64 | "type": "Feature", 65 | "properties": { 66 | "objectid": 1 67 | }, 68 | "geometry": { 69 | "type": "Polygon", 70 | "coordinates": [ 71 | [ 72 | [ 73 | -65.6231319, 74 | 31.7127058 75 | ], 76 | [ 77 | -65.6144566, 78 | 31.7020286 79 | ], 80 | [ 81 | -65.6231319, 82 | 31.698692 83 | ], 84 | [ 85 | -65.6231319, 86 | 31.7127058 87 | ] 88 | ] 89 | ] 90 | } 91 | } 92 | ``` 93 | 94 | # Development 95 | 96 | `esri-dump` is written in TypeScript. To compile it locally, run: 97 | 98 | ```sh 99 | npx tsc 100 | ``` 101 | 102 | See `/dist` for the compiled code. 103 | -------------------------------------------------------------------------------- /cli.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | import EsriDump from './index.js'; 3 | import minimist from 'minimist'; 4 | 5 | const argv = minimist(process.argv, { 6 | string: ['approach', 'header'], 7 | boolean: ['help'] 8 | }); 9 | 10 | if (argv.help) { 11 | console.log(); 12 | console.log('Usage:'); 13 | console.log(' node cli.js [mode] [--help]'); 14 | console.log(); 15 | console.log('Args:'); 16 | console.log(' --help Display this message'); 17 | console.log('Mode: fetch [--approach] '); 18 | console.log(' Retrieve all geospatial features from a single layer'); 19 | console.log(' --header \'key=value\' IE --header \'Content-Type=123\''); 20 | console.log(' --approach [approach] Download Approach'); 21 | console.log(' "bbox" Download features by iterating over bboxes'); 22 | console.log(' slowest but most reliable approach'); 23 | console.log(' "iter" Iterate over OIDs'); 24 | console.log(' faster but not supported by all servers'); 25 | console.log('Mode: schema '); 26 | console.log(' Return a JSON Schema for a given layer'); 27 | console.log('Mode: discover '); 28 | console.log(' Locate all geospatial layers on a given server'); 29 | console.log(); 30 | process.exit(); 31 | } 32 | 33 | if (!argv._[2]) throw new Error('Mode required'); 34 | 35 | const url = argv._[3]; 36 | if (!url) throw new Error('url required'); 37 | 38 | const headers: any = {}; 39 | if (argv.header) { 40 | if (typeof argv.header === 'string') argv.header = [ argv.header ]; 41 | for (const header of argv.header) { 42 | const parsed = header.split('='); 43 | headers[parsed[0]] = parsed.slice(1, parsed.length).join('='); 44 | } 45 | } 46 | 47 | 48 | const esri = new EsriDump(url, { 49 | approach: argv.approach, 50 | headers 51 | }); 52 | 53 | 54 | if (argv._[2] === 'fetch') { 55 | esri.on('error', (err) => { 56 | console.error(err); 57 | }).on('feature', (feature) => { 58 | console.log(JSON.stringify(feature)); 59 | }); 60 | 61 | await esri.fetch(); 62 | } else if (argv._[2] === 'schema') { 63 | console.log(JSON.stringify(await esri.schema(), null, 4)); 64 | } else if (argv._[2] === 'discover') { 65 | esri.on('error', (err) => { 66 | console.error(err); 67 | }).on('service', (service) => { 68 | console.log(JSON.stringify(service)); 69 | }).on('layer', (layer) => { 70 | console.log(JSON.stringify(layer)); 71 | }); 72 | 73 | await esri.discover(); 74 | } else { 75 | throw new Error('Unknown Mode'); 76 | } 77 | 78 | -------------------------------------------------------------------------------- /test/fixtures/rewind/geomcollection.output.geojson: -------------------------------------------------------------------------------- 1 | { 2 | "type": "GeometryCollection", 3 | "geometries": [ 4 | { 5 | "type": "Point", 6 | "coordinates": [ 7 | 102, 8 | 0.5 9 | ] 10 | }, 11 | { 12 | "type": "LineString", 13 | "coordinates": [ 14 | [ 15 | 102, 16 | 0 17 | ], 18 | [ 19 | 103, 20 | 1 21 | ], 22 | [ 23 | 104, 24 | 0 25 | ], 26 | [ 27 | 105, 28 | 1 29 | ] 30 | ] 31 | }, 32 | { 33 | "type": "Polygon", 34 | "coordinates": [ 35 | [ 36 | [ 37 | 100, 38 | 0 39 | ], 40 | [ 41 | 101, 42 | 0 43 | ], 44 | [ 45 | 101, 46 | 1 47 | ], 48 | [ 49 | 100, 50 | 1 51 | ], 52 | [ 53 | 100, 54 | 0 55 | ] 56 | ] 57 | ] 58 | }, 59 | { 60 | "type": "Polygon", 61 | "coordinates": [ 62 | [ 63 | [ 64 | 100, 65 | 0 66 | ], 67 | [ 68 | 101, 69 | 0 70 | ], 71 | [ 72 | 101, 73 | 1 74 | ], 75 | [ 76 | 100, 77 | 1 78 | ], 79 | [ 80 | 100, 81 | 0 82 | ] 83 | ], 84 | [ 85 | [ 86 | 100.2, 87 | 0.2 88 | ], 89 | [ 90 | 100.2, 91 | 0.8 92 | ], 93 | [ 94 | 100.8, 95 | 0.8 96 | ], 97 | [ 98 | 100.8, 99 | 0.2 100 | ], 101 | [ 102 | 100.2, 103 | 0.2 104 | ] 105 | ] 106 | ] 107 | } 108 | ] 109 | } -------------------------------------------------------------------------------- /test/server.ts: -------------------------------------------------------------------------------- 1 | import http from 'http'; 2 | import fs from 'fs'; 3 | 4 | const server = http.createServer(handleRequest); 5 | 6 | let options: any; 7 | 8 | if (process.argv[2] === 'start') { 9 | Server({ 10 | mode: process.argv[3] 11 | }, (() =>{ 12 | console.log('Server Started'); 13 | })); 14 | } 15 | 16 | export default function Server(opts: any, cb: any) { 17 | if (!opts.mode) { throw new Error('options.mode must be set'); } 18 | options = opts; 19 | 20 | server.listen(3000, () =>{ 21 | cb(stop); 22 | }); 23 | 24 | } 25 | 26 | function stop(cb: any) { 27 | server.close(cb); 28 | } 29 | 30 | const r: any = { 31 | '/arcgis/rest/services/images/ImageServer?f=json': { 32 | download: { 33 | data: JSON.parse(String(fs.readFileSync(new URL('./fixtures/ImageServer_json_Download.json', import.meta.url)))) 34 | }, 35 | noDownload: { 36 | data: JSON.parse(String(fs.readFileSync(new URL('./fixtures/ImageServer_json_noDownload.json', import.meta.url)))) 37 | } 38 | }, 39 | '/arcgis/rest/services/images/ImageServer/1?f=json': { 40 | download: { 41 | data: JSON.parse(String(fs.readFileSync(new URL('./fixtures/ImageServer-1_json_Download.json', import.meta.url)))) 42 | } 43 | }, 44 | '/arcgis/rest/services/images/ImageServer/2?f=json': { 45 | download: { 46 | data: JSON.parse(String(fs.readFileSync(new URL('./fixtures/ImageServer-2_json_Download.json', import.meta.url)))) 47 | } 48 | }, 49 | '/arcgis/rest/services/images/ImageServer/3?f=json': { 50 | download: { 51 | data: JSON.parse(String(fs.readFileSync(new URL('./fixtures/ImageServer-3_json_Download.json', import.meta.url)))) 52 | } 53 | }, 54 | '/arcgis/rest/services/images/ImageServer/download?rasterIds=1&geometryType=esriGeometryEnvelope&f=json': { 55 | download: { 56 | data: JSON.parse(String(fs.readFileSync(new URL('./fixtures/ImageServer-Download-1_json_Download.json', import.meta.url)))) 57 | } 58 | }, 59 | '/arcgis/rest/services/images/ImageServer/download?rasterIds=2&geometryType=esriGeometryEnvelope&f=json': { 60 | download: { 61 | data: JSON.parse(String(fs.readFileSync(new URL('./fixtures/ImageServer-Download-2_json_Download.json', import.meta.url)))) 62 | } 63 | } 64 | }; 65 | 66 | function handleRequest(request: any, response: any) { 67 | if (options.debug) { 68 | console.log('#', request.url); 69 | } 70 | if (r[request.url] && r[request.url][options.mode]) { 71 | response.writeHead( 72 | r[request.url][options.mode].code ? r[request.url][options.mode].code : 200, 73 | r[request.url][options.mode].header ? r[request.url][options.mode].header : { 'Content-Type': 'application/json' } 74 | ); 75 | response.end(JSON.stringify(r[request.url][options.mode].data)); 76 | } else { 77 | throw new Error(request.url + ' NOT FOUND'); 78 | } 79 | } 80 | 81 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # CHANGELOG 2 | 3 | ## Emoji Cheatsheet 4 | - :pencil2: doc updates 5 | - :bug: when fixing a bug 6 | - :rocket: when making general improvements 7 | - :white_check_mark: when adding tests 8 | - :arrow_up: when upgrading dependencies 9 | - :tada: when adding new features 10 | 11 | ## Version History 12 | 13 | ### v5.5.1 14 | 15 | - :bug: Fix ESRI Dump Approach config 16 | 17 | ### v5.5.0 18 | 19 | - :rocket: Allow QueryTopFeatures option 20 | 21 | ### v5.4.0 22 | 23 | - :rocket: Surface upstream response if no count is present 24 | 25 | ### v5.3.0 26 | 27 | - :bug: If no `esriFeature.geometry` is returned, skip the Feature 28 | 29 | ### v5.2.0 30 | 31 | - :bug: Fix `null` parsing as date in geometry parser 32 | - :rocket: Switch to Flat Config ESLint 33 | - :arrow_up: Update Core Deps 34 | 35 | ### v5.1.0 36 | 37 | - :tada: Add optional config parameter with `map` feature for modifying features with full `Geometry` context 38 | 39 | ### v5.0.0 40 | 41 | - :tada: Support the `format` tag in the Schema for Dates 42 | - :rocket: Return Dates as ISO formatted strings to conform to schema 43 | 44 | ### v4.6.2 45 | 46 | - :bug: Fix the way headers were appended 47 | 48 | ### v4.6.1 49 | 50 | - :arrow_up: Update base deps 51 | 52 | ### v4.6.0 53 | 54 | - :bug: If `meta.count` was 0 the unable to determine count error would be thrown incorrectly 55 | 56 | ### v4.5.0 57 | 58 | - :rocket: Use user provided `where` parameter where possible 59 | 60 | ### v4.4.1 61 | 62 | - :bug: Include built version 63 | 64 | ### v4.4.0 65 | 66 | - :bug: Cleaner error messages in the CLI 67 | - :tada: Set `feature.id` as the OID field 68 | - :bug: Fix double `f=json` when fetching metadata causing a failure 69 | 70 | ### v4.3.2 71 | 72 | - :bug: Don't throw on metadata.fields not being present 73 | 74 | ### v4.3.1 75 | 76 | - :bug: Include dist directory for now 77 | 78 | ### v4.3.0 79 | 80 | - :rocket: Add initial `discovery` mode for finding all services and layers on a server 81 | 82 | ### v4.2.1 83 | 84 | - :bug: Output Schema to STDOut when using CLI 85 | 86 | ### v4.2.0 87 | 88 | - :bug: Use `gzip` Encoding by default 89 | 90 | ### v4.1.1 91 | 92 | - :bug: Fix JS build 93 | 94 | ### v4.1.0 95 | 96 | - :rocket: Be more specific about type of returned schema 97 | 98 | ### v4.0.0 99 | 100 | - :tada: Adds a `schema` mode to allow parsing a Feature Layer as JSON Schema 101 | - :rocket: **Breaking** Update the CLI to have a "mode" that must be specified 102 | - :rocket: Automatically rewind GeoJSON Polygons to enforce the Right-Hand-Rule 103 | - :rocket: Add Support for `--header` param in CLI 104 | 105 | ### v3.1.0 106 | 107 | - :rocket: Fix ESRI Dump Bin 108 | 109 | ### v3.0.1 110 | 111 | - :bug: Include Types Field & Build for ES 112 | 113 | ### v3.0.0 114 | 115 | - :rocket: Migrate library to TypeScript 116 | - :tada: Add support for `params` and `headers` properties in initial config 117 | 118 | ### v2.0.0 119 | 120 | - :tada: Rewrite as ES Module 121 | - :tada: Add BBOX & Iter Fetch Methodologies 122 | -------------------------------------------------------------------------------- /test/geometry.test.ts: -------------------------------------------------------------------------------- 1 | import test from 'tape'; 2 | import Geometry from '../lib/geometry.js'; 3 | 4 | test('geometry#splitBbox', (t) => { 5 | t.deepEquals(Geometry.splitBbox({ 6 | xmin: -97.0189932385465, 7 | ymin: 20.52053000026018, 8 | xmax: -88.57449931419137, 9 | ymax: 29.116263085773653 10 | }), [ 11 | { 12 | xmax: -92.79674627636894, 13 | xmin: -97.0189932385465, 14 | ymax: 24.818396543016917, 15 | ymin: 20.52053000026018 16 | }, { 17 | xmax: -88.57449931419137, 18 | xmin: -92.79674627636894, 19 | ymax: 24.818396543016917, 20 | ymin: 20.52053000026018 21 | }, { 22 | xmax: -92.79674627636894, 23 | xmin: -97.0189932385465, 24 | ymax: 29.116263085773653, 25 | ymin: 24.818396543016917 26 | }, { 27 | xmax: -88.57449931419137, 28 | xmin: -92.79674627636894, 29 | ymax: 29.116263085773653, 30 | ymin: 24.818396543016917 31 | } 32 | ], 'returns split bbox'); 33 | 34 | t.deepEquals(Geometry.splitBbox({ 35 | xmin: 2, 36 | ymin: 2, 37 | xmax: 4, 38 | ymax: 4 39 | }), [ 40 | { 41 | xmax: 3, 42 | xmin: 2, 43 | ymax: 3, 44 | ymin: 2 45 | }, { 46 | xmax: 4, 47 | xmin: 3, 48 | ymax: 3, 49 | ymin: 2 50 | }, { 51 | xmax: 3, 52 | xmin: 2, 53 | ymax: 4, 54 | ymin: 3 55 | }, { 56 | xmax: 4, 57 | xmin: 3, 58 | ymax: 4, 59 | ymin: 3 60 | } 61 | ], 'returns split bbox'); 62 | t.end(); 63 | }); 64 | 65 | test('geometry#findOidField', (t) =>{ 66 | t.equals(Geometry.findOidField([{ 67 | name: 'test', 68 | type: 'esriFieldTypeOID', 69 | alias: 'st_length(shape)', 70 | domain: null 71 | }]), 'test', 'Find Oid Field'); 72 | 73 | t.equals(Geometry.findOidField([{ 74 | name: 'id', 75 | type: 'esriTypeDouble', 76 | alias: 'st_length(shape)', 77 | domain: null 78 | }]), 'id', 'Finds a suitable ID field'); 79 | 80 | t.equals(Geometry.findOidField([{ 81 | name: 'id', 82 | type: 'esriTypeDouble', 83 | alias: 'st_length(shape)', 84 | domain: null 85 | },{ 86 | name: 'objectid', 87 | type: 'esriTypeString', 88 | alias: 'st_length(shape)', 89 | domain: null 90 | }]), 'objectid', 'Finds the best available ID field'); 91 | 92 | t.throws(() => { 93 | Geometry.findOidField([{ 94 | name: 'test', 95 | type: 'esriTypeDouble', 96 | alias: 'st_length(shape)', 97 | domain: null 98 | }]); 99 | }, /Could not determine OBJECTID field./, 'Recognizes absense of any likely OBJECTID field'); 100 | 101 | t.end(); 102 | }); 103 | -------------------------------------------------------------------------------- /test/fixtures/rewind/collection.output.geojson: -------------------------------------------------------------------------------- 1 | { 2 | "type": "FeatureCollection", 3 | "features": [ 4 | { 5 | "type": "Feature", 6 | "geometry": { 7 | "type": "Point", 8 | "coordinates": [ 9 | 102, 10 | 0.5 11 | ] 12 | }, 13 | "properties": { 14 | "prop0": "value0" 15 | } 16 | }, 17 | { 18 | "type": "Feature", 19 | "geometry": { 20 | "type": "LineString", 21 | "coordinates": [ 22 | [ 23 | 102, 24 | 0 25 | ], 26 | [ 27 | 103, 28 | 1 29 | ], 30 | [ 31 | 104, 32 | 0 33 | ], 34 | [ 35 | 105, 36 | 1 37 | ] 38 | ] 39 | }, 40 | "properties": { 41 | "prop0": "value0", 42 | "prop1": 0 43 | } 44 | }, 45 | { 46 | "type": "Feature", 47 | "geometry": { 48 | "type": "Polygon", 49 | "coordinates": [ 50 | [ 51 | [ 52 | 100, 53 | 0 54 | ], 55 | [ 56 | 101, 57 | 0 58 | ], 59 | [ 60 | 101, 61 | 1 62 | ], 63 | [ 64 | 100, 65 | 1 66 | ], 67 | [ 68 | 100, 69 | 0 70 | ] 71 | ] 72 | ] 73 | }, 74 | "properties": { 75 | "prop0": "value0", 76 | "prop1": { 77 | "this": "that" 78 | } 79 | } 80 | }, 81 | { 82 | "type": "Feature", 83 | "geometry": { 84 | "type": "Polygon", 85 | "coordinates": [ 86 | [ 87 | [ 88 | 100, 89 | 0 90 | ], 91 | [ 92 | 101, 93 | 0 94 | ], 95 | [ 96 | 101, 97 | 1 98 | ], 99 | [ 100 | 100, 101 | 1 102 | ], 103 | [ 104 | 100, 105 | 0 106 | ] 107 | ], 108 | [ 109 | [ 110 | 100.2, 111 | 0.2 112 | ], 113 | [ 114 | 100.2, 115 | 0.8 116 | ], 117 | [ 118 | 100.8, 119 | 0.8 120 | ], 121 | [ 122 | 100.8, 123 | 0.2 124 | ], 125 | [ 126 | 100.2, 127 | 0.2 128 | ] 129 | ] 130 | ] 131 | }, 132 | "properties": { 133 | "prop0": "value0", 134 | "prop1": { 135 | "this": "that" 136 | } 137 | } 138 | } 139 | ] 140 | } -------------------------------------------------------------------------------- /lib/discovery.ts: -------------------------------------------------------------------------------- 1 | import EventEmitter from 'node:events'; 2 | import Fetch from './fetch.js'; 3 | import { EsriDumpConfig } from '../index.js'; 4 | import Schema from './schema.js'; 5 | 6 | export interface DiscoveryDocument { 7 | version?: string, 8 | collections: object[] 9 | } 10 | 11 | interface InternalService { 12 | name: string; 13 | service: string; 14 | url: URL; 15 | } 16 | 17 | export default class Discovery extends EventEmitter { 18 | baseUrl: URL; 19 | document: DiscoveryDocument; 20 | 21 | constructor(url: URL) { 22 | super(); 23 | 24 | url.pathname = url.pathname.replace(/\/rest\/services.*/, '/rest/services'); 25 | this.baseUrl = url; 26 | 27 | this.document = { 28 | version: undefined, 29 | collections: [], 30 | } 31 | 32 | } 33 | 34 | async fetch(config: EsriDumpConfig): Promise { 35 | if (process.env.DEBUG) console.error(String(this.baseUrl)); 36 | let base = await Fetch(config, this.baseUrl); 37 | if (!base.ok) return this.emit('error', await base.text()); 38 | base = await base.json(); 39 | 40 | this.document.version = String(base.version); 41 | 42 | await this.#request(config, base); 43 | 44 | this.emit('done'); 45 | 46 | return this.document; 47 | } 48 | 49 | async #request(config: EsriDumpConfig, base: any) { 50 | const services: InternalService[] = base.services.map((service: any) => { 51 | const url = new URL(this.baseUrl); 52 | url.pathname = url.pathname + '/' + service.name + '/' + service.type; 53 | return { 54 | url, 55 | name: String(service.name), 56 | type: String(service.type), 57 | }; 58 | }); 59 | 60 | services.push(...(await this.#folders(config, base.folders))); 61 | 62 | for (const service_meta of services) { 63 | const service = await this.#service(config, service_meta); 64 | this.emit('service', service); 65 | 66 | if (!service.layers) service.layers = []; 67 | for (const layer_meta of service.layers) { 68 | const url = new URL(service_meta.url) 69 | url.pathname = url.pathname + '/' + layer_meta.id; 70 | 71 | const layer = await this.#layer(config, url); 72 | layer.schema = Schema(layer); 73 | this.emit('layer', layer); 74 | } 75 | } 76 | } 77 | 78 | async #layer(config: EsriDumpConfig, layer_url: URL): Promise { 79 | if (process.env.DEBUG) console.error(String(layer_url)); 80 | 81 | const req = await Fetch(config, layer_url); 82 | if (!req.ok) { 83 | this.emit('error', await req.text()); 84 | } 85 | 86 | const service = await req.json(); 87 | 88 | return service; 89 | } 90 | 91 | async #service(config: EsriDumpConfig, service_meta: InternalService): Promise { 92 | const url = new URL(service_meta.url); 93 | if (process.env.DEBUG) console.error(String(url)); 94 | 95 | const req = await Fetch(config, url); 96 | if (!req.ok) { 97 | this.emit('error', await req.text()); 98 | } 99 | 100 | const service = await req.json(); 101 | 102 | return service; 103 | } 104 | 105 | async #folders(config: EsriDumpConfig, folders: string[]): Promise { 106 | const services: any[] = []; 107 | 108 | for (const folder of folders) { 109 | const url = new URL(this.baseUrl); 110 | url.pathname = url.pathname + '/' + folder; 111 | if (process.env.DEBUG) console.error(String(url)); 112 | 113 | let req = await Fetch(config, url); 114 | if (!req.ok) { 115 | this.emit('error', await req.text()); 116 | return services; 117 | } 118 | 119 | req = await req.json(); 120 | 121 | if (req.folders && Array.isArray(req.folders) && req.folders.length) { 122 | services.push(...await this.#folders(config, req.folders)); 123 | } 124 | 125 | services.push(...req.services.map((service: any) => { 126 | const url = new URL(this.baseUrl); 127 | url.pathname = url.pathname + '/' + service.name + '/' + service.type; 128 | return { 129 | url, 130 | name: service.name, 131 | type: service.type, 132 | }; 133 | })); 134 | } 135 | 136 | return services; 137 | } 138 | } 139 | 140 | -------------------------------------------------------------------------------- /test/fixtures/pass-imageserver.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "FeatureCollection", 3 | "features": [ 4 | { 5 | "type": "Feature", 6 | "properties": { 7 | "OBJECTID": 1, 8 | "Name": "image", 9 | "MinPS": 0, 10 | "MaxPS": 20, 11 | "LowPS": 0.5, 12 | "HighPS": 2, 13 | "Category": 1, 14 | "Tag": "Dataset", 15 | "GroupName": "", 16 | "ProductName": "", 17 | "CenterX": 2335067.800079886, 18 | "CenterY": 1538781.926635325, 19 | "ZOrder": null, 20 | "Shape_Length": 25246.863514616693, 21 | "Shape_Area": 38837213.29536865, 22 | "files": [ 23 | { 24 | "url": "http://localhost:3000/image.tif", 25 | "name": "image.tif" 26 | }, 27 | { 28 | "url": "http://localhost:3000/image.tif.ovr", 29 | "name": "image.tif.ovr" 30 | }, 31 | { 32 | "url": "http://localhost:3000/image.tif.aux.xml", 33 | "name": "image.tif.aux.xml" 34 | } 35 | ] 36 | }, 37 | "geometry": { 38 | "type": "Polygon", 39 | "coordinates": [ 40 | [ 41 | [ 42 | -116.12798172582659, 43 | 44.99864757218559 44 | ], 45 | [ 46 | -116.12698194609382, 47 | 45.064443651862696 48 | ], 49 | [ 50 | -116.05954236196241, 51 | 45.06390817934289 52 | ], 53 | [ 54 | -116.06061934691981, 55 | 44.99811331935956 56 | ], 57 | [ 58 | -116.12798172582659, 59 | 44.99864757218559 60 | ] 61 | ] 62 | ] 63 | } 64 | }, 65 | { 66 | "type": "Feature", 67 | "properties": { 68 | "OBJECTID": 2, 69 | "Name": "image", 70 | "MinPS": 0, 71 | "MaxPS": 20, 72 | "LowPS": 0.5, 73 | "HighPS": 2, 74 | "Category": 1, 75 | "Tag": "Dataset", 76 | "GroupName": "", 77 | "ProductName": "", 78 | "CenterX": 2350782.578273649, 79 | "CenterY": 1955351.2145930487, 80 | "ZOrder": null, 81 | "Shape_Length": 24625.204791258868, 82 | "Shape_Area": 36542245.57806721, 83 | "files": [ 84 | { 85 | "url": "http://localhost:3000/image.tif", 86 | "name": "image.tif" 87 | }, 88 | { 89 | "url": "http://localhost:3000/image.tif.ovr", 90 | "name": "image.tif.ovr" 91 | }, 92 | { 93 | "url": "http://localhost:3000/image.tif.aux.xml", 94 | "name": "image.tif.aux.xml" 95 | } 96 | ] 97 | }, 98 | "geometry": { 99 | "type": "Polygon", 100 | "coordinates": [ 101 | [ 102 | [ 103 | -116.06583526675134, 104 | 48.74859692655888 105 | ], 106 | [ 107 | -116.06461248296286, 108 | 48.81443857239383 109 | ], 110 | [ 111 | -115.99665178513598, 112 | 48.81386703827789 113 | ], 114 | [ 115 | -115.99796337504938, 116 | 48.74802670688809 117 | ], 118 | [ 119 | -116.06583526675134, 120 | 48.74859692655888 121 | ] 122 | ] 123 | ] 124 | } 125 | } 126 | ] 127 | } 128 | -------------------------------------------------------------------------------- /lib/rings2geojson.ts: -------------------------------------------------------------------------------- 1 | import { Geometry } from 'geojson'; 2 | 3 | /* Code from https://github.com/Esri/Terraformer 4 | and https://github.com/Esri/terraformer-arcgis-parser 5 | Copyright (c) 2013 Esri, Inc 6 | */ 7 | 8 | // Determine if polygon ring coordinates are clockwise. clockwise signifies outer ring, counter-clockwise an inner ring 9 | // or hole. this logic was found at http://stackoverflow.com/questions/1165647/how-to-determine-if-a-list-of-polygon- 10 | // points-are-in-clockwise-order 11 | function ringIsClockwise(ringToTest: number[][]) { 12 | let total = 0, 13 | i = 0, 14 | pt1 = ringToTest[i], 15 | pt2; 16 | const rLength = ringToTest.length; 17 | for (i; i < rLength - 1; i++) { 18 | pt2 = ringToTest[i + 1]; 19 | total += (pt2[0] - pt1[0]) * (pt2[1] + pt1[1]); 20 | pt1 = pt2; 21 | } 22 | return (total >= 0); 23 | } 24 | 25 | // checks if the first and last points of a ring are equal and closes the ring 26 | 27 | function closeRing(coordinates: number[][]): number[][] { 28 | if (!pointsEqual(coordinates[0], coordinates[coordinates.length - 1])) { 29 | coordinates.push(coordinates[0]); 30 | } 31 | return coordinates; 32 | } 33 | 34 | // checks if 2 x,y points are equal 35 | 36 | function pointsEqual(a: number[], b: number[]): boolean { 37 | for (let i = 0; i < a.length; i++) { 38 | if (a[i] !== b[i]) { 39 | return false; 40 | } 41 | } 42 | return true; 43 | } 44 | 45 | function coordinatesContainCoordinates(outer: number[][], inner: number[][]): boolean { 46 | const intersects = arraysIntersectArrays(outer, inner); 47 | const contains = coordinatesContainPoint(outer, inner[0]); 48 | if (!intersects && contains) { 49 | return true; 50 | } 51 | return false; 52 | } 53 | 54 | function coordinatesContainPoint(coordinates: number[][], point: number[]): boolean { 55 | let contains = false; 56 | for (let i = -1, l = coordinates.length, j = l - 1; ++i < l; j = i) { 57 | if (((coordinates[i][1] <= point[1] && point[1] < coordinates[j][1]) || 58 | (coordinates[j][1] <= point[1] && point[1] < coordinates[i][1])) && 59 | (point[0] < (coordinates[j][0] - coordinates[i][0]) * (point[1] - coordinates[i][1]) / (coordinates[j][1] - coordinates[i][1]) + coordinates[i][0])) { 60 | contains = !contains; 61 | } 62 | } 63 | return contains; 64 | } 65 | 66 | function isNumber(n: unknown): boolean { 67 | return !isNaN(parseFloat(n as string)) && isFinite(parseFloat(n as string)); 68 | } 69 | 70 | function edgeIntersectsEdge(a1: number[], a2: number[], b1: number[], b2: number[]): boolean { 71 | const ua_t = (b2[0] - b1[0]) * (a1[1] - b1[1]) - (b2[1] - b1[1]) * (a1[0] - b1[0]); 72 | const ub_t = (a2[0] - a1[0]) * (a1[1] - b1[1]) - (a2[1] - a1[1]) * (a1[0] - b1[0]); 73 | const u_b = (b2[1] - b1[1]) * (a2[0] - a1[0]) - (b2[0] - b1[0]) * (a2[1] - a1[1]); 74 | 75 | if (u_b !== 0) { 76 | const ua = ua_t / u_b; 77 | const ub = ub_t / u_b; 78 | 79 | if (0 <= ua && ua <= 1 && 0 <= ub && ub <= 1) { 80 | return true; 81 | } 82 | } 83 | 84 | return false; 85 | } 86 | 87 | function arraysIntersectArrays(a: number[][] | number[], b: number[][] | number[]): boolean { 88 | if (Array.isArray(a[0]) && isNumber(a[0][0])) { 89 | if (Array.isArray(b[0]) && isNumber(b[0][0])) { 90 | a = a as number[][]; 91 | b = b as number[][]; 92 | 93 | for (let i = 0; i < a.length - 1; i++) { 94 | for (let j = 0; j < b.length - 1; j++) { 95 | if (edgeIntersectsEdge(a[i], a[i + 1], b[j], b[j + 1])) { 96 | return true; 97 | } 98 | } 99 | } 100 | } else { 101 | a = a as number[] 102 | b = b as number[][]; 103 | 104 | for (let k = 0; k < b.length; k++) { 105 | if (arraysIntersectArrays(a, b[k])) { 106 | return true; 107 | } 108 | } 109 | } 110 | } else { 111 | a = a as number[][] 112 | b = b as number[]; 113 | for (let l = 0; l < a.length; l++) { 114 | if (arraysIntersectArrays(a[l], b)) { 115 | return true; 116 | } 117 | } 118 | } 119 | return false; 120 | } 121 | 122 | // Do any polygons in this array contain any other polygons in this array? 123 | // used for checking for holes in arcgis rings 124 | // from https://github.com/Esri/terraformer-arcgis-parser/blob/master/terraformer-arcgis-parser.js#L170 125 | 126 | export default function (rings: number[][][]): Geometry { 127 | const outerRings = []; 128 | const holes = []; 129 | 130 | // for each ring 131 | for (let r = 0; r < rings.length; r++) { 132 | const ring = closeRing(rings[r].slice(0)); 133 | if (ring.length < 4) { 134 | continue; 135 | } 136 | // is this ring an outer ring? is it clockwise? 137 | if (ringIsClockwise(ring)) { 138 | const polygon = [ring]; 139 | outerRings.push(polygon); // push to outer rings 140 | } else { 141 | holes.push(ring); // counterclockwise push to holes 142 | } 143 | } 144 | 145 | // while there are holes left... 146 | while (holes.length) { 147 | // pop a hole off out stack 148 | const hole = holes.pop(); 149 | let matched = false; 150 | 151 | // loop over all outer rings and see if they contain our hole. 152 | for (let x = outerRings.length - 1; x >= 0; x--) { 153 | const outerRing = outerRings[x][0]; 154 | if (coordinatesContainCoordinates(outerRing, hole)) { 155 | // the hole is contained push it into our polygon 156 | outerRings[x].push(hole); 157 | 158 | // we matched the hole 159 | matched = true; 160 | 161 | // stop checking to see if other outer rings contian this hole 162 | break; 163 | } 164 | } 165 | 166 | // no outer rings contain this hole turn it into and outer ring (reverse it) 167 | if (!matched) { 168 | outerRings.push([hole.reverse()]); 169 | } 170 | } 171 | 172 | if (outerRings.length === 1) { 173 | return { 174 | type: 'Polygon', 175 | coordinates: outerRings[0] 176 | }; 177 | } else { 178 | return { 179 | type: 'MultiPolygon', 180 | coordinates: outerRings 181 | }; 182 | } 183 | } 184 | -------------------------------------------------------------------------------- /index.ts: -------------------------------------------------------------------------------- 1 | import Geometry from './lib/geometry.js'; 2 | import Discovery from './lib/discovery.js'; 3 | import Fetch from './lib/fetch.js'; 4 | import EventEmitter from 'node:events'; 5 | import { Feature } from 'geojson'; 6 | import Err from '@openaddresses/batch-error'; 7 | import rewind from './lib/rewind.js'; 8 | import Schema from './lib/schema.js'; 9 | import { 10 | JSONSchema6, 11 | } from 'json-schema'; 12 | 13 | const SUPPORTED = ['FeatureServer', 'MapServer']; 14 | 15 | export enum EsriDumpConfigApproach { 16 | BBOX = 'bbox', 17 | ITER = 'iter', 18 | TOP_FEATURES_BBOX = 'top_features_bbox', 19 | TOP_FEATURES_ITER = 'top_features_iter', 20 | } 21 | 22 | export enum EsriResourceType { 23 | FeatureServer = 'FeatureServer', 24 | MapServer = 'MapServer' 25 | } 26 | 27 | export interface EsriDumpConfigInput { 28 | approach?: EsriDumpConfigApproach; 29 | headers?: { [k: string]: string; }; 30 | params?: { [k: string]: string; }; 31 | } 32 | 33 | export interface EsriDumpConfig { 34 | approach: EsriDumpConfigApproach; 35 | headers: { [k: string]: string; }; 36 | params: { [k: string]: string; }; 37 | } 38 | 39 | export default class EsriDump extends EventEmitter { 40 | url: URL; 41 | config: EsriDumpConfig; 42 | geomType: null | string; 43 | resourceType: EsriResourceType; 44 | 45 | constructor(url: string, config: EsriDumpConfigInput = {}) { 46 | super(); 47 | 48 | this.url = new URL(url); 49 | 50 | this.config = { 51 | approach: config.approach || EsriDumpConfigApproach.BBOX, 52 | headers: config.headers || {}, 53 | params: config.params || {} 54 | }; 55 | 56 | // Validate URL is a "/rest/services/" endpoint 57 | if (!this.url.pathname.includes('/rest/services/')) throw new Err(400, null, 'Did not recognize ' + url + ' as an ArcGIS /rest/services/ endpoint.'); 58 | 59 | this.geomType = null; 60 | 61 | const occurrence = SUPPORTED.map((d) => { return url.lastIndexOf(d); }); 62 | const known = SUPPORTED[occurrence.indexOf(Math.max.apply(null, occurrence))]; 63 | if (known === 'MapServer') this.resourceType = EsriResourceType.MapServer; 64 | else if (known === 'FeatureServer') this.resourceType = EsriResourceType.FeatureServer; 65 | else throw new Err(400, null, 'Unknown or unsupported ESRI URL Format'); 66 | 67 | this.emit('type', this.resourceType); 68 | } 69 | 70 | async schema(): Promise { 71 | const metadata = await this.#fetchMeta(); 72 | return Schema(metadata); 73 | } 74 | 75 | async discover() { 76 | try { 77 | const discover = new Discovery(this.url); 78 | discover.fetch(this.config); 79 | 80 | discover.on('layer', (layer: any) => { 81 | this.emit('layer', layer); 82 | }).on('schema', (schema: JSONSchema6) => { 83 | this.emit('schema', schema); 84 | }).on('error', (error: Err) => { 85 | this.emit('error', error); 86 | }).on('done', () => { 87 | this.emit('done'); 88 | }); 89 | } catch (err) { 90 | this.emit('error', err); 91 | } 92 | } 93 | 94 | async fetch(config?: { 95 | map?: (g: Geometry, f: Feature) => Feature 96 | }) { 97 | if (!config) config = {}; 98 | const metadata = await this.#fetchMeta(); 99 | 100 | try { 101 | const geom = new Geometry(this.url, metadata); 102 | geom.fetch(this.config); 103 | 104 | geom.on('feature', (feature: Feature) => { 105 | feature = rewind(feature) as Feature; 106 | if (config.map) feature = config.map(geom, feature); 107 | this.emit('feature', feature); 108 | }).on('error', (error: Err) => { 109 | this.emit('error', error); 110 | }).on('done', () => { 111 | this.emit('done'); 112 | }); 113 | } catch (err) { 114 | this.emit('error', err); 115 | } 116 | } 117 | 118 | async #fetchMeta() { 119 | const url = new URL(this.url); 120 | 121 | if (process.env.DEBUG) console.error(String(url)); 122 | const res = await Fetch(this.config, url); 123 | 124 | if (!res.ok) this.emit('error', await res.text()); 125 | 126 | // TODO: Type Defs 127 | const metadata: any = await res.json(); 128 | 129 | if (metadata.error) { 130 | return this.emit('error', new Err(400, null, 'Server metadata error: ' + metadata.error.message)); 131 | } else if (metadata.capabilities && metadata.capabilities.indexOf('Query') === -1 ) { 132 | return this.emit('error', new Err(400, null, 'Layer doesn\'t support query operation.')); 133 | } else if (metadata.folders || metadata.services) { 134 | let errorMessage = 'Endpoint provided is not a Server resource.\n'; 135 | if (metadata.folders.length > 0) { 136 | errorMessage += '\nChoose a Layer from a Service in one of these Folders: \n ' 137 | + metadata.folders.join('\n ') + '\n'; 138 | } 139 | 140 | if (metadata.services.length > 0 && Array.isArray(metadata.services)) { 141 | errorMessage += '\nChoose a Layer from one of these Services: \n ' 142 | + metadata.services.map((d: any) => { return d.name; }).join('\n ') + '\n'; 143 | } 144 | 145 | return this.emit('error', new Err(400, null, errorMessage)); 146 | } else if (metadata.layers) { 147 | let errorMessage = 'Endpoint provided is not a Server resource.\n'; 148 | if (metadata.layers.length > 0 && Array.isArray(metadata.layers)) { 149 | errorMessage += '\nChoose one of these Layers: \n ' 150 | + metadata.layers.map((d: any) => { return d.name; }).join('\n ') + '\n'; 151 | } 152 | return this.emit('error', new Err(400, null, errorMessage)); 153 | } else if (!this.resourceType) { 154 | return this.emit('error', new Err(400, null, 'Could not determine server type of ' + url)); 155 | } 156 | 157 | this.geomType = metadata.geometryType; 158 | 159 | if (!this.geomType) { 160 | return this.emit('error', new Err(400, null, 'no geometry')); 161 | } else if (!metadata.extent) { 162 | return this.emit('error', new Err(400, null, 'Layer doesn\'t list an extent.')); 163 | } else if ('subLayers' in metadata && metadata.subLayers.length > 0) { 164 | return this.emit('error', new Err(400, null, 'Specified layer has sublayers.')); 165 | } 166 | 167 | return metadata; 168 | } 169 | } 170 | 171 | export { 172 | Geometry 173 | } 174 | 175 | -------------------------------------------------------------------------------- /test/fixtures/ImageServer_json_noDownload.json: -------------------------------------------------------------------------------- 1 | { 2 | "currentVersion": 10.3, 3 | "serviceDescription": "Lots of words", 4 | "name": "image/images", 5 | "description": "Even more words", 6 | "extent": { 7 | "xmin": 2240369.8233000003, 8 | "ymin": 1192725.063299999, 9 | "xmax": 2748620.3233000003, 10 | "ymax": 1987830.063299999, 11 | "spatialReference": { 12 | "wkid": 102605, 13 | "latestWkid": 102605 14 | } 15 | }, 16 | "initialExtent": { 17 | "xmin": 2240369.8233000003, 18 | "ymin": 1192725.063299999, 19 | "xmax": 2748620.3233000003, 20 | "ymax": 1987830.063299999, 21 | "spatialReference": { 22 | "wkid": 102605, 23 | "latestWkid": 102605 24 | } 25 | }, 26 | "fullExtent": { 27 | "xmin": 2240369.8233000003, 28 | "ymin": 1192725.063299999, 29 | "xmax": 2748620.3233000003, 30 | "ymax": 1987830.063299999, 31 | "spatialReference": { 32 | "wkid": 102605, 33 | "latestWkid": 102605 34 | } 35 | }, 36 | "pixelSizeX": 0.5, 37 | "pixelSizeY": 0.5, 38 | "bandCount": 4, 39 | "pixelType": "U8", 40 | "minPixelSize": 0, 41 | "maxPixelSize": 0, 42 | "copyrightText": "Lawyers :/", 43 | "serviceDataType": "esriImageServiceDataTypeGeneric", 44 | "minValues": [ 45 | 15, 46 | 28, 47 | 38, 48 | 4 49 | ], 50 | "maxValues": [ 51 | 248, 52 | 248, 53 | 246, 54 | 238 55 | ], 56 | "meanValues": [ 57 | 123.41572786850693, 58 | 127.70524268237573, 59 | 114.37285365465922, 60 | 116.39141229657389 61 | ], 62 | "stdvValues": [ 63 | 43.602364971748756, 64 | 34.16795615629869, 65 | 30.46078238260486, 66 | 27.60464799718197 67 | ], 68 | "objectIdField": "OBJECTID", 69 | "fields": [ 70 | { 71 | "name": "OBJECTID", 72 | "type": "esriFieldTypeOID", 73 | "alias": "OBJECTID", 74 | "domain": null 75 | }, 76 | { 77 | "name": "Shape", 78 | "type": "esriFieldTypeGeometry", 79 | "alias": "Shape", 80 | "domain": null 81 | }, 82 | { 83 | "name": "Name", 84 | "type": "esriFieldTypeString", 85 | "alias": "Name", 86 | "domain": null, 87 | "length": 50 88 | }, 89 | { 90 | "name": "MinPS", 91 | "type": "esriFieldTypeDouble", 92 | "alias": "MinPS", 93 | "domain": null 94 | }, 95 | { 96 | "name": "MaxPS", 97 | "type": "esriFieldTypeDouble", 98 | "alias": "MaxPS", 99 | "domain": null 100 | }, 101 | { 102 | "name": "LowPS", 103 | "type": "esriFieldTypeDouble", 104 | "alias": "LowPS", 105 | "domain": null 106 | }, 107 | { 108 | "name": "HighPS", 109 | "type": "esriFieldTypeDouble", 110 | "alias": "HighPS", 111 | "domain": null 112 | }, 113 | { 114 | "name": "Category", 115 | "type": "esriFieldTypeInteger", 116 | "alias": "Category", 117 | "domain": { 118 | "type": "codedValue", 119 | "name": "MosaicCatalogItemCategoryDomain", 120 | "codedValues": [ 121 | { 122 | "name": "Unknown", 123 | "code": 0 124 | }, 125 | { 126 | "name": "Primary", 127 | "code": 1 128 | }, 129 | { 130 | "name": "Overview", 131 | "code": 2 132 | }, 133 | { 134 | "name": "Unprocessed Overview", 135 | "code": 3 136 | }, 137 | { 138 | "name": "Partial Overview", 139 | "code": 4 140 | }, 141 | { 142 | "name": "Uploaded", 143 | "code": 253 144 | }, 145 | { 146 | "name": "Incomplete", 147 | "code": 254 148 | }, 149 | { 150 | "name": "Custom", 151 | "code": 255 152 | } 153 | ] 154 | } 155 | }, 156 | { 157 | "name": "Tag", 158 | "type": "esriFieldTypeString", 159 | "alias": "Tag", 160 | "domain": null, 161 | "length": 20 162 | }, 163 | { 164 | "name": "GroupName", 165 | "type": "esriFieldTypeString", 166 | "alias": "GroupName", 167 | "domain": null, 168 | "length": 50 169 | }, 170 | { 171 | "name": "ProductName", 172 | "type": "esriFieldTypeString", 173 | "alias": "ProductName", 174 | "domain": null, 175 | "length": 50 176 | }, 177 | { 178 | "name": "CenterX", 179 | "type": "esriFieldTypeDouble", 180 | "alias": "CenterX", 181 | "domain": null 182 | }, 183 | { 184 | "name": "CenterY", 185 | "type": "esriFieldTypeDouble", 186 | "alias": "CenterY", 187 | "domain": null 188 | }, 189 | { 190 | "name": "ZOrder", 191 | "type": "esriFieldTypeInteger", 192 | "alias": "ZOrder", 193 | "domain": null 194 | }, 195 | { 196 | "name": "Shape_Length", 197 | "type": "esriFieldTypeDouble", 198 | "alias": "Shape_Length", 199 | "domain": null 200 | }, 201 | { 202 | "name": "Shape_Area", 203 | "type": "esriFieldTypeDouble", 204 | "alias": "Shape_Area", 205 | "domain": null 206 | } 207 | ], 208 | "capabilities": "Catalog,Mensuration,Image,Metadata", 209 | "defaultMosaicMethod": "Northwest", 210 | "allowedMosaicMethods": "NorthWest,Center,LockRaster,ByAttribute,Nadir,Viewpoint,Seamline,None", 211 | "sortField": "", 212 | "sortValue": null, 213 | "mosaicOperator": "First", 214 | "defaultCompressionQuality": 75, 215 | "defaultResamplingMethod": "Bilinear", 216 | "maxImageHeight": 20000, 217 | "maxImageWidth": 25000, 218 | "maxRecordCount": 1000, 219 | "maxDownloadImageCount": 10000, 220 | "maxDownloadSizeLimit": 1000000, 221 | "maxMosaicImageCount": 20, 222 | "allowRasterFunction": true, 223 | "rasterFunctionInfos": [ 224 | { 225 | "name": "None", 226 | "description": "A No-Op Function.", 227 | "help": "" 228 | }, 229 | { 230 | "name": "False Color Composite", 231 | "description": "A raster function template to display false color (4,1,2).", 232 | "help": "" 233 | } 234 | ], 235 | "rasterTypeInfos": [ 236 | { 237 | "name": "Raster Dataset", 238 | "description": "Supports all ArcGIS Raster Datasets", 239 | "help": "" 240 | } 241 | ], 242 | "mensurationCapabilities": "Basic", 243 | "hasHistograms": true, 244 | "hasColormap": false, 245 | "hasRasterAttributeTable": false, 246 | "minScale": 0, 247 | "maxScale": 0, 248 | "exportTilesAllowed": false, 249 | "hasMultidimensions": false, 250 | "supportsStatistics": true, 251 | "supportsAdvancedQueries": true, 252 | "editFieldsInfo": null, 253 | "ownershipBasedAccessControlForRasters": null, 254 | "allowComputeTiePoints": false, 255 | "useStandardizedQueries": true, 256 | "spatialReference": { 257 | "wkid": 102605, 258 | "latestWkid": 102605 259 | } 260 | } 261 | -------------------------------------------------------------------------------- /test/fixtures/ImageServer_json_Download.json: -------------------------------------------------------------------------------- 1 | { 2 | "currentVersion": 10.3, 3 | "serviceDescription": "Lots of words", 4 | "name": "image/images", 5 | "description": "Even more words", 6 | "extent": { 7 | "xmin": 2240369.8233000003, 8 | "ymin": 1192725.063299999, 9 | "xmax": 2748620.3233000003, 10 | "ymax": 1987830.063299999, 11 | "spatialReference": { 12 | "wkid": 102605, 13 | "latestWkid": 102605 14 | } 15 | }, 16 | "initialExtent": { 17 | "xmin": 2240369.8233000003, 18 | "ymin": 1192725.063299999, 19 | "xmax": 2748620.3233000003, 20 | "ymax": 1987830.063299999, 21 | "spatialReference": { 22 | "wkid": 102605, 23 | "latestWkid": 102605 24 | } 25 | }, 26 | "fullExtent": { 27 | "xmin": 2240369.8233000003, 28 | "ymin": 1192725.063299999, 29 | "xmax": 2748620.3233000003, 30 | "ymax": 1987830.063299999, 31 | "spatialReference": { 32 | "wkid": 102605, 33 | "latestWkid": 102605 34 | } 35 | }, 36 | "pixelSizeX": 0.5, 37 | "pixelSizeY": 0.5, 38 | "bandCount": 4, 39 | "pixelType": "U8", 40 | "minPixelSize": 0, 41 | "maxPixelSize": 0, 42 | "copyrightText": "Lawyers :/", 43 | "serviceDataType": "esriImageServiceDataTypeGeneric", 44 | "minValues": [ 45 | 15, 46 | 28, 47 | 38, 48 | 4 49 | ], 50 | "maxValues": [ 51 | 248, 52 | 248, 53 | 246, 54 | 238 55 | ], 56 | "meanValues": [ 57 | 123.41572786850693, 58 | 127.70524268237573, 59 | 114.37285365465922, 60 | 116.39141229657389 61 | ], 62 | "stdvValues": [ 63 | 43.602364971748756, 64 | 34.16795615629869, 65 | 30.46078238260486, 66 | 27.60464799718197 67 | ], 68 | "objectIdField": "OBJECTID", 69 | "fields": [ 70 | { 71 | "name": "OBJECTID", 72 | "type": "esriFieldTypeOID", 73 | "alias": "OBJECTID", 74 | "domain": null 75 | }, 76 | { 77 | "name": "Shape", 78 | "type": "esriFieldTypeGeometry", 79 | "alias": "Shape", 80 | "domain": null 81 | }, 82 | { 83 | "name": "Name", 84 | "type": "esriFieldTypeString", 85 | "alias": "Name", 86 | "domain": null, 87 | "length": 50 88 | }, 89 | { 90 | "name": "MinPS", 91 | "type": "esriFieldTypeDouble", 92 | "alias": "MinPS", 93 | "domain": null 94 | }, 95 | { 96 | "name": "MaxPS", 97 | "type": "esriFieldTypeDouble", 98 | "alias": "MaxPS", 99 | "domain": null 100 | }, 101 | { 102 | "name": "LowPS", 103 | "type": "esriFieldTypeDouble", 104 | "alias": "LowPS", 105 | "domain": null 106 | }, 107 | { 108 | "name": "HighPS", 109 | "type": "esriFieldTypeDouble", 110 | "alias": "HighPS", 111 | "domain": null 112 | }, 113 | { 114 | "name": "Category", 115 | "type": "esriFieldTypeInteger", 116 | "alias": "Category", 117 | "domain": { 118 | "type": "codedValue", 119 | "name": "MosaicCatalogItemCategoryDomain", 120 | "codedValues": [ 121 | { 122 | "name": "Unknown", 123 | "code": 0 124 | }, 125 | { 126 | "name": "Primary", 127 | "code": 1 128 | }, 129 | { 130 | "name": "Overview", 131 | "code": 2 132 | }, 133 | { 134 | "name": "Unprocessed Overview", 135 | "code": 3 136 | }, 137 | { 138 | "name": "Partial Overview", 139 | "code": 4 140 | }, 141 | { 142 | "name": "Uploaded", 143 | "code": 253 144 | }, 145 | { 146 | "name": "Incomplete", 147 | "code": 254 148 | }, 149 | { 150 | "name": "Custom", 151 | "code": 255 152 | } 153 | ] 154 | } 155 | }, 156 | { 157 | "name": "Tag", 158 | "type": "esriFieldTypeString", 159 | "alias": "Tag", 160 | "domain": null, 161 | "length": 20 162 | }, 163 | { 164 | "name": "GroupName", 165 | "type": "esriFieldTypeString", 166 | "alias": "GroupName", 167 | "domain": null, 168 | "length": 50 169 | }, 170 | { 171 | "name": "ProductName", 172 | "type": "esriFieldTypeString", 173 | "alias": "ProductName", 174 | "domain": null, 175 | "length": 50 176 | }, 177 | { 178 | "name": "CenterX", 179 | "type": "esriFieldTypeDouble", 180 | "alias": "CenterX", 181 | "domain": null 182 | }, 183 | { 184 | "name": "CenterY", 185 | "type": "esriFieldTypeDouble", 186 | "alias": "CenterY", 187 | "domain": null 188 | }, 189 | { 190 | "name": "ZOrder", 191 | "type": "esriFieldTypeInteger", 192 | "alias": "ZOrder", 193 | "domain": null 194 | }, 195 | { 196 | "name": "Shape_Length", 197 | "type": "esriFieldTypeDouble", 198 | "alias": "Shape_Length", 199 | "domain": null 200 | }, 201 | { 202 | "name": "Shape_Area", 203 | "type": "esriFieldTypeDouble", 204 | "alias": "Shape_Area", 205 | "domain": null 206 | } 207 | ], 208 | "capabilities": "Catalog,Mensuration,Download,Image,Metadata", 209 | "defaultMosaicMethod": "Northwest", 210 | "allowedMosaicMethods": "NorthWest,Center,LockRaster,ByAttribute,Nadir,Viewpoint,Seamline,None", 211 | "sortField": "", 212 | "sortValue": null, 213 | "mosaicOperator": "First", 214 | "defaultCompressionQuality": 75, 215 | "defaultResamplingMethod": "Bilinear", 216 | "maxImageHeight": 20000, 217 | "maxImageWidth": 25000, 218 | "maxRecordCount": 1000, 219 | "maxDownloadImageCount": 10000, 220 | "maxDownloadSizeLimit": 1000000, 221 | "maxMosaicImageCount": 20, 222 | "allowRasterFunction": true, 223 | "rasterFunctionInfos": [ 224 | { 225 | "name": "None", 226 | "description": "A No-Op Function.", 227 | "help": "" 228 | }, 229 | { 230 | "name": "False Color Composite", 231 | "description": "A raster function template to display false color (4,1,2).", 232 | "help": "" 233 | } 234 | ], 235 | "rasterTypeInfos": [ 236 | { 237 | "name": "Raster Dataset", 238 | "description": "Supports all ArcGIS Raster Datasets", 239 | "help": "" 240 | } 241 | ], 242 | "mensurationCapabilities": "Basic", 243 | "hasHistograms": true, 244 | "hasColormap": false, 245 | "hasRasterAttributeTable": false, 246 | "minScale": 0, 247 | "maxScale": 0, 248 | "exportTilesAllowed": false, 249 | "hasMultidimensions": false, 250 | "supportsStatistics": true, 251 | "supportsAdvancedQueries": true, 252 | "editFieldsInfo": null, 253 | "ownershipBasedAccessControlForRasters": null, 254 | "allowComputeTiePoints": false, 255 | "useStandardizedQueries": true, 256 | "spatialReference": { 257 | "wkid": 102605, 258 | "latestWkid": 102605 259 | } 260 | } 261 | -------------------------------------------------------------------------------- /lib/geometry.ts: -------------------------------------------------------------------------------- 1 | import EventEmitter from 'node:events'; 2 | import Err from '@openaddresses/batch-error'; 3 | import rings2geojson from './rings2geojson.js'; 4 | import Fetch from './fetch.js'; 5 | import { Feature, GeoJsonProperties } from 'geojson'; 6 | import Schema from './schema.js' 7 | import { JSONSchema6, JSONSchema6Definition } from 'json-schema'; 8 | import { 9 | EsriDumpConfig, 10 | EsriDumpConfigApproach 11 | } from '../index.js'; 12 | 13 | interface Field { 14 | name: string; 15 | type: string; 16 | alias?: string; 17 | domain?: unknown; 18 | } 19 | 20 | interface Path { 21 | xmin: number; 22 | ymin: number; 23 | xmax: number; 24 | ymax: number; 25 | } 26 | 27 | export default class Geometry extends EventEmitter { 28 | baseUrl: URL; 29 | geomType: string; 30 | maxRecords: null | number; 31 | set: Set; 32 | oidField: string; 33 | paths: Path[]; 34 | schema: JSONSchema6; 35 | 36 | constructor( 37 | url: URL, 38 | metadata: any 39 | ) { 40 | super(); 41 | 42 | this.baseUrl = url; 43 | this.paths = [metadata.extent as Path]; 44 | 45 | this.geomType = metadata.geometryType; 46 | this.maxRecords = metadata.maxRecordCount || null; 47 | this.set = new Set(); 48 | this.oidField = Geometry.findOidField(metadata.fields); 49 | this.schema = Schema(metadata); 50 | } 51 | 52 | async fetch(config: EsriDumpConfig) { 53 | try { 54 | if ( 55 | config.approach === EsriDumpConfigApproach.BBOX 56 | || config.approach === EsriDumpConfigApproach.TOP_FEATURES_BBOX 57 | ) { 58 | await this.fetch_bbox(config); 59 | } else if (config.approach === EsriDumpConfigApproach.ITER 60 | || config.approach === EsriDumpConfigApproach.TOP_FEATURES_ITER 61 | ) { 62 | await this.fetch_iter(config); 63 | } else { 64 | throw new Err(400, null, 'Unknown Approach'); 65 | } 66 | } catch (err) { 67 | this.emit('error', err); 68 | } 69 | } 70 | 71 | async fetch_iter(config: EsriDumpConfig) { 72 | if (!this.oidField) this.emit('error', new Err(400, null, 'Cannot use iter function as oidField could not be determined')); 73 | 74 | const queryFragment = config.approach === EsriDumpConfigApproach.TOP_FEATURES_ITER ? '/queryTopFeatures' : '/query'; 75 | 76 | const url = new URL(String(this.baseUrl) + queryFragment); 77 | url.searchParams.append('returnCountOnly', 'true'); 78 | if (!config.params.where) url.searchParams.append('where', '1=1'); 79 | 80 | if (process.env.DEBUG) console.error(String(url)); 81 | const res = await Fetch(config, url); 82 | 83 | if (!res.ok) return this.emit('error', await res.text()); 84 | 85 | const meta = await res.json(); 86 | if (isNaN(meta.count)) { 87 | return this.emit(`error', 'Unable to determine feature count - ${JSON.stringify(meta)}`); 88 | } 89 | 90 | const count = meta.count; 91 | let curr = 0; 92 | 93 | while (curr < count) { 94 | let attempts = 0; 95 | 96 | const url = new URL(String(this.baseUrl) + queryFragment); 97 | if (!config.params.where) url.searchParams.append('where', '1=1'); 98 | url.searchParams.append('geometryPrecision', '7'); 99 | url.searchParams.append('returnGeometry', 'true'); 100 | url.searchParams.append('outSR', '4326'); 101 | url.searchParams.append('outFields', '*'); 102 | url.searchParams.append('resultOffset', String(curr)); 103 | 104 | let data = null; 105 | while (attempts <= 5) { 106 | attempts++; 107 | 108 | if (process.env.DEBUG) console.error(String(url)); 109 | const res = await Fetch(config, url); 110 | 111 | if (!res.ok) return this.emit('error', await res.text()); 112 | 113 | data = await res.json(); 114 | 115 | if (data && data.error) continue; 116 | 117 | if (data && data.features) { 118 | curr += data.features.length; 119 | 120 | for (const feature of data.features) { 121 | if (!this.set.has(feature.attributes[this.oidField])) { 122 | this.set.add(feature.attributes[this.oidField]); 123 | 124 | try { 125 | const feat = this.toGeoJSON(feature); 126 | this.emit('feature', feat) 127 | } catch (err) { 128 | // This usually errors if it's an attribute only feature 129 | if (process.env.DEBUG) console.error('Invalid Feature', feature, err instanceof Error ? err.message : err); 130 | } 131 | } 132 | } 133 | 134 | break; 135 | } else if (!data) { 136 | return this.emit('error', new Err(400, null, 'Data from' + url + ' undefined')); 137 | } else { 138 | return this.emit('error', new Err(400, null, 'Error with ' + url)); 139 | } 140 | } 141 | 142 | if (attempts > 5) return this.emit('error', 'Query of ' + url + ' unsuccessful: ' + data.error.details); 143 | } 144 | 145 | this.emit('done'); 146 | } 147 | 148 | async fetch_bbox(config: EsriDumpConfig) { 149 | const queryFragment = config.approach === EsriDumpConfigApproach.TOP_FEATURES_BBOX ? '/queryTopFeatures' : '/query'; 150 | 151 | while (this.paths.length) { 152 | const bounds = this.paths.pop(); 153 | 154 | const url = new URL(String(this.baseUrl) + queryFragment); 155 | url.searchParams.append('geometry', [bounds.xmin, bounds.ymin, bounds.xmax, bounds.ymax].join(',')); 156 | url.searchParams.append('geometryType', 'esriGeometryEnvelope'); 157 | url.searchParams.append('spatialRel', 'esriSpatialRelIntersects'); 158 | url.searchParams.append('geometryPrecision', '7'); 159 | url.searchParams.append('returnGeometry', 'true'); 160 | url.searchParams.append('outSR', '4326'); 161 | url.searchParams.append('outFields', '*'); 162 | 163 | let attempts = 0; 164 | 165 | let data = null; 166 | while (attempts <= 5) { 167 | attempts++; 168 | 169 | if (process.env.DEBUG) console.error(String(url)); 170 | const res = await Fetch(config, url); 171 | 172 | if (!res.ok) return this.emit('error', await res.text()); 173 | 174 | data = await res.json(); 175 | 176 | if (data && data.error) continue; 177 | 178 | if (data && data.features) { 179 | if (this.maxRecords === null) { 180 | // Since we can't reliably get the configured maximum result size from the server, 181 | // assume that the first request will exceed it and use the results length 182 | // to set the maxRecords value for further requests. 183 | this.maxRecords = data.features.length; 184 | } 185 | 186 | if (data.exceededTransferLimit || data.features.length === this.maxRecords) { 187 | // If we get back the maximum number of results, break the 188 | // bbox up into 4 smaller chunks and request those. 189 | Geometry.splitBbox(bounds).forEach((subbox) => { this.paths.push(subbox); }); 190 | } else { 191 | for (const feature of data.features) { 192 | if (!this.set.has(feature.attributes[this.oidField])) { 193 | this.set.add(feature.attributes[this.oidField]); 194 | try { 195 | const feat = this.toGeoJSON(feature); 196 | this.emit('feature', feat) 197 | } catch (err) { 198 | // This usually errors if it's an attribute only feature 199 | if (process.env.DEBUG) console.error('Invalid Feature', feature, err instanceof Error ? err.message : err); 200 | } 201 | } 202 | } 203 | } 204 | 205 | break; 206 | } else if (!data) { 207 | return this.emit('error', new Err(400, null, 'Data from' + url + ' undefined')); 208 | } else { 209 | return this.emit('error', new Err(400, null, 'Error with ' + url)); 210 | } 211 | } 212 | 213 | if (attempts > 5) return this.emit('error', 'Query of ' + url + ' unsuccessful: ' + data.error.details); 214 | } 215 | 216 | this.emit('done'); 217 | } 218 | 219 | toGeoJSON(esrifeature: any): Feature { 220 | const id = esrifeature.attributes[this.oidField]; 221 | const type = 'Feature'; 222 | const properties: GeoJsonProperties = {} 223 | for (const prop in esrifeature.attributes) { 224 | const schema: JSONSchema6Definition = this.schema.properties[prop]; 225 | 226 | if ( 227 | typeof schema !== 'boolean' 228 | && schema.format === 'date-time' 229 | && esrifeature.attributes[prop] 230 | ) { 231 | properties[prop] = new Date(esrifeature.attributes[prop]).toISOString(); 232 | } else { 233 | properties[prop] = esrifeature.attributes[prop]; 234 | } 235 | } 236 | 237 | if (this.geomType === 'esriGeometryPolygon') { 238 | return { 239 | id, type, properties, 240 | geometry: rings2geojson(esrifeature.geometry.rings) 241 | }; 242 | } else if (this.geomType === 'esriGeometryPolyline') { 243 | return { 244 | id, type, properties, 245 | geometry: { 246 | type: 'MultiLineString', 247 | coordinates: esrifeature.geometry.paths 248 | } 249 | }; 250 | } else if (this.geomType === 'esriGeometryPoint') { 251 | return { 252 | id, type, properties, 253 | geometry: { 254 | type: 'Point', 255 | coordinates: [esrifeature.geometry.x, esrifeature.geometry.y] 256 | } 257 | }; 258 | } 259 | } 260 | 261 | static splitBbox(bbox: Path): Path[] { 262 | const halfWidth = (bbox.xmax - bbox.xmin) / 2.0; 263 | const halfHeight = (bbox.ymax - bbox.ymin) / 2.0; 264 | 265 | return [ 266 | { xmin: bbox.xmin, ymin: bbox.ymin, ymax: bbox.ymin + halfHeight, xmax: bbox.xmin + halfWidth }, 267 | { xmin: bbox.xmin + halfWidth, ymin: bbox.ymin, ymax: bbox.ymin + halfHeight, xmax: bbox.xmax }, 268 | { xmin: bbox.xmin, ymin: bbox.ymin + halfHeight, xmax: bbox.xmin + halfWidth, ymax: bbox.ymax }, 269 | { xmin: bbox.xmin + halfWidth, ymin: bbox.ymin + halfHeight, xmax: bbox.xmax, ymax: bbox.ymax } 270 | ]; 271 | } 272 | 273 | static findOidField(fields: Field[]): string { 274 | const oidField = fields.filter((field) => { 275 | return (field.type === 'esriFieldTypeOID'); 276 | })[0]; 277 | 278 | if (oidField) { 279 | return oidField.name; 280 | } else { 281 | const possibleIds = ['OBJECTID', 'objectid', 'FID', 'ID', 'fid', 'id']; 282 | const nextBestOidField = fields.filter((field) => { 283 | return (possibleIds.indexOf(field.name) > -1); 284 | }).sort((a: Field, b: Field) => { 285 | return possibleIds.indexOf(a.name) - possibleIds.indexOf(b.name); 286 | })[0]; 287 | if (nextBestOidField) { 288 | return nextBestOidField.name; 289 | } else { 290 | throw new Err(400, null, 'Could not determine OBJECTID field.'); 291 | } 292 | } 293 | } 294 | } 295 | 296 | --------------------------------------------------------------------------------