├── .DS_Store ├── .gitignore ├── README.md ├── dist └── buildings.geojson ├── index.js ├── matchClasses.js ├── package.json ├── src.json ├── test.js ├── tilesToFeatures.js └── yarn.lock /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShitMonkey/amap-building-crawler/28a1ccb43b42e83d196fb93358163237208037f9/.DS_Store -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *node_modules/* 2 | *dist/* -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # amap-building-crawler 2 | 3 | A crawler project for fetching 3d building data from amap and tramsform its to GeoJSON.
4 | 高德地图3D建筑信息爬虫项目，用于爬取高德地图的 3D 建筑物数据，并将其转存为 GeoJSON 数据格式 5 | 6 | 7 | ## Doc 8 | 9 | #### 1. Clone remote codes to local, then use the `yarn` command to install the project in local directory. 10 | #### 2. Execute `mkdir dist` in the project root directory (do not need to do this if the directory already exists). 11 | #### 3. Rewrite the latitude and longitude range in `index.js` file where you want to crawl the data (upper left corner & bottom right corner). 12 | ``` 13 | const lnglatRange = [ 14 | [118.01307678222655, 24.596143627409358], 15 | [118.15830230712889, 24.452462684995407], 16 | ]; 17 | ``` 18 | #### 4. Execute `yarn start` in the project root directory, the retrieved data will be stored in the `dist` directory as a `.geojson` file. 19 | 20 | 21 | ## 说明 22 | 23 | 24 | #### 1. clone 代码到本地，根目录使用 `yarn` 命令安装项目 25 | #### 2. 项目根目录下执行 `mkdir dist`（若目录已存在则不需要） 26 | #### 3. 修改 `index.js` 文件中需要爬取的坐标经纬度范围： 27 | ``` 28 | // 需要爬取的经纬度范围（左上角、右下角） 29 | const lnglatRange = [ 30 | [118.01307678222655, 24.596143627409358], 31 | [118.15830230712889, 24.452462684995407], 32 | ]; 33 | ``` 34 | #### 4. 项目根目录下执行 `yarn start`，爬取的 `.geojson` 数据将存储到 `dist` 目录下 35 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const path = require('path'); 3 | const Axios = require('axios'); 4 | const chunk = require('lodash').chunk; 5 | const TileLnglatTransform = require('tile-lnglat-transform'); 6 | const tilesToFeatures = require('./tilesToFeatures'); 7 | 8 | // 需要爬取的经纬度范围（左上角、右下角） 9 | const lnglatRange = [ 10 | [118.01307678222655, 24.596143627409358], 11 | [118.15830230712889, 24.452462684995407], 12 | ]; 13 | 14 | // 单个请求的瓦片数量 15 | const tilesCountPerReq = 100; 16 | 17 | const axios = Axios.create({ 18 | baseURL: 'http://vector.amap.com/vector/', 19 | timeout: 1e4, 20 | headers: { 21 | Accept: '*/*', 22 | 'Accept-Encoding': 'gzip, deflate', 23 | 'Accept-Language': 'zh-CN,zh;q=0.8,en;q=0.6', 24 | 'Cache-Control': 'no-cache', 25 | Connection: 'keep-alive', 26 | Cookie: 27 | 'guid=79aa-8b3a-7e1d-21b3; UM_distinctid=15d73c745f317b-08246e6485b3bc-30667808-1fa400-15d73c745f4104; Kdw4_5279_saltkey=iQi1nA59; Kdw4_5279_lastvisit=1500882510; Kdw4_5279_visitedfid=59; Kdw4_5279_st_p=0%7C1500887117%7C17f2b67a0a07513e13a8bc49e0b84de5; Kdw4_5279_viewid=tid_26343; cna=m/FxEUQMe1ICAXGMGAQdRR4k; isg=Ajg4VzV1leVAFPlqlMwe-GBYCeDAqWw7Zfu2N3KrynMmjdF3GLUluqXtM7PG; key=8325164e247e15eea68b59e89200988b', 28 | Host: 'vector.amap.com', 29 | Pragma: 'no-cache', 30 | Referer: 'http://lbs.amap.com/console/show/tools', 31 | 'User-Agent': 32 | 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36', 33 | }, 34 | }); 35 | 36 | const lnglatToTilesRange = lnglat => 37 | TileLnglatTransform.TileLnglatTransformGaode.lnglatToTile( 38 | lnglat[0], 39 | lnglat[1], 40 | 17 41 | ); 42 | 43 | const tilesRange = { 44 | minX: lnglatToTilesRange(lnglatRange[0]).tileX, 45 | minY: lnglatToTilesRange(lnglatRange[0]).tileY, 46 | maxX: lnglatToTilesRange(lnglatRange[1]).tileX, 47 | maxY: lnglatToTilesRange(lnglatRange[1]).tileY, 48 | }; 49 | 50 | console.log(tilesRange); 51 | 52 | const tileList = []; 53 | const area = { 54 | x: tilesRange.minX, 55 | y: tilesRange.minY, 56 | }; 57 | while (area.x <= tilesRange.maxX && area.y <= tilesRange.maxY) { 58 | area.x += 1; 59 | tileList.push([area.x, area.y]); 60 | if (area.x === tilesRange.maxX) { 61 | area.x = tilesRange.minX; 62 | area.y += 1; 63 | } 64 | } 65 | const reqQueue = chunk(tileList, tilesCountPerReq).map(chunk => { 66 | const tileStr = chunk.map(tile => `${tile[0]},${tile[1]};`).join(''); 67 | const url = `buildings?tiles=${tileStr}&level=17`; 68 | return url; 69 | }); 70 | 71 | const writeGeoJsonFile = features => { 72 | const geojson = { 73 | type: 'FeatureCollection', 74 | features, 75 | }; 76 | fs.writeFile( 77 | path.join(__dirname, './dist/buildings.geojson'), 78 | JSON.stringify(geojson), 79 | 'utf-8', 80 | err => { 81 | if (err) { 82 | throw err; 83 | } 84 | console.log('geojson生成完毕！'); 85 | } 86 | ); 87 | }; 88 | 89 | let fullFeatures = []; 90 | const total = reqQueue.length; 91 | 92 | const execQueue = () => { 93 | const curUrl = reqQueue.shift(); 94 | 95 | axios 96 | .get(curUrl) 97 | .then(res => { 98 | const json = JSON.parse( 99 | JSON.stringify(res.data).replace(/(jsonp[^\(]+\(|\);)/g, '') 100 | ); 101 | fullFeatures = fullFeatures.concat(tilesToFeatures(json)); 102 | 103 | console.log(`爬取瓦片源：${total - reqQueue.length}/${total}`); 104 | 105 | if (reqQueue.length > 0) { 106 | execQueue(); 107 | } else { 108 | writeGeoJsonFile(fullFeatures); 109 | } 110 | }) 111 | .catch(err => { 112 | throw err; 113 | }); 114 | }; 115 | 116 | execQueue(); 117 | -------------------------------------------------------------------------------- /matchClasses.js: -------------------------------------------------------------------------------- 1 | const pattOfClasses = [ 2 | { 3 | classes: 'schools', 4 | patt: /(小学|中学|大学|学校|学院|教育|职业|培训)/gi, 5 | }, 6 | { 7 | classes: 'companies', 8 | patt: /(公司|集团)/gi, 9 | }, 10 | { 11 | classes: 'hotels', 12 | patt: /(酒店|饭店)/gi, 13 | }, 14 | { 15 | classes: 'govs', 16 | patt: /(局|委|政府|检察院|法院)/gi, 17 | }, 18 | { 19 | classes: 'cybercafes', 20 | patt: /(网吧|网咖)/gi, 21 | }, 22 | ]; 23 | 24 | const matchClasses = properties => { 25 | const newProp = Object.assign({}, properties); 26 | for (const type of pattOfClasses) { 27 | if (type.patt.test(properties.name)) { 28 | newProp.description = properties.name; 29 | newProp.classes = type.classes; 30 | break; 31 | } 32 | } 33 | return newProp; 34 | }; 35 | 36 | module.exports = matchClasses; 37 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "amap-building-crawler", 3 | "version": "1.0.0", 4 | "description": "A crawler project for fetching 3d building data from amap and tramsform its to GeoJSON", 5 | "main": "index.js", 6 | "license": "MIT", 7 | "dependencies": { 8 | "axios": "^0.16.2", 9 | "coordtransform": "^2.1.2", 10 | "lodash": "^4.17.4", 11 | "tile-lnglat-transform": "^1.3.1" 12 | }, 13 | "scripts": { 14 | "start": "node index.js", 15 | "convert": "node test.js" 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /test.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const path = require('path'); 3 | const tilesToFeatures = require('./tilesToFeatures'); 4 | 5 | const srcJson = fs.readFileSync(path.join(__dirname, './src.json')); 6 | 7 | const writeGeoJsonFile = () => { 8 | const geojson = { 9 | type: 'FeatureCollection', 10 | features: tilesToFeatures(JSON.parse(srcJson)), 11 | }; 12 | fs.writeFile(path.join(__dirname, './dist/buildings.geojson'), JSON.stringify(geojson), 'utf-8', (err) => { 13 | if (err) { 14 | throw err; 15 | } 16 | console.log('geojson生成完毕！'); 17 | }); 18 | }; 19 | 20 | writeGeoJsonFile(); 21 | -------------------------------------------------------------------------------- /tilesToFeatures.js: -------------------------------------------------------------------------------- 1 | const TileLnglatTransform = require('tile-lnglat-transform'); 2 | const coordtransform = require('coordtransform'); 3 | const chunk = require('lodash').chunk; 4 | const matchClasses = require('./matchClasses'); 5 | const hashString = str => { 6 | const hash = require('crypto').createHash('md5'); 7 | hash.update(str); 8 | return hash.digest('hex'); 9 | }; 10 | 11 | const pixelToLnglat = (pixels, index) => { 12 | const points = chunk(pixels, 2); 13 | const lnglatPoints = points.map(point => { 14 | // 像素转坐标值 15 | const gaodeTransfer = TileLnglatTransform.TileLnglatTransformGaode; 16 | const location = gaodeTransfer.pixelToLnglat(point[0], point[1], index.x, index.y, index.z); 17 | // 国测局坐标转wgs84坐标 18 | return coordtransform.gcj02towgs84(location.lng, location.lat); 19 | }); 20 | return lnglatPoints; 21 | }; 22 | 23 | const tilesToFeatures = tilesJson => { 24 | const features = []; 25 | 26 | if (tilesJson.list) { 27 | tilesJson.list.forEach((tileData, i) => { 28 | tileData.tile.forEach((region, j) => { 29 | const feature = { 30 | type: 'Feature', 31 | properties: { 32 | level: 1, 33 | name: region.name ? region.name : hashString(`tile_${i}_region_${j}`), 34 | height: parseInt(region.floor, 10) * 10, 35 | base_height: 0, 36 | color: '#ddd', 37 | }, 38 | geometry: { 39 | type: 'Polygon', 40 | coordinates: [ 41 | pixelToLnglat(region.coords, tileData.index), 42 | ], 43 | }, 44 | }; 45 | feature.properties = matchClasses(feature.properties); 46 | features.push(feature); 47 | }); 48 | }); 49 | } 50 | 51 | return features; 52 | }; 53 | 54 | module.exports = tilesToFeatures; 55 | -------------------------------------------------------------------------------- /yarn.lock: -------------------------------------------------------------------------------- 1 | # THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. 2 | # yarn lockfile v1 3 | 4 | 5 | axios@^0.16.2: 6 | version "0.16.2" 7 | resolved "http://registry.npm.taobao.org/axios/download/axios-0.16.2.tgz#ba4f92f17167dfbab40983785454b9ac149c3c6d" 8 | dependencies: 9 | follow-redirects "^1.2.3" 10 | is-buffer "^1.1.5" 11 | 12 | coordtransform@^2.1.2: 13 | version "2.1.2" 14 | resolved "http://registry.npm.taobao.org/coordtransform/download/coordtransform-2.1.2.tgz#090349cd62b1dc05fae47b291bf9f0b7ce827f66" 15 | 16 | debug@^2.4.5: 17 | version "2.6.8" 18 | resolved "http://registry.npm.taobao.org/debug/download/debug-2.6.8.tgz#e731531ca2ede27d188222427da17821d68ff4fc" 19 | dependencies: 20 | ms "2.0.0" 21 | 22 | follow-redirects@^1.2.3: 23 | version "1.2.4" 24 | resolved "http://registry.npm.taobao.org/follow-redirects/download/follow-redirects-1.2.4.tgz#355e8f4d16876b43f577b0d5ce2668b9723214ea" 25 | dependencies: 26 | debug "^2.4.5" 27 | 28 | is-buffer@^1.1.5: 29 | version "1.1.5" 30 | resolved "http://registry.npm.taobao.org/is-buffer/download/is-buffer-1.1.5.tgz#1f3b26ef613b214b88cbca23cc6c01d87961eecc" 31 | 32 | lodash@^4.17.4: 33 | version "4.17.4" 34 | resolved "http://registry.npm.taobao.org/lodash/download/lodash-4.17.4.tgz#78203a4d1c328ae1d86dca6460e369b57f4055ae" 35 | 36 | ms@2.0.0: 37 | version "2.0.0" 38 | resolved "http://registry.npm.taobao.org/ms/download/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" 39 | 40 | tile-lnglat-transform@^1.3.1: 41 | version "1.3.1" 42 | resolved "http://registry.npm.taobao.org/tile-lnglat-transform/download/tile-lnglat-transform-1.3.1.tgz#1136e8be8452ca163b7ee7266d1bad2c94cf78fb" 43 | --------------------------------------------------------------------------------