├── .gitignore ├── test ├── fixtures │ ├── example.dbf │ └── example.shp ├── run.js ├── interface.js └── stream.js ├── lib ├── stringify.js └── ShapeFileStream.js ├── .travis.yml ├── index.js ├── example └── file.js ├── package.json └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | npm-debug.log 3 | .DS_Store 4 | *.zip 5 | data 6 | -------------------------------------------------------------------------------- /test/fixtures/example.dbf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geopipes/shapefile-stream/HEAD/test/fixtures/example.dbf -------------------------------------------------------------------------------- /test/fixtures/example.shp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geopipes/shapefile-stream/HEAD/test/fixtures/example.shp -------------------------------------------------------------------------------- /test/run.js: -------------------------------------------------------------------------------- 1 | 2 | var tape = require('tape'); 3 | 4 | var common = {}; 5 | 6 | var tests = [ 7 | require('./interface.js'), 8 | require('./stream.js') 9 | ]; 10 | 11 | tests.map(function(t) { 12 | t.all(tape, common) 13 | }); -------------------------------------------------------------------------------- /lib/stringify.js: -------------------------------------------------------------------------------- 1 | 2 | var through = require('through2'); 3 | 4 | // convenience function for converting object streams back to strings 5 | var stringify = through.obj( function( data, enc, next ){ 6 | this.push( JSON.stringify( data, null, 2 ), 'utf-8' ); 7 | next(); 8 | }); 9 | 10 | module.exports = stringify; -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - 0.12 4 | - 4 5 | - 5 6 | - 6 7 | install: 8 | - npm install 9 | before_script: 10 | - npm install npm 11 | script: 12 | - npm test 13 | sudo: false 14 | cache: 15 | directories: 16 | - node_modules 17 | matrix: 18 | fast_finish: true 19 | allow_failures: 20 | - node_js: 6 21 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | 2 | var ShapeFileStream = require('./lib/ShapeFileStream'), 3 | stringify = require('./lib/stringify'); 4 | 5 | function createReadStream( filename, shapeFileOptions ){ 6 | return new ShapeFileStream( filename, shapeFileOptions ); 7 | } 8 | 9 | module.exports = { 10 | createReadStream: createReadStream, 11 | stringify: stringify 12 | } -------------------------------------------------------------------------------- /example/file.js: -------------------------------------------------------------------------------- 1 | 2 | var shapefile = require('../'), 3 | path = require('path'), 4 | filename = path.resolve( __dirname + '/../test/fixtures/example.shp' ); 5 | 6 | shapefile.createReadStream( filename ) 7 | // .pipe( require('through2').obj( function( data, enc, next ){ 8 | // console.log( 9 | // data.type, 10 | // data.properties.qs_adm0, 11 | // data.properties.qs_a1, 12 | // data.geometry.type, 13 | // data.geometry.coordinates[0].length 14 | // ); 15 | // next(); 16 | // })); 17 | .pipe( shapefile.stringify ) 18 | .pipe( process.stdout ); -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "shapefile-stream", 3 | "author": "mapzen", 4 | "version": "0.0.4", 5 | "description": "Streaming shapefile parser", 6 | "homepage": "https://github.com/geopipes/shapefile-stream", 7 | "license": "MIT", 8 | "main": "index.js", 9 | "scripts": { 10 | "test": "node test/run.js | tap-spec" 11 | }, 12 | "repository": { 13 | "type": "git", 14 | "url": "git://github.com/geopipes/shapefile-stream.git" 15 | }, 16 | "keywords": [ 17 | "shapefile", 18 | "stream", 19 | "parser" 20 | ], 21 | "bugs": { 22 | "url": "https://github.com/geopipes/shapefile-stream/issues" 23 | }, 24 | "engines": { 25 | "node": ">=0.10.26", 26 | "npm": ">=1.4.3" 27 | }, 28 | "dependencies": { 29 | "shapefile": "^0.3.0", 30 | "readable-stream": "~1.0.17", 31 | "through2": "^0.5.1" 32 | }, 33 | "devDependencies": { 34 | "tape": "^2.13.4", 35 | "tap-spec": "^0.2.0" 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /test/interface.js: -------------------------------------------------------------------------------- 1 | 2 | var shapefile = require('../'); 3 | 4 | module.exports.interface = {}; 5 | 6 | module.exports.interface.createReadStream = function(test, common) { 7 | test('createReadStream()', function(t) { 8 | t.equal(typeof shapefile.createReadStream, 'function', 'valid function'); 9 | t.equal(shapefile.createReadStream.length, 2, 'consistent arguments length'); 10 | t.end(); 11 | }); 12 | } 13 | 14 | module.exports.interface.stringify = function(test, common) { 15 | test('stringify', function(t) { 16 | t.equal(typeof shapefile.stringify, 'object', 'valid function'); 17 | t.equal(typeof shapefile.stringify._read, 'function', 'readable stream'); 18 | t.equal(typeof shapefile.stringify._write, 'function', 'writable stream'); 19 | t.end(); 20 | }); 21 | } 22 | 23 | module.exports.all = function (tape, common) { 24 | 25 | function test(name, testFunction) { 26 | return tape('external interface: ' + name, testFunction) 27 | } 28 | 29 | for( var testCase in module.exports.interface ){ 30 | module.exports.interface[testCase](test, common); 31 | } 32 | } -------------------------------------------------------------------------------- /lib/ShapeFileStream.js: -------------------------------------------------------------------------------- 1 | 2 | var util = require('util'), 3 | shapefile = require('shapefile'), 4 | Readable = require('readable-stream'); 5 | 6 | function ShapeFileStream( filename, shapeFileOptions ){ 7 | Readable.call( this, { objectMode: true } ); 8 | if( !shapeFileOptions ){ shapeFileOptions = { encoding: 'UTF-8' }; } 9 | this.reader = shapefile.reader( filename, shapeFileOptions ); 10 | this.reader.readHeader( function( err, header ){ 11 | if( err ){ 12 | console.error( 'ShapeFileStream', err ); 13 | return this.emit( 'error', err ); 14 | } 15 | this.ready = true; 16 | }.bind(this)); 17 | } 18 | 19 | util.inherits( ShapeFileStream, Readable ); 20 | 21 | // Headers must be read before we can read from 22 | // the rest of the file. 23 | ShapeFileStream.prototype._onready = function( cb ){ 24 | if( this.ready ){ return cb(); } 25 | setTimeout( this._onready.bind( this, cb ), 100 ); 26 | } 27 | 28 | ShapeFileStream.prototype.next = function(){ 29 | this.reader.readRecord( function( err, record ){ 30 | if( err ){ 31 | console.error( 'ShapeFileStream', err ); 32 | } 33 | if( record === shapefile.end ){ 34 | this.push( null ); // eof 35 | } 36 | else { 37 | // This error should never trigger 38 | if( this.eof ){ 39 | console.log( 'ShapeFileStream got record after EOF' ); 40 | console.log( record ); 41 | } 42 | var pause = !!this.push( record ); 43 | if( !pause ) this.next(); 44 | } 45 | }.bind(this) ); 46 | } 47 | 48 | ShapeFileStream.prototype._read = function(){ 49 | this._onready( function(){ 50 | this.next(); 51 | }.bind(this) ); 52 | } 53 | 54 | module.exports = ShapeFileStream; -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Installation 2 | 3 | ```bash 4 | $ npm install shapefile-stream 5 | ``` 6 | 7 | [![NPM](https://nodei.co/npm/shapefile-stream.png?downloads=true&stars=true)](https://nodei.co/npm/shapefile-stream) 8 | 9 | Note: you will need `node` and `npm` installed first. 10 | 11 | The easiest way to install `node.js` is with [nave.sh](https://github.com/isaacs/nave) by executing `[sudo] ./nave.sh usemain stable` 12 | 13 | ## Usage 14 | 15 | You can extract the shapefile data from a file stream: 16 | 17 | ```javascript 18 | var shapefile = require('shapefile-stream'); 19 | 20 | // both the .shp and the .dbf files are required 21 | shapefile.createReadStream( 'example.shp' ) 22 | .pipe( shapefile.stringify ) 23 | .pipe( process.stdout ); 24 | ``` 25 | 26 | ## Roll your own 27 | 28 | The easiest way to get started writing your own pipes is to use `through2`; just make sure you call `next()`. 29 | 30 | ```javascript 31 | var shapefile = require('shapefile-stream'), 32 | through = require('through2'); 33 | 34 | shapefile.createReadStream( 'example.shp' ) 35 | .pipe( through.obj( function( data, enc, next ){ 36 | console.log( 37 | data.type, 38 | data.properties.qs_adm0, 39 | data.properties.qs_a1, 40 | data.geometry.type, 41 | data.geometry.coordinates[0].length 42 | ); 43 | next(); 44 | })); 45 | ``` 46 | 47 | ```bash 48 | Feature France MARNE Polygon 184 49 | Feature France MEURTHE-ET-MOSELLE MultiPolygon 1 50 | Feature France NIEVRE Polygon 162 51 | Feature France NORD Polygon 167 52 | ``` 53 | 54 | ## Schema 55 | 56 | Each shapefile is different, but the example file outputs objects which look like this: 57 | 58 | ```javascript 59 | { 60 | "type": "Feature", 61 | "properties": { 62 | "qs_adm0_a3": "FRA", 63 | "qs_adm0": "France", 64 | "qs_level": "adm2_region", 65 | "qs_iso_cc": "FR", 66 | "qs_a0": "France", 67 | "qs_a0_lc": null, 68 | "qs_a1r": "ALSACE", 69 | "qs_a1r_alt": null, 70 | "qs_a1r_lc": "42", 71 | "qs_a1": "BAS-RHIN", 72 | "qs_a1_alt": null, 73 | "qs_a1_lc": "67", 74 | "qs_a2r": "STRASBOURG", 75 | "qs_a2r_alt": null, 76 | "qs_a2r_lc": "6", 77 | "qs_type": "Arrondissement", 78 | "qs_source": "France IGN", 79 | "qs_pop": null, 80 | "qs_id": null, 81 | "qs_gn_id": null, 82 | "qs_woe_id": null, 83 | "qs_scale": null 84 | }, 85 | "geometry": { 86 | "type": "Polygon", 87 | "coordinates": [<>] 88 | } 89 | } 90 | ``` 91 | 92 | ## NPM Module 93 | 94 | The `shapefile-stream` npm module can be found here: 95 | 96 | [https://npmjs.org/package/shapefile-stream](https://npmjs.org/package/shapefile-stream) 97 | 98 | ## Contributing 99 | 100 | Please fork and pull request against upstream master on a feature branch. 101 | 102 | Pretty please; provide unit tests and script fixtures in the `test` directory. 103 | 104 | ### Running Unit Tests 105 | 106 | ```bash 107 | $ npm test 108 | ``` 109 | 110 | ### Continuous Integration 111 | 112 | Travis tests every release against node version `0.10` 113 | 114 | [![Build Status](https://travis-ci.org/geopipes/shapefile-stream.png?branch=master)](https://travis-ci.org/geopipes/shapefile-stream) -------------------------------------------------------------------------------- /test/stream.js: -------------------------------------------------------------------------------- 1 | 2 | var fs = require('fs'), 3 | path = require('path'), 4 | through = require('through2'), 5 | ShapeFileStream = require('../lib/ShapeFileStream'); 6 | 7 | var fixtures = { 8 | example: { 9 | shp: path.resolve( __dirname + '/fixtures/example.shp' ), 10 | dbf: path.resolve( __dirname + '/fixtures/example.dbf' ) 11 | } 12 | } 13 | 14 | // extract a single record from the stream 15 | function getRecordNo( i, cb ){ 16 | var x = 0; 17 | return through.obj( function( chunk, enc, next ){ 18 | if( x++ === i ){ cb( chunk, enc ); } 19 | this.push( chunk, enc ); 20 | next(); 21 | }); 22 | } 23 | 24 | module.exports.stream = {}; 25 | 26 | // check the fixtures files are present 27 | module.exports.stream.fixtures = function(test, common) { 28 | test('check fixtures are present', function(t) { 29 | t.equal( fs.existsSync( fixtures.example.shp ), true, 'fixture file present' ); 30 | t.equal( fs.existsSync( fixtures.example.dbf ), true, 'fixture file present' ); 31 | t.end(); 32 | }); 33 | } 34 | 35 | // check the stream correctly parses the shapefile 36 | module.exports.stream.parse = function(test, common) { 37 | test('parse shapefiles', function(t) { 38 | var stream = new ShapeFileStream( fixtures.example.shp ); 39 | stream.pipe( through.obj( function( chunk, enc, next ){ 40 | t.equal( typeof chunk.type, 'string', 'type parsed correctly' ); 41 | t.equal( chunk.type, 'Feature', 'type parsed correctly' ); 42 | t.equal( typeof chunk.properties, 'object', 'properties parsed correctly' ); 43 | t.equal( Object.keys(chunk.properties).length, 22, 'properties parsed correctly' ); 44 | t.equal( typeof chunk.geometry, 'object', 'geometry parsed correctly' ); 45 | t.equal( typeof chunk.geometry.type, 'string', 'geometry parsed correctly' ); 46 | t.equal( Array.isArray(chunk.geometry.coordinates), true, 'geometry parsed correctly' ); 47 | })); 48 | t.end(); 49 | }); 50 | } 51 | 52 | // check the stream correctly parses the first shapefile 53 | module.exports.stream.first = function(test, common) { 54 | test('first shapefile', function(t) { 55 | var stream = new ShapeFileStream( fixtures.example.shp ); 56 | stream.pipe( getRecordNo( 1, function( chunk, enc ){ 57 | t.equal( chunk.type, 'Feature', 'type parsed correctly' ); 58 | t.equal( chunk.geometry.type, 'MultiPolygon', 'geometry parsed correctly' ); 59 | 60 | t.equal( Array.isArray(chunk.geometry.coordinates), true, 'geometry parsed correctly' ); 61 | t.equal( chunk.geometry.coordinates.length, 2, 'geometry parsed correctly' ); 62 | 63 | var firstCoordinate = chunk.geometry.coordinates[ 0 ]; 64 | t.equal( Array.isArray(firstCoordinate), true, 'coordinates parsed correctly' ); 65 | t.equal( firstCoordinate.length, 1, 'coordinates parsed correctly' ); 66 | 67 | var firstSubCoordinate = firstCoordinate[ 0 ]; 68 | t.equal( Array.isArray(firstSubCoordinate), true, 'coordinates parsed correctly' ); 69 | t.equal( firstSubCoordinate.length, 33, 'coordinates parsed correctly' ); 70 | 71 | var firstSubSubCoordinate = firstSubCoordinate[ 0 ]; 72 | var expected = [ 8.717200499999791, 47.69064399999985 ]; 73 | t.equal( Array.isArray(firstSubSubCoordinate), true, 'coordinates parsed correctly' ); 74 | t.equal( firstSubSubCoordinate.length, 2, 'coordinates parsed correctly' ); 75 | t.deepEqual( firstSubSubCoordinate, expected, 'coordinates parsed correctly' ); 76 | })); 77 | t.end(); 78 | }); 79 | } 80 | 81 | // check all shapefiles are extracted 82 | // @note: this number can be confirmed with the following command: 83 | // dbfcat test/fixtures/example.dbf | grep 'record' | wc -l 84 | module.exports.stream.allrecords = function(test, common) { 85 | var expected = 350; 86 | test('extract all shapefiles', function(t) { 87 | t.plan( expected ); 88 | var i = 0; 89 | new ShapeFileStream( fixtures.example.shp ) 90 | .pipe( through.obj( function( chunk, enc, next ){ 91 | t.equal( ++i <= expected, true, 'shapefile extracted' ); 92 | next(); 93 | })); 94 | }); 95 | } 96 | 97 | // check parser stops reading when hwm reached 98 | // this should only recieve 16 records (because that is the HWM of the through stream) 99 | // it will stay stuck in this state of 16 records until the next stream in the pipe 100 | // accepts some data (in this case never, since there is no next stream) 101 | module.exports.stream.hwm = function(test, common) { 102 | test('pause on hwm', function(t) { 103 | var stream = new ShapeFileStream( fixtures.example.shp ); 104 | 105 | var i = 0; 106 | var consumer = through.obj( function( chunk, enc, next ){ 107 | t.equal( ++i <= this._readableState.highWaterMark, true, 'shapefile extracted' ); 108 | this.push( chunk, enc ); 109 | next(); 110 | }); 111 | 112 | t.plan( consumer._readableState.highWaterMark ); 113 | stream.pipe( consumer ); 114 | }); 115 | } 116 | 117 | module.exports.all = function (tape, common) { 118 | 119 | function test(name, testFunction) { 120 | return tape('stream: ' + name, testFunction) 121 | } 122 | 123 | for( var testCase in module.exports.stream ){ 124 | module.exports.stream[testCase](test, common); 125 | } 126 | } --------------------------------------------------------------------------------