├── .gitignore ├── LICENSE ├── README.md ├── examples ├── data │ ├── sample.json │ └── sample.osm ├── stream-to-stdout.js └── write-to-json.js ├── index.js ├── lib ├── logger.js └── node-osm-stream.js └── package.json /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | 5 | # Runtime data 6 | pids 7 | *.pid 8 | *.seed 9 | 10 | # Directory for instrumented libs generated by jscoverage/JSCover 11 | lib-cov 12 | 13 | # Coverage directory used by tools like istanbul 14 | coverage 15 | 16 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 17 | .grunt 18 | 19 | # Compiled binary addons (http://nodejs.org/api/addons.html) 20 | build/Release 21 | 22 | # Dependency directory 23 | # Deployed apps should consider commenting this line out: 24 | # see https://npmjs.org/doc/faq.html#Should-I-check-my-node_modules-folder-into-git 25 | node_modules 26 | .idea 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Hafiz Ismail 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # node-osm-stream 2 | 3 | ## Overview 4 | A fast and flexible NodeJS-based streaming parser for OpenStreetMap (.osm) files. 5 | 6 | * Both incoming and outgoing streams available for piping from/to other streams 7 | * Able to deal with large .osm files easily without loading the whole file onto memory 8 | * Provided with the flexibility to work with and modify OSM objects (nodes, ways, relations) easily as it streams to you 9 | 10 | *Powered by [node-expat](https://github.com/node-xmpp/node-expat) for blazing fast parsing.* 11 | 12 | ### Table of Content 13 | * [Quick Start](#quick-start) 14 | * [Install](#install) 15 | * [Basic example](#example-reading-from-a-local-osm-file) 16 | * [More examples](#more-examples) 17 | * [API](#api) 18 | * [Class: OSMStream](#class-osmstream) 19 | * [Methods](#methods) 20 | * [Events](#events) 21 | * [Events: 'node', 'way', 'relation'](#events-node-way-relation) 22 | * [Event: 'writeable'](#event-writeable) 23 | * [Event: 'flush'](#event-flush) 24 | * [Events inherited from stream.Transform](#events-inherited-from-streamtransform) 25 | * [Test](#test) 26 | * [Known Issues](#known-issues) 27 | * [Credits](#credits) 28 | * [Links](#links) 29 | * [License](#license) 30 | 31 | ---- 32 | 33 | ## Quick Start 34 | ### Install 35 | ```` 36 | npm install node-osm-stream 37 | ```` 38 | No fuss, no lints 39 | 40 | ### Example: Reading from a local .osm file 41 | ```` 42 | var fs = require('fs'); 43 | var OSMStream = require('node-osm-stream'); 44 | var parser = OSMStream(); 45 | 46 | // open a local .osm filestream 47 | fs.createReadStream('./path/to/file.osm') 48 | .pipe(parser); 49 | 50 | parser.on('node', function(node, callback){ 51 | // Modify current node object as you wish 52 | // and pass it back to the callback. 53 | // Or pass 'null' or 'false' to prevent the object being 54 | // written to outgoing stream 55 | console.log(node); 56 | callback(node); 57 | }); 58 | 59 | parser.on('way', function(way, callback){ callback(way); }); 60 | 61 | parser.on('relation', function(way, callback){ callback(relation); }); 62 | ```` 63 | 64 | Easy-peasy, lemon-squeezy. 65 | 66 | 67 | ### More examples 68 | More advanced examples are available in the ```./examples``` folder 69 | 70 | #### Stream and format outgoing data to console (process.stdout) 71 | Source: [/examples/stream-to-stdout.js](https://github.com/sogko/node-osm-stream/blob/master/examples/stream-to-stdout.js) 72 | 73 | To run example: 74 | ```` 75 | node ./examples/stream-to-stdout.js 76 | ```` 77 | 78 | #### Writing to a JSON file using Writeable steam (fs.createWriteStream) 79 | Source: [/examples/write-to-json.js](https://github.com/sogko/node-osm-stream/blob/master/examples/write-to-json.js) 80 | 81 | To run example: 82 | ```` 83 | node ./examples/write-to-json.js 84 | ```` 85 | 86 | ## API 87 | 88 | ### Class: OSMStream 89 | Class inherited from [stream.Transform](http://nodejs.org/api/stream.html#stream_class_stream_transform) 90 | 91 | ### Methods 92 | All methods are inherited from [stream.Transform](http://nodejs.org/api/stream.html#stream_class_stream_transform) 93 | 94 | ### Events 95 | #### Events: 'node', 'way', 'relation' 96 | When an object (node/way/relation) from the .osm file has been parsed fully with its attributes and children (if any) ready, it will emit a 'node' or 'way' or 'relation' event, depending on the object type. 97 | 98 | You can modify the outgoing data and passing it back to the callback. 99 | Or you can prevent the data from being passed downstream by passing back a *null* or *false* 100 | 101 | It's important to note that since this is a streaming parser, any other objects (ways/relations) that may have referenced a skipped node may still hold its reference. It is up to the implementation to remove its references. 102 | 103 | To see an example of a possible implementation, take a look at ```` /examples/write-to-json.js```` 104 | 105 | Note: If this event was registered, the callback must be passed back. 106 | 107 | ```` 108 | parser.on('node', function(node, callback) { 109 | // modify the node object as necessary and pass back to callback 110 | // or send a null or false to prevent it from going downstream 111 | callback(node); 112 | }); 113 | 114 | parser.on('way', function(way, callback) { 115 | ... 116 | callback(way); 117 | }); 118 | parser.on('relation', function(relation, callback) { 119 | ... 120 | callback(relation); 121 | }); 122 | ```` 123 | 124 | #### Event: 'writeable' 125 | When a chunk of data is ready to be written to the outgoing stream, it will emit a 'writeable' event. 126 | 127 | You can modify the outgoing data and passing it back to the callback. 128 | Or you can prevent the data from being passed downstream by passing back a *null* or *false* 129 | 130 | Note: If this event was registered, the callback must be passed back. 131 | 132 | ```` 133 | parser.on('writeable', function(data, callback) { 134 | // there is some data to be passed to outgoing stream 135 | // modify 'data' as needed 136 | callback(data); 137 | }); 138 | ```` 139 | 140 | #### Event: 'flush' 141 | After all the written data has been consumed through the outgoing stream, it will emit a 'flush' event. 142 | This will happened before the 'end' event is sent to signal the end of the readable side. 143 | 144 | You can choose to pass in as much data as needed by passing it through the callback. 145 | Any data passed back will be written to the outgoing stream before the 'end' event is emitted. 146 | 147 | Note: If this event was registered, the callback must be passed back. 148 | 149 | ```` 150 | parser.on('flush', function(callback) { 151 | var last_data = 'This is the last string sent to the outgoing stream'; 152 | callback(last_data); 153 | }); 154 | ```` 155 | 156 | #### Events inherited from stream.Transform 157 | In addition to the events above, the following are events inherited from stream.Transform class. 158 | Please refer to the offical documentation for more info: [NodeJS API Documentation: stream.Transform](http://nodejs.org/api/stream.html#stream_class_stream_transform) 159 | 160 | * Event: 'readable' 161 | * Event: 'data' 162 | * Event: 'end' 163 | * Event: 'close' 164 | * Event: 'error' 165 | * Event: 'drain' 166 | * Event: 'finish' 167 | * Event: 'pipe' 168 | * Event: 'unpipe' 169 | 170 | 171 | ## Test 172 | ```` 173 | npm test 174 | ````` 175 | 176 | ## Known Issues 177 | 178 | 179 | 180 | ## Credits 181 | 182 | * [Hafiz Ismail](https://github.com/sogko) 183 | * [node-expat](https://github.com/node-xmpp/node-expat) 184 | 185 | ## Links 186 | * [twitter.com/sogko](https://twitter.com/sogko) 187 | * [github.com/sogko](https://github.com/sogko) 188 | * [medium.com/@sogko](https://medium.com/@sogko) 189 | 190 | ## License 191 | Copyright (c) 2014 Hafiz Ismail. This software is licensed under the [MIT License](https://github.com/sogko/node-osm-stream/raw/master/LICENSE). 192 | -------------------------------------------------------------------------------- /examples/data/sample.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"lon":103.9532473,"lat":1.3305096,"type":"node","tags":{"name":"MRT East West Line (EW)"},"changeset":12508529,"version":2,"visible":true,"timestamp":"2012-07-27T09:06:22.000Z","uid":741163,"user":"JaLooNz","id":1558637821,"_key":1558637821}, 3 | {"lon":103.7880596,"lat":1.4436946,"type":"node","tags":{},"changeset":12912977,"version":1,"visible":true,"timestamp":"2012-08-30T03:59:05.000Z","uid":741163,"user":"JaLooNz","id":1889319692,"_key":1889319692}, 4 | {"lon":103.7886344,"lat":1.4428549,"type":"node","tags":{},"changeset":12912977,"version":1,"visible":true,"timestamp":"2012-08-30T03:59:05.000Z","uid":741163,"user":"JaLooNz","id":1889319691,"_key":1889319691}, 5 | {"lon":103.7889661,"lat":1.4422971,"type":"node","tags":{},"changeset":12912977,"version":1,"visible":true,"timestamp":"2012-08-30T03:59:05.000Z","uid":741163,"user":"JaLooNz","id":1889319690,"_key":1889319690}, 6 | {"lon":103.9622502,"lat":1.3358121,"type":"node","tags":{},"changeset":12665844,"version":2,"visible":true,"timestamp":"2012-08-09T09:02:03.000Z","uid":473466,"user":"zomgvivian","id":1840701015,"_key":1840701015}, 7 | {"lon":103.9606557,"lat":1.3369151,"type":"node","tags":{},"changeset":12665844,"version":2,"visible":true,"timestamp":"2012-08-09T09:01:57.000Z","uid":473466,"user":"zomgvivian","id":1840701285,"_key":1840701285}, 8 | {"lon":103.9604927,"lat":1.3374706,"type":"node","tags":{},"changeset":12665844,"version":2,"visible":true,"timestamp":"2012-08-09T09:01:59.000Z","uid":473466,"user":"zomgvivian","id":1840701403,"_key":1840701403}, 9 | {"lon":103.9606557,"lat":1.338389,"type":"node","tags":{},"changeset":12665844,"version":2,"visible":true,"timestamp":"2012-08-09T09:02:03.000Z","uid":473466,"user":"zomgvivian","id":1840701531,"_key":1840701531}, 10 | {"nodes":[1840701015,1840701285,1840701403,1840701531],"type":"way","tags":{"color":"#0354A6","colour":"#0354A6","construction":"subway","designation":"Downtown Line","layer":"-2","name":"Downtown Line MRT","railway":"construction","ref":"DTL","tunnel":"yes","wheelchair":"yes"},"changeset":19425811,"version":10,"visible":true,"timestamp":"2013-12-13T09:48:24.000Z","uid":741163,"user":"JaLooNz","id":178063325,"_key":178063325}, 11 | {"nodes":[1889319692,1889319691,1889319690],"type":"way","tags":{"electrified":"yes","fixme":"Alignment","name":"Thomson Line (TSL)","proposed":"subway","railway":"proposed","ref":"TSL","tunnel":"yes"},"changeset":15968162,"version":4,"visible":true,"timestamp":"2013-05-04T08:06:56.000Z","uid":622060,"user":"Cort Wee","id":178591205,"_key":178591205}, 12 | {"members":["0"],"type":"relation","tags":{"colour":"blue","from":"Bukit Panjang","name":"MRT Downtown Line (DTL)","network":"Singapore Rail","operator":"SBS Transit","ref":"Downtown Line","route":"subway","to":"Expo","type":"route","wheelchair":"yes"},"changeset":19425811,"version":12,"visible":true,"timestamp":"2013-12-13T09:42:30.000Z","uid":741163,"user":"JaLooNz","id":2313458,"_key":2313458}, 13 | {"members":["0","1"],"type":"relation","tags":{"color":"#734538","colour":"#734538","from":"Woodlands North","name":"MRT Thomson Line (TSL)","network":"Singapore Rail","operator":"NA","ref":"Thomson Line","route":"subway","to":"Gardens by the Bay","type":"route","wheelchair":"yes"},"changeset":17391195,"version":2,"visible":true,"timestamp":"2013-08-18T04:44:25.000Z","uid":741163,"user":"JaLooNz","id":2383439,"_key":2383439}, 14 | {"members":["0","1"],"type":"relation","tags":{},"changeset":19425811,"version":12,"visible":true,"timestamp":"2013-12-13T09:42:30.000Z","uid":741163,"user":"JaLooNz","id":9313458,"_key":9313458} 15 | ] 16 | -------------------------------------------------------------------------------- /examples/data/sample.osm: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | -------------------------------------------------------------------------------- /examples/stream-to-stdout.js: -------------------------------------------------------------------------------- 1 | /** 2 | * =========================================================== 3 | * How to stream outgoing pipe from node-osm-stream to stdout 4 | * ========================================================== 5 | * 6 | * This example shows the basic principle of piping to both incoming and outgoing streams. 7 | * 8 | * We will 9 | * - pipe in data from a ReadStream and 10 | * - pipe out the data to process.stdout (console) 11 | * 12 | */ 13 | 14 | var fs = require('fs'); 15 | var OSMStream = require('./../'); 16 | var EOL = require('os').EOL; 17 | 18 | var source = __dirname + '/data/sample.osm'; 19 | 20 | // open .osm file as a Readable file stream 21 | var readstream = fs.createReadStream(source); 22 | 23 | // initialize our .osm stream parser 24 | var parser = OSMStream(); // or new OSMStream() 25 | 26 | // attach our pipelines 27 | readstream 28 | .pipe(parser) 29 | .pipe(process.stdout); 30 | 31 | readstream.on('open', function () { 32 | console.log('Opened .osm file:', source, '\n'); 33 | }); 34 | 35 | parser.on('writeable', function (data, callback) { 36 | // simply format the outgoing data that's going to be printed on our console as a string 37 | callback(['Type: ', data.type, EOL, 'Id: ', data.id, EOL, EOL].join('')); 38 | }); 39 | 40 | parser.on('end', function () { 41 | console.log('Finished parsing our .osm file'); 42 | }); -------------------------------------------------------------------------------- /examples/write-to-json.js: -------------------------------------------------------------------------------- 1 | /** 2 | * ======================================================= 3 | * How to use node-osm-stream to read an .osm file stream 4 | * and write it to a file as a valid JSON 5 | * ====================================================== 6 | * 7 | * In this example, we will parse through the .osm file and selectively filter 8 | * out certain nodes that we are not interested in. 9 | * 10 | * The rest of the data are then written out to a file as JSON objects in an array. 11 | * 12 | * After which, we will ensure that its a valid JSON file. 13 | * 14 | */ 15 | 16 | var fs = require('fs'); 17 | var OSMStream = require('./../'); 18 | var JSONStream = require('JSONStream'); 19 | var EOL = require('os').EOL; 20 | 21 | // list of node ids that we want to skip when writing our data to file 22 | var skipNodes = [1840700822, 1889319397]; 23 | 24 | var source = __dirname + '/data/sample.osm'; 25 | var destination = __dirname + '/data/sample.json'; 26 | 27 | // open .osm file as a Readable file stream 28 | var readstream = fs.createReadStream(source); 29 | 30 | // initialize our .osm stream parser 31 | var parser = OSMStream(); // or new OSMStream() 32 | 33 | // open a Writeable file stream to write our JSON file 34 | var writestream = fs.createWriteStream(destination); 35 | 36 | // a JSON parser that we will use later to ensure that we have our valid JSON file 37 | var jsonParser; 38 | 39 | // attach our pipelines 40 | readstream 41 | .pipe(parser) 42 | .pipe(writestream); 43 | 44 | readstream.on('open', function () { 45 | console.log('Opened .osm file:', source, '\n'); 46 | }); 47 | 48 | var firstLine = true; 49 | parser.on('writeable', function (data, callback) { 50 | if (firstLine) { 51 | firstLine = false; 52 | // add an opening square bracket before the first JSON object 53 | // that we are about to write to file 54 | callback('[' + EOL + ' ' + JSON.stringify(data)); 55 | } else { 56 | // prepend a comma to the rest of the JSON objects 57 | callback(',' + EOL + ' ' + JSON.stringify(data)); 58 | } 59 | }); 60 | parser.on('flush', function (callback) { 61 | // add closing square bracket after all data has been written to file 62 | callback(EOL + ']' + EOL); 63 | }); 64 | 65 | parser.on('node', function (node, callback) { 66 | // check current node id against list of nodes we want to skip 67 | if (skipNodes.indexOf(node.id) > -1) { 68 | // send back null or false to skip object 69 | callback(); 70 | } else { 71 | callback(node); 72 | } 73 | }); 74 | parser.on('way', function (way, callback) { 75 | 76 | // remove node ids that we do not want from current way object 77 | // note that we are directly modifying the given way object and passing it back 78 | for (var i in skipNodes){ 79 | var skipNode = skipNodes[i]; 80 | if (way.nodes.indexOf(skipNode) > -1){ 81 | way.nodes.splice(way.nodes.indexOf(skipNode), 1); 82 | } 83 | } 84 | 85 | // passing back the modified object 86 | callback(way); 87 | }); 88 | parser.on('relation', function (relation, callback) { 89 | 90 | // remove node ids that we do not want from current relation object 91 | var newMembers = []; 92 | for (var member in relation.members) { 93 | var ref = member.ref; 94 | if (skipNodes.indexOf(ref) < 0) { 95 | newMembers.push(member); 96 | } 97 | } 98 | 99 | // update current relation with the filtered list of relation.members 100 | relation.members = newMembers; 101 | 102 | // passing back the modified object 103 | callback(relation); 104 | }); 105 | 106 | parser.on('end', function () { 107 | console.log('Finished parsing our .osm file'); 108 | console.log('Bytes read from incoming stream:', parser.bytesRead, 'bytes'); 109 | console.log('Bytes written to outgoing stream:', parser.bytesWritten, 'bytes\n'); 110 | 111 | console.log('Checking that written file is a valid JSON:', destination); 112 | 113 | jsonParser = JSONStream.parse(['rows', true]); 114 | fs.createReadStream(destination).pipe(jsonParser); 115 | 116 | var isValidJSON = true; 117 | jsonParser.on('error', function(err){ 118 | // if we receive error, json is invalid 119 | console.log('JSON error', err); 120 | isValidJSON = false; 121 | }); 122 | jsonParser.on('close', function(){ 123 | console.log('JSON file check:', (isValidJSON)?'OK' : 'ERROR'); 124 | console.log(); 125 | }); 126 | 127 | }); -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | module.exports = require('./lib/node-osm-stream'); -------------------------------------------------------------------------------- /lib/logger.js: -------------------------------------------------------------------------------- 1 | /** 2 | * NodeJS logger with datetime stamp just because. 3 | * =============================================== 4 | * 5 | * Hold onto your socks now. 6 | * 7 | * Usage example 8 | * var logger = new (require('./logger'))().log; 9 | * var prefixedLogger = new (require('./logger'))('[PREFIX]').log; 10 | * 11 | * var a = 'Hey girl.', b = { smooth: 'water' } 12 | * 13 | * logger('Hello world.', a , b); 14 | * prefixedLogger('Hello world.', a , b); 15 | * -> [2014-07-05T17:59:24Z] Hello world. Hey girl. { smooth: 'water' } 16 | * -> [2014-07-05T17:59:24Z] [PREFIX] Hello world. Hey girl. { smooth: 'water' } 17 | * 18 | * 19 | * References: 20 | * NodeJS implementation of console.log 21 | * - https://github.com/joyent/node/blob/master/lib/console.js#L52-L54 22 | * 23 | * For padded datetime stamp 24 | * - http://stackoverflow.com/a/12550320/245196 25 | * 26 | * Gisted by Hafiz Ismail / @sogko 27 | * Available @ https://gist.github.com/sogko/fcb7c1620850a362968c 28 | * 29 | */ 30 | var util = require('util'); 31 | var EOL = require('os').EOL; 32 | var pad = function (n){ return n<10 ? ['0',n].join('') : n; }; 33 | function Logger(prefix){ 34 | if (!(this instanceof Logger)) { return new Logger(prefix); } 35 | this.prefix = prefix || ''; 36 | this.log = function (){ 37 | if (arguments instanceof Object){ arguments = Array.prototype.slice.call(arguments, 0);} 38 | var d = new Date(); 39 | d = ['[',d.getUTCFullYear(),'-',pad(d.getUTCMonth()+1),'-',pad(d.getUTCDate()),'T',pad(d.getUTCHours()),':',pad(d.getUTCMinutes()),':',pad(d.getUTCSeconds()),'Z] ', this.prefix, ' ' ].join(''); 40 | process.stdout.write(d); 41 | arguments.forEach(function(o){process.stdout.write(util.format(o)+' ');}); 42 | process.stdout.write(EOL); 43 | }.bind(this); 44 | } 45 | module.exports = Logger; -------------------------------------------------------------------------------- /lib/node-osm-stream.js: -------------------------------------------------------------------------------- 1 | var expat = require('node-expat'); 2 | var util = require('util'); 3 | var extend = util._extend; 4 | var Transform = require('stream').Transform; 5 | 6 | var logger = require('./logger')('[OSMStream]').log; 7 | logger = function () {}; // comment out to show debug logs 8 | 9 | function OSMStream(options) { 10 | if (!(this instanceof OSMStream)) return new OSMStream(options); 11 | 12 | Transform.call(this, options); 13 | 14 | var primaryElements = ['node', 'way', 'relation']; // emit-ables 15 | var subElementName = ['tag', 'nd', 'member']; 16 | 17 | // size of bytes read from source so far 18 | this.bytesRead = 0; 19 | // size of bytes written to stream so far 20 | this.bytesWritten = 0; 21 | 22 | // initialize parser and its event handlers 23 | this._parser = new expat.Parser(); 24 | 25 | this._parser.on('pipe', (function onParserPipe(src) { 26 | logger('_parser received pipe from ', src.path || ''); 27 | }).bind(this)); 28 | 29 | this._parser.on('startElement', (function onParserStartElement(name, attrs) { 30 | var funcname = ['__parse_', name].join(''); 31 | if ((primaryElements.indexOf(name) > -1 && (OSMStream.prototype[funcname])) || 32 | (subElementName.indexOf(name) > -1 && this._currentElement !== null && (OSMStream.prototype[funcname]))) { 33 | OSMStream.prototype[funcname].call(this, attrs); 34 | } 35 | }).bind(this)); 36 | 37 | this._parser.on('endElement', (function onParserEndElement(name) { 38 | if ((primaryElements.indexOf(name) < 0)) return; 39 | 40 | var clone = extend({}, this._currentElement); 41 | this._currentElement = null; 42 | 43 | var pushData = function pushData(data) { 44 | if (!data) return; 45 | 46 | if (this.listeners('writeable').length > 0) { 47 | this.emit('writeable', data, (function onWriteable(data) { 48 | this.__push(data); 49 | }).bind(this)); 50 | } else { 51 | this.__push(data); 52 | } 53 | 54 | }.bind(this); 55 | 56 | if (this.listeners(name).length > 0) { 57 | this.emit(name, clone, (function onElement(data) { 58 | pushData(data); 59 | }.bind(this))); 60 | } else { 61 | pushData(clone); 62 | } 63 | }).bind(this)); 64 | 65 | this._parser.on('error', (function onParserError(err) { 66 | this.emit('error', err); 67 | }).bind(this)); 68 | 69 | this._parser.on('end', (function onParserEnd() { 70 | }).bind(this)); 71 | 72 | this.on('pipe', (function onPipe(src) { 73 | if (src === this._parser) return; 74 | // attaching stream pipe to parser 75 | logger('Received pipe from ', src.toString(), ', attaching pipe to _parser'); 76 | src.pipe(this._parser); 77 | this._parser.pipe(this); 78 | }).bind(this)); 79 | 80 | } 81 | 82 | util.inherits(OSMStream, Transform); 83 | 84 | OSMStream.prototype._transform = function (chunk, encoding, callback) { 85 | this.bytesRead += chunk.length; 86 | // Let data pass through here, transformation and 87 | // writing to stream will be done by this._parser. 88 | callback(); 89 | }; 90 | 91 | OSMStream.prototype._flush = function (callback) { 92 | if (this.listeners('flush').length > 0) { 93 | this.emit('flush', (function onFlush(data) { 94 | this.__push(data); 95 | callback(); 96 | }).bind(this)); 97 | } else { 98 | callback(); 99 | } 100 | }; 101 | 102 | OSMStream.prototype.__push = function (object) { 103 | // a wrapper to _push(), to ensure that data are Buffer-able strings 104 | // and keep count of bytes written 105 | 106 | if (!object) return; 107 | 108 | if (typeof object !== 'string' && !(object instanceof String)) { 109 | object = JSON.stringify(object); 110 | } 111 | 112 | var chunk = new Buffer(object); 113 | this.push(chunk); 114 | this.bytesWritten += chunk.length; 115 | 116 | // Note: how about the convention of .push(EOF) or .push(null) to stop fs.writeStream? 117 | }; 118 | 119 | OSMStream.prototype.__parseCommonAttributes = function (attrs) { 120 | // Reference: http://wiki.openstreetmap.org/wiki/Data_Primitives#Common_attributes 121 | var obj = {}; 122 | obj.id = parseInt(attrs.id); 123 | obj.user = attrs.user; 124 | obj.uid = parseInt(attrs.uid); 125 | obj.timestamp = new Date(attrs.timestamp); 126 | obj.visible = (attrs.visible) ? (['', attrs.visible].join('').toLowerCase() !== 'false') : true; // default: true // TODO: unittest 127 | obj.version = parseInt(attrs.version); 128 | obj.changeset = parseInt(attrs.changeset); 129 | obj.tags = {}; 130 | return obj; 131 | }; 132 | 133 | // OSMStream.prototype.__parse_* are not free from side-effects 134 | // They modify the object's internal state (this._currentElement) 135 | OSMStream.prototype.__parse_node = function (attrs) { 136 | var obj = this.__parseCommonAttributes(attrs); 137 | obj.type = 'node'; 138 | obj.lat = parseFloat(attrs.lat); 139 | obj.lon = parseFloat(attrs.lon); 140 | this._currentElement = obj; 141 | }; 142 | 143 | OSMStream.prototype.__parse_way = function (attrs) { 144 | var obj = this.__parseCommonAttributes(attrs); 145 | obj.type = 'way'; 146 | obj.nodes = []; 147 | this._currentElement = obj; 148 | }; 149 | 150 | OSMStream.prototype.__parse_relation = function (attrs) { 151 | var obj = this.__parseCommonAttributes(attrs); 152 | obj.type = 'relation'; 153 | obj.members = []; 154 | this._currentElement = obj; 155 | }; 156 | 157 | OSMStream.prototype.__parse_tag = function (attrs) { 158 | if (!this._currentElement) return; 159 | if (!this._currentElement.tags) this._currentElement.tags = {}; 160 | this._currentElement.tags[attrs.k] = attrs.v; 161 | }; 162 | 163 | OSMStream.prototype.__parse_nd = function (attrs) { 164 | if (!this._currentElement) return; 165 | if (!this._currentElement.nodes) this._currentElement.nodes = []; 166 | this._currentElement.nodes.push(parseInt(attrs.ref)); 167 | }; 168 | 169 | OSMStream.prototype.__parse_member = function (attrs) { 170 | var member = { 171 | type: attrs.type, 172 | role: attrs.role || null, 173 | ref: parseInt(attrs.ref) 174 | }; 175 | 176 | if (!this._currentElement) return; 177 | if (!this._currentElement.members) this._currentElement.members = []; 178 | this._currentElement.members.push(member); 179 | 180 | }; 181 | module.exports = OSMStream; -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "node-osm-stream", 3 | "version": "0.2.2", 4 | "description": "A fast and flexible NodeJS-based streaming parser for OpenStreetMap (.osm) files.", 5 | "homepage": "https://sogko.github.io/node-osm-stream", 6 | "main": "index.js", 7 | "scripts": { 8 | "test": "npm test" 9 | }, 10 | "repository": { 11 | "type": "git", 12 | "url": "https://github.com/sogko/node-osm-stream" 13 | }, 14 | "keywords": [ 15 | "openstreetmap", 16 | "osm", 17 | "stream", 18 | "streamable", 19 | "parser" 20 | ], 21 | "author": { 22 | "name": "Hafiz Ismail", 23 | "email": "hafiz@wehavefaces.net", 24 | "url": "http://wehavefaces.net" 25 | }, 26 | "license": "MIT", 27 | "dependencies": { 28 | "node-expat": "^2.3.0" 29 | }, 30 | "devDependencies": { 31 | "JSONStream": "^0.8.4" 32 | } 33 | } 34 | --------------------------------------------------------------------------------