├── .gitignore ├── examples ├── live.js ├── simple.js └── cmd.js ├── package.json ├── README.md └── index.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | rec/ -------------------------------------------------------------------------------- /examples/live.js: -------------------------------------------------------------------------------- 1 | var MetaUtil = require('../'); 2 | 3 | 4 | //Live Mode! Updates every second 5 | var meta = MetaUtil().pipe(process.stdout) 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /examples/simple.js: -------------------------------------------------------------------------------- 1 | var MetaUtil = require('../'); 2 | 3 | var meta = MetaUtil({ 4 | 'delay': 1000, 5 | 'start': '000598424', //file number 6 | 'end': '001122000' //file number 7 | }).pipe(process.stdout) //outputs to console -------------------------------------------------------------------------------- /examples/cmd.js: -------------------------------------------------------------------------------- 1 | var MetaUtil = require('../'); 2 | 3 | //Using as a command line utility 4 | MetaUtil({ 5 | 'start': process.argv[2], 6 | 'end': process.argv[3], 7 | 'delay': process.argv[4] 8 | }).pipe(process.stdout) 9 | 10 | //Call with node cmd.js 001181708 001181721 -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "osm-meta-util", 3 | "version": "0.1.4", 4 | "description": "A tool to download OSM metadata", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "https://github.com/osmlab/osm-meta-util.git" 12 | }, 13 | "keywords": [ 14 | "osm", 15 | "metadata", 16 | "changesets" 17 | ], 18 | "author": "Marc Farra ", 19 | "contributors": [ 20 | "Alireza J ", 21 | "Dale Kunce " 22 | ], 23 | "license": "BSD", 24 | "bugs": { 25 | "url": "https://github.com/osmlab/osm-meta-util/issues" 26 | }, 27 | "homepage": "https://github.com/osmlab/osm-meta-util", 28 | "dependencies": { 29 | "node-expat": "^2.3.5", 30 | "request": "^2.53.0" 31 | }, 32 | "devDependencies": { 33 | "through": "^2.3.6" 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # OSM-Meta-util 2 | 3 | A tool to download and process OSM Metadata. This data contains the most recent annotations around a commit to OSM. Specifically, `commit text`, `username`, `bounding box`, `creation date` and `number of edits`. The data is downloaded from the [planet](http://planet.osm.org/replication/changesets/) repository, which contains minutely changesets to OSM. 4 | 5 | Once installed the tool can be used to pipe in compressed XML data between two dates and output it in JSON. OSM Meta Util can also be used in polling mode and continuously download the latest data every minute. 6 | 7 | A joint project built by [Development Seed](https://github.com/developmentseed) and the [American Red Cross](https://github.com/americanredcross). 8 | 9 | ## Installing 10 | 11 | Clone the repo or download it as a zip. `npm install` the dependencies. 12 | 13 | ## Running 14 | 15 | Require `osm-meta-util` in your node app. 16 | 17 | ```javascript 18 | var MetaUtil = require('osm-meta-util'); 19 | ``` 20 | 21 | The `MetaUtil` constructor builds a Node Stream, so you can pipe it into stream transformers or into `process.stdout` 22 | 23 | There are a few ways of using the utility: 24 | 25 | ### 1. Downloading between two dates 26 | 27 | The files are named in numerical order since February 28th, 2012. They're incremented every minute. You need the file name related to the start and end date. For example, `001181708` refers to [http://planet.osm.org/replication/changesets/001/181/708.osm.gz](http://planet.osm.org/replication/changesets/001/181/708.osm.gz), created on `2015-02-10 20:56`. 28 | 29 | ```javascript 30 | var MetaUtil = require('osm-meta-util'); 31 | // Getting historical metadata, specify a start & end 32 | var meta = MetaUtil({ 33 | 'delay': 1000, 34 | 'start': '001181708', //2015-02-10 20:56 35 | 'end': '001181721' //2015-02-10 21:09 36 | }).pipe(process.stdout) 37 | ``` 38 | 39 | ### 2. Continuously 40 | 41 | ```javascript 42 | // Live Mode! Updates every minute 43 | var meta = MetaUtil().pipe(process.stdout) 44 | ``` 45 | 46 | ### 3. Using as a command line utility 47 | 48 | ```javascript 49 | MetaUtil({ 50 | 'start': process.argv[2], 51 | 'end': process.argv[3], 52 | 'delay': process.argv[4] 53 | }).pipe(process.stdout) 54 | ``` 55 | 56 | Use it in combination with [jq](https://stedolan.github.io/jq/) 57 | 58 | ```sh 59 | node app 001181708 001181721 1000 | jq -c '{user:.user, date: .closed_at}' 60 | ``` 61 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var zlib = require('zlib'); 2 | var request = require('request'); 3 | var fs = require('fs'); 4 | var expat = require('node-expat'); 5 | var Readable = require('stream').Readable; 6 | var util = require('util'); 7 | 8 | util.inherits(MetaUtil, Readable); 9 | 10 | function MetaUtil(opts) { 11 | if (!(this instanceof MetaUtil)) return new MetaUtil(opts); 12 | 13 | opts = opts || {}; 14 | Readable.call(this, opts); 15 | 16 | var that = this; 17 | this.liveMode = (!opts.start && !opts.end && !opts.delay); 18 | this.state = Number(opts.start) || 0; 19 | this.end = Number(opts.end) || 1; 20 | this.diff = this.end - this.state + 1; 21 | this.delay = (opts.delay || 60000); 22 | this.initialized = true; 23 | 24 | this.baseURL = opts.baseURL || 'http://planet.osm.org/replication/changesets'; 25 | this._changesetAttrs = {}; 26 | this.started = false; 27 | } 28 | 29 | MetaUtil.prototype._read = function() { 30 | var that = this; 31 | if (!this.started) { 32 | if (this.liveMode) { 33 | request.get('http://planet.osm.org/replication/changesets/state.yaml', 34 | function(err, response, body) { 35 | that.state = Number(body.substr(body.length - 8)); 36 | that.end = Infinity; //don't stop 37 | that.delay = 60000; //every minute 38 | that.run(); 39 | that.started = true; 40 | } 41 | ); 42 | } else { 43 | this.run(); 44 | this.started = true; 45 | } 46 | } 47 | }; 48 | 49 | MetaUtil.prototype.run = function() { 50 | var that = this; 51 | var numProcessed = 0; 52 | 53 | function queueNext() { 54 | that.diff -= 1; 55 | if (that.diff > 0 || that.liveMode) { 56 | setTimeout(function() { 57 | next(); 58 | }, that.delay); 59 | } 60 | } 61 | 62 | var parserEnd = function(name, attrs) { 63 | if (name === 'changeset') { 64 | that.push(new Buffer(JSON.stringify(that._changesetAttrs) + '\n'), 'utf8'); 65 | } 66 | if (name === 'osm') { 67 | queueNext(); 68 | if (!that.liveMode && that.diff === 0) { 69 | that.push(null); 70 | } 71 | } 72 | }; 73 | 74 | var parserStart = function(name, attrs) { 75 | if (name === 'changeset') { 76 | if (attrs) { 77 | that._changesetAttrs = attrs; 78 | } 79 | } 80 | if (name === 'tag' && that._changesetAttrs && that._changesetAttrs.open === 'false') { 81 | that._changesetAttrs[attrs.k] = attrs.v; 82 | } 83 | }; 84 | 85 | function next() { 86 | //Add padding 87 | var stateStr = that.state.toString().split('').reverse(); 88 | var diff = 9 - stateStr.length; 89 | for (var i=0; i < diff; i++) { stateStr.push('0'); } 90 | stateStr = stateStr.join(''); 91 | 92 | //Create URL 93 | var url = ''; 94 | for (i=0; i<(stateStr.length/3); i++) { 95 | url += stateStr[i*3] + stateStr[i*3 + 1] + stateStr[i*3 + 2] + '/'; 96 | } 97 | 98 | //XML Parser 99 | var xmlParser = new expat.Parser('UTF-8'); 100 | xmlParser.on('startElement', parserStart); 101 | xmlParser.on('endElement', parserEnd); 102 | 103 | //Get YAML state file 104 | request.get('http://planet.osm.org/replication/changesets/state.yaml', 105 | function(err, response, body) { 106 | var nodata = true; 107 | //If YAML state is bigger, we can get a new file 108 | if (Number(body.substr(body.length - 8)) >= that.state) { 109 | var ss = request.get(that.baseURL + url.split('').reverse().join('') + '.osm.gz') 110 | .pipe(zlib.createUnzip()) 111 | .on('data', function(data) { 112 | nodata = (data.length === 0) && nodata; 113 | }) 114 | .on('end', function() { 115 | if (nodata) { 116 | queueNext(); 117 | ss.end(); 118 | } 119 | }) 120 | .pipe(xmlParser); 121 | 122 | that.state += 1; 123 | } 124 | } 125 | ); 126 | } 127 | next(); 128 | }; 129 | 130 | module.exports = MetaUtil; --------------------------------------------------------------------------------