├── .gitignore
├── LICENSE.txt
├── README.md
├── benchmark.js
├── index.js
├── package.json
└── test
    ├── 14-4831-6159-undecorated.vector.pbfz
    ├── benchmark-tile.vector.pbf.gz
    └── index.js


/.gitignore:
--------------------------------------------------------------------------------
1 | node_modules
2 | .nyc_output/
3 | coverage/
4 | dump.rdb
5 | 


--------------------------------------------------------------------------------
/LICENSE.txt:
--------------------------------------------------------------------------------
 1 | 
 2 | ISC License
 3 | 
 4 | Copyright (c) 2017, Mapbox
 5 | 
 6 | Permission to use, copy, modify, and/or distribute this software for any
 7 | purpose with or without fee is hereby granted, provided that the above
 8 | copyright notice and this permission notice appear in all copies.
 9 | 
10 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 | WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 | MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 | ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 | WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 | ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 | OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.<Paste>
17 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # tilelive-decorator
 2 | 
 3 | [![Circle CI](https://circleci.com/gh/mapbox/tilelive-decorator.svg?style=svg&circle-token=c22fed6001fd3757877eba8c55f119dd19f66702)](https://circleci.com/gh/mapbox/tilelive-decorator)
 4 | 
 5 | Load vector tiles from a tilelive source and decorate them with properties from redis. So if you use
 6 | tilelive-s3 it can load tiles from s3, and add properties to features from redis.
 7 | 
 8 | ## usage
 9 | 
10 | #### with tilelive
11 | 
12 | Tilelive decorator registers several tilelive protocols:
13 | 
14 | - `decorator+s3:` for reading tiles from S3
15 | - `decorator+mbtiles:` for reading tiles from an mbtiles
16 | 
17 | ```js
18 | var tilelive = require('tilelive');
19 | var TileliveDecorator = require('tilelive-decorator');
20 | var TileliveS3 = require('tilelive-s3');
21 | 
22 | TileliveDecorator.registerProtocols(tilelive);
23 | TileliveS3.registerProtocols(tilelive);
24 | 
25 | tilelive.load('decorator+s3://test/{z}/{x}/{y}?key=id&sourceProps={"keep":["id","name"]}&redis=redis://localhost:6379', function(err, source) {
26 |   // source.getTile(z, x, y, callback);
27 | });
28 | ```
29 | 
30 | 
31 | 
32 | #### manually
33 | 
34 | ```js
35 | var TileliveDecorator = require('tilelive-decorator');
36 | 
37 | var uri = 'decorator+s3://test/{z}/{x}/{y}?key=id&sourceProps={"keep":["id","name"]}&redis=redis://localhost:6379'
38 | new TileliveDecorator(uri, function (err, source) {
39 |   // source.getTile(z, x, y, callback);
40 | });
41 | ```
42 | 
43 | 
44 | 
45 | #### options
46 | 
47 | **key** (required) - specifies what property in the source tiles will be matched to keys in redis.
48 | 
49 | **sourceProps** - a json object, specifying properties to `keep` from the source tile, and properties that are `required` to exist on features in the source tile. example: `{"keep": ["id", "class"], "required": ["rating"]}`. If all `required` properties don't exist on a feature, that feature is filtered out.
50 | 
51 | **redisProps** - a json object, specifying properties to `keep` from redis records, and properties that are `required` to exist on redis records. example: `{"keep": ["congestion"], "required": ["speed"]}`. If all `required` properties don't exist on a record, that record is filtered out and no new properties will be applied to features that match the record key.
52 | 
53 | **outputProps** - a json object, specifying properties to `keep` in the output tile after decoration, and properties that are `required` to exist on features in the output tile. example: `{"keep": ["class", "congestion"], "required": ["congestion"]}`. If all `required` properties don't exist on a feature, that feature is filtered out.
54 | 
55 | **redis** - a redis connection string, e.g. `redis://localhost:6379`.
56 | 
57 | **hashes** - If `hashes=true` is included, redis keys are treated as hash types as opposed to stringified JSON data in string type keys. In this case `hget` is used instead of the default `get` commands.
58 | 
59 | 
60 | ## development
61 | 
62 | #### setup
63 | 
64 | Tests and benchmarks require a local redis server
65 | 
66 | ```
67 | brew install redis
68 | git clone https://github.com/mapbox/tilelive-decorator
69 | cd tilelive-decorator
70 | npm install
71 | redis-server --save "" &
72 | ```
73 | 
74 | 
75 | #### benchmarks
76 | 
77 | ```js
78 | redis-cli flushall && node benchmark.js
79 | ```
80 | 
81 | #### tests
82 | 
83 | ```js
84 | redis-cli flushall && npm test
85 | ```
86 | 


--------------------------------------------------------------------------------
/benchmark.js:
--------------------------------------------------------------------------------
 1 | var fs = require('fs'),
 2 |   path = require('path'),
 3 |   zlib = require('zlib'),
 4 |   redis = require('redis'),
 5 |   Decorator = require('./index'),
 6 |   Benchmark = require('benchmark'),
 7 |   TileDecorator = require('tile-decorator');
 8 | 
 9 | 
10 | var tilePath = path.join(__dirname, 'test/benchmark-tile.vector.pbf.gz'),
11 |   benchOptions =  {defer: true, delay: 0.25, minSamples: 20},
12 |   tile = TileDecorator.read(zlib.gunzipSync(fs.readFileSync(tilePath))),
13 |   ids = TileDecorator.getLayerValues(tile.layers[0], '@id'),
14 |   client = redis.createClient();
15 | 
16 | var source = {
17 |   getTile: function (z, x, y, callback) {
18 |     fs.readFile(tilePath, callback);
19 |   }
20 | };
21 | 
22 | ids.forEach(function (id, i) {
23 |   var props = {
24 |     foo: Math.round(Math.random() * 100),
25 |     bar: Math.round(Math.random() * 100),
26 |   };
27 |   if (i % 2 === 0) props.baz = Math.round(Math.random() * 100);
28 | 
29 |   client.set(id, JSON.stringify(props));
30 | });
31 | 
32 | client.quit();
33 | client.on('end', function () {
34 |   console.log('starting benchmarking');
35 | 
36 |   var suite = new Benchmark.Suite('tilelive-decorator');
37 |   suite
38 |     .add('decorator',function (deferred) {
39 |       new Decorator({source: source, key: '@id', sourceProps: {keep: '@id,highway'}}, function (err, dec) {
40 |         dec.getTile(1, 1, 1, function (err, data) {
41 |           dec.close(function () {
42 |             deferred.resolve();
43 |           });
44 |         });
45 |       });
46 |     }, benchOptions)
47 |     .add('decorator#requiredKeysRedis',function (deferred) {
48 |       new Decorator({source: source, key: '@id', sourceProps: {keep: '@id,highway'}, redis: {requried: 'baz'}}, function (err, dec) {
49 |         dec.getTile(1, 1, 1, function (err, data) {
50 |           dec.close(function () {
51 |             deferred.resolve();
52 |           });
53 |         });
54 |       });
55 |     }, benchOptions)
56 |     .add('decorator#requiredKeys',function (deferred) {
57 |       new Decorator({source: source, key: '@id', sourceProps: {keep: '@id,highway', required: 'railway'}}, function (err, dec) {
58 |         dec.getTile(1, 1, 1, function (err, data) {
59 |           dec.close(function () {
60 |             deferred.resolve();
61 |           });
62 |         });
63 |       });
64 |     }, benchOptions)
65 |     .on('cycle', function (event) {
66 |       console.log(String(event.target));
67 |     })
68 |     .on('complete', function () { })
69 |     .run();
70 | });
71 | 


--------------------------------------------------------------------------------
/index.js:
--------------------------------------------------------------------------------
  1 | 'use strict';
  2 | 
  3 | var util = require('util');
  4 | var EventEmitter = require('events').EventEmitter;
  5 | var redis = require('redis');
  6 | var zlib = require('zlib');
  7 | var TileDecorator = require('@mapbox/tile-decorator');
  8 | var qs = require('querystring');
  9 | var url = require('url');
 10 | var tilelive = require('tilelive');
 11 | var LRU = require('lru-cache');
 12 | 
 13 | module.exports = Decorator;
 14 | util.inherits(Decorator, EventEmitter);
 15 | 
 16 | module.exports.loadAttributes = loadAttributes;
 17 | 
 18 | /**
 19 |  * @constructor
 20 |  * @param {object} uri
 21 |  * @param {function} callback
 22 |  */
 23 | function Decorator(uri, callback) {
 24 |     if (typeof uri === 'string') uri = url.parse(uri, true);
 25 |     if (typeof uri.query === 'string') uri.query = qs.parse(uri.query);
 26 |     var query = uri.query || uri;
 27 | 
 28 |     this.key = query.key;
 29 |     this.client = redis.createClient(query.redis);
 30 |     this.hashes = query.hashes === 'true';
 31 |     this.cache = new LRU({max: 10000});
 32 | 
 33 |     /*
 34 |         Each `props` supports `keep` and `required`.
 35 | 
 36 |         If a feature / record does not have all `required` properties at
 37 |         the given stage of the decoration cycle, it is rejected.
 38 | 
 39 |         `keep` specifies which columns should be retained at that stage
 40 |             - sourceProps.keep pulls only the named properties before decoration
 41 |             - redisProps.keep controls which properties will be queried from redis
 42 |             - outputProps.keep pulls only the named properties after decoration
 43 |     */
 44 |     this.sourceProps = parsePropertiesOption(query.sourceProps);
 45 |     this.redisProps = parsePropertiesOption(query.redisProps);
 46 |     this.outputProps = parsePropertiesOption(query.outputProps);
 47 | 
 48 |     // Source is loaded and provided explicitly.
 49 |     if (uri.source) {
 50 |         this._fromSource = uri.source;
 51 |         callback(null, this);
 52 |     } else {
 53 |         uri.protocol = uri.protocol.replace(/^decorator\+/, '');
 54 |         tilelive.auto(uri);
 55 |         tilelive.load(uri, function(err, fromSource) {
 56 |             if (err) return callback(err);
 57 |             this._fromSource = fromSource;
 58 |             callback(null, this);
 59 |         }.bind(this));
 60 |     }
 61 | }
 62 | 
 63 | Decorator.prototype.getInfo = function(callback) {
 64 |     this._fromSource.getInfo(callback);
 65 | };
 66 | 
 67 | // Fetch a tile from the source and extend its features' properties with data stored in Redis.
 68 | Decorator.prototype.getTile = function(z, x, y, callback) {
 69 |     var self = this;
 70 |     var client = this.client;
 71 |     var cache = this.cache;
 72 |     var useHashes = this.hashes;
 73 | 
 74 |     self._fromSource.getTile(z, x, y, function(err, buffer) {
 75 |         if (err) return callback(err);
 76 |         zlib.gunzip(buffer, function(err, buffer) {
 77 |             if (err) return callback(err);
 78 | 
 79 |             var tile = TileDecorator.read(buffer);
 80 |             var layer = tile.layers[0];
 81 |             if (!layer) return callback(new Error('No layers found'));
 82 |             if (self.sourceProps.required) TileDecorator.filterLayerByKeys(layer, self.sourceProps.required);
 83 | 
 84 |             var keysToGet = TileDecorator.getLayerValues(layer, self.key);
 85 | 
 86 |             loadAttributes(useHashes, keysToGet, client, cache, function(err, replies) {
 87 |                 if (err) return callback(err);
 88 |                 if (!replies) replies = [];
 89 |                 if (!useHashes) replies = replies.map(JSON.parse);
 90 | 
 91 |                 for (var i = 0; i < replies.length; i++) {
 92 |                     if (typeof replies[i] !== 'object') {
 93 |                         return callback(new Error('Invalid attribute data: ' + replies[i]));
 94 |                     }
 95 | 
 96 |                     if (replies[i] === null) continue; // skip checking
 97 | 
 98 |                     if (self.redisProps.required) {
 99 |                         for (var k = 0; k < self.redisProps.required.length; k++) {
100 |                             if (!replies[i].hasOwnProperty(self.redisProps.required[k])) {
101 |                                 replies[i] = null; // empty this reply
102 |                                 break;
103 |                             }
104 |                         }
105 |                     }
106 |                 }
107 | 
108 |                 if (self.redisProps.keep) {
109 |                     replies = replies.map(function(reply) {
110 |                         if (reply === null) return reply;
111 | 
112 |                         var keep = {};
113 |                         for (var k = 0; k < self.redisProps.keep.length; k++) {
114 |                             var key = self.redisProps.keep[k];
115 |                             if (reply.hasOwnProperty(key)) keep[key] = reply[key];
116 |                         }
117 |                         return keep;
118 |                     });
119 |                 }
120 | 
121 |                 if (self.sourceProps.keep) TileDecorator.selectLayerKeys(layer, self.sourceProps.keep);
122 |                 TileDecorator.updateLayerProperties(layer, replies);
123 |                 if (self.outputProps.required) TileDecorator.filterLayerByKeys(layer, self.outputProps.required);
124 |                 if (self.outputProps.keep) TileDecorator.selectLayerKeys(layer, self.outputProps.keep);
125 | 
126 |                 TileDecorator.mergeLayer(layer);
127 |                 zlib.gzip(new Buffer(TileDecorator.write(tile)), callback);
128 |             });
129 |         });
130 |     });
131 | };
132 | 
133 | function parsePropertiesOption(option) {
134 |     if (!option) return {};
135 |     if (typeof option === 'string') option = JSON.parse(option);
136 |     for (var key in option) {
137 |         option[key] = parseListOption(option[key]);
138 |     }
139 |     return option;
140 | }
141 | 
142 | function parseListOption(option) {
143 |     if (typeof option === 'string') return option.split(',');
144 |     return option;
145 | }
146 | 
147 | function loadAttributes(useHashes, keys, client, cache, callback) {
148 |     // Grab cached values from LRU, leave
149 |     // remaining for retrieval from redis.
150 |     var replies = [];
151 |     var loadKeys = [];
152 |     var loadPos = [];
153 |     var multi = client.multi();
154 | 
155 |     for (var i = 0; i < keys.length; i++) {
156 |         var cached = cache.get(keys[i]);
157 | 
158 |         if (cached) {
159 |             replies[i] = cached;
160 |         } else {
161 |             if (useHashes) {
162 |                 multi.hgetall(keys[i]);
163 |             } else {
164 |                 multi.get(keys[i]);
165 |             }
166 |             loadKeys.push(keys[i]);
167 |             loadPos.push(i);
168 |         }
169 |     }
170 | 
171 |     // Nothing left to hit redis for.
172 |     if (!loadKeys.length) return callback(null, replies, 0);
173 | 
174 |     multi.exec(function(err, loaded) {
175 |         if (err) return callback(err);
176 | 
177 |         // Insert redis-loaded values into the right positions and set in LRU cache.
178 |         for (var i = 0; i < loaded.length; i++) {
179 |             replies[loadPos[i]] = loaded[i];
180 |             cache.set(loadKeys[i], loaded[i]);
181 |         }
182 | 
183 |         return callback(null, replies, loaded.length);
184 |     });
185 | }
186 | 
187 | Decorator.prototype.close = function(callback) {
188 |     this.client.unref();
189 |     callback();
190 | };
191 | 
192 | Decorator.registerProtocols = function(tilelive) {
193 |     tilelive.protocols['decorator:'] = Decorator;
194 |     tilelive.protocols['decorator+s3:'] = Decorator;
195 |     tilelive.protocols['decorator+mbtiles:'] = Decorator;
196 | };
197 | 


--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "tilelive-decorator",
 3 |   "version": "4.0.1",
 4 |   "description": "Load vector tiles from another source and decorate them with properties from redis.",
 5 |   "main": "index.js",
 6 |   "directories": {
 7 |     "test": "test"
 8 |   },
 9 |   "scripts": {
10 |     "pretest": "eslint index.js test/index.js",
11 |     "test": "tape test/*.js",
12 |     "coverage": "nyc --reporter html tape test/*.js && open coverage/index.html"
13 |   },
14 |   "author": "",
15 |   "license": "ISC",
16 |   "devDependencies": {
17 |     "benchmark": "^2.1.3",
18 |     "eslint": "^1.10.3",
19 |     "eslint-config-mourner": "^1.0.1",
20 |     "nyc": "^5.5.0",
21 |     "pbf": "^1.3.5",
22 |     "tape": "^4.2.0",
23 |     "tilelive-s3": "^6.1.0",
24 |     "vector-tile": "^1.1.3"
25 |   },
26 |   "dependencies": {
27 |     "@mapbox/tile-decorator": "^4.0.0",
28 |     "lru-cache": "^4.0.0",
29 |     "redis": "^2.0.1",
30 |     "tilelive": "5.12.x"
31 |   },
32 |   "eslintConfig": {
33 |     "extends": "mourner",
34 |     "rules": {
35 |       "space-before-function-paren": [
36 |         2,
37 |         "never"
38 |       ],
39 |       "no-new": 0,
40 |       "camelcase": 0
41 |     }
42 |   }
43 | }
44 | 


--------------------------------------------------------------------------------
/test/14-4831-6159-undecorated.vector.pbfz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mapbox/tilelive-decorator/09fa58a6665a896fde03268175ae3e72b8f3df7e/test/14-4831-6159-undecorated.vector.pbfz


--------------------------------------------------------------------------------
/test/benchmark-tile.vector.pbf.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mapbox/tilelive-decorator/09fa58a6665a896fde03268175ae3e72b8f3df7e/test/benchmark-tile.vector.pbf.gz


--------------------------------------------------------------------------------
/test/index.js:
--------------------------------------------------------------------------------
  1 | 'use strict';
  2 | 
  3 | var TileliveDecorator = require('..');
  4 | var TileliveS3 = require('tilelive-s3');
  5 | var tape = require('tape');
  6 | var redis = require('redis');
  7 | var zlib = require('zlib');
  8 | var VectorTile = require('vector-tile').VectorTile;
  9 | var Protobuf = require('pbf');
 10 | var fs = require('fs');
 11 | var path = require('path');
 12 | var tilelive = require('tilelive');
 13 | var LRU = require('lru-cache');
 14 | 
 15 | function TestSource(uri, callback) {
 16 |     return callback(null, this);
 17 | }
 18 | TestSource.prototype.getTile = function(z, x, y, callback) {
 19 |     var key = [z, x, y].join('-');
 20 |     fs.readFile(path.join(__dirname, key + '-undecorated.vector.pbfz'), function(err, zdata) {
 21 |         if (err && err.code === 'ENOENT') return callback(new Error('Tile does not exist'));
 22 |         if (err) return callback(err);
 23 |         callback(null, zdata, {});
 24 |     });
 25 | };
 26 | 
 27 | var client = redis.createClient();
 28 | 
 29 | tape('setup', function(assert) {
 30 |     client.set('4', JSON.stringify({foo: 3, bar: 'baz'}), redis.print);
 31 |     client.unref();
 32 |     assert.end();
 33 | });
 34 | 
 35 | tape('load with decorator+s3 uri', function(assert) {
 36 |     TileliveDecorator.registerProtocols(tilelive);
 37 |     TileliveS3.registerProtocols(tilelive);
 38 |     tilelive.load('decorator+s3://test/{z}/{x}/{y}' +
 39 |             '?key=BoroCode&sourceProps={"keep":"BoroCode,BoroName,Shape_Area","required":"BoroCode"}' +
 40 |             '&redis=redis://localhost:6379', function(err, source) {
 41 |         assert.ifError(err);
 42 |         assert.equal(source.key, 'BoroCode');
 43 |         assert.equal(source.client.address, 'localhost:6379');
 44 |         assert.equal(source._fromSource instanceof TileliveS3, true);
 45 |         assert.deepEqual(source.sourceProps.required, ['BoroCode']);
 46 |         source.client.unref();
 47 |         assert.end();
 48 |     });
 49 | });
 50 | 
 51 | tape('setup source directly', function(assert) {
 52 |     new TestSource(null, function(err, testSource) {
 53 |         assert.ifError(err);
 54 |         var options = {
 55 |             key: 'BoroCode',
 56 |             source: testSource,
 57 |             sourceProps: {
 58 |                 keep: ['BoroCode', 'BoroName', 'Shape_Area']
 59 |             }
 60 |         };
 61 |         new TileliveDecorator(options, function(err, source) {
 62 |             assert.ifError(err);
 63 |             source.getTile(14, 4831, 6159, function(err, tile) {
 64 |                 assert.ifError(err);
 65 |                 assert.equal(tile.length, 489, 'buffer size check');
 66 |                 zlib.gunzip(tile, function(err, buffer) {
 67 |                     assert.ifError(err);
 68 |                     var tile = new VectorTile(new Protobuf(buffer));
 69 |                     var decorated = tile.layers.nycneighborhoods.feature(tile.layers.nycneighborhoods.length - 1);
 70 | 
 71 |                     assert.deepEqual(decorated.properties, {
 72 |                         BoroCode: 4,
 73 |                         BoroName: 'Queens',
 74 |                         Shape_Area: 316377640.656,
 75 |                         bar: 'baz',
 76 |                         foo: 3
 77 |                     });
 78 | 
 79 |                     assert.deepEqual(decorated.loadGeometry(), [
 80 |                         [{x: 1363, y: -128},
 81 |                          {x: 1609, y: 150},
 82 |                          {x: 1821, y: 443},
 83 |                          {x: 1909, y: 598},
 84 |                          {x: 2057, y: 917},
 85 |                          {x: 2188, y: 1241},
 86 |                          {x: 2530, y: 2026},
 87 |                          {x: 2635, y: 2197},
 88 |                          {x: 2743, y: 2432},
 89 |                          {x: 2811, y: 2726},
 90 |                          {x: 2836, y: 2781},
 91 |                          {x: 2894, y: 2849},
 92 |                          {x: 3097, y: 3019},
 93 |                          {x: 3784, y: 3700},
 94 |                          {x: 3731, y: 3796},
 95 |                          {x: 3652, y: 3874},
 96 |                          {x: 3475, y: 3949},
 97 |                          {x: 3554, y: 4002},
 98 |                          {x: 3754, y: 3942},
 99 |                          {x: 4025, y: 3828},
100 |                          {x: 4224, y: 3759},
101 |                          {x: 4224, y: 1426},
102 |                          {x: 4202, y: 1389},
103 |                          {x: 4110, y: 1300},
104 |                          {x: 3996, y: 1237},
105 |                          {x: 3870, y: 1199},
106 |                          {x: 3784, y: 1159},
107 |                          {x: 3706, y: 1104},
108 |                          {x: 3638, y: 1036},
109 |                          {x: 3585, y: 959},
110 |                          {x: 3429, y: 632},
111 |                          {x: 3284, y: 410},
112 |                          {x: 3174, y: 170},
113 |                          {x: 3026, y: 14},
114 |                          {x: 3537, y: -128},
115 |                          {x: 1363, y: -128}
116 |                     ]]);
117 | 
118 |                     source.client.unref();
119 |                     assert.end();
120 |                 });
121 |             });
122 |         });
123 |     });
124 | });
125 | 
126 | tape('redis config', function(assert) {
127 |     new TestSource(null, function(err, testSource) {
128 |         assert.ifError(err);
129 |         var options = {
130 |             key: 'BoroCode',
131 |             source: testSource,
132 |             redis: 'redis://foo',
133 |             sourceProps: {keep: ['BoroCode']}
134 |         };
135 |         new TileliveDecorator(options, function(err, source) {
136 |             assert.ifError(err);
137 |             source.client.once('error', function(err) {
138 |                 assert.equal(err.message.indexOf('Redis connection to foo:6379 failed - getaddrinfo'), 0);
139 |                 source.client.end();
140 |                 source.client.unref();
141 |                 assert.end();
142 |             });
143 |         });
144 |     });
145 | });
146 | 
147 | tape('setup', function(assert) {
148 |     client = redis.createClient();
149 |     client.set('4', '"bad data"', redis.print);
150 |     client.unref();
151 |     assert.end();
152 | });
153 | 
154 | tape('fail on bad redis data', function(assert) {
155 |     new TestSource(null, function(err, testSource) {
156 |         assert.ifError(err);
157 |         new TileliveDecorator({key: 'BoroCode', source: testSource, sourceProps: {keep: 'BoroCode'}}, function(err, source) {
158 |             assert.ifError(err);
159 |             source.getTile(14, 4831, 6159, function(err) {
160 |                 assert.ok(err, 'expected error');
161 |                 assert.equal(err.message, 'Invalid attribute data: bad data', 'expected error message');
162 |                 source.close(function() {
163 |                     assert.end();
164 |                 });
165 |             });
166 |         });
167 |     });
168 | });
169 | 
170 | 
171 | tape('setup', function(assert) {
172 |     client.set('QN99', JSON.stringify({foo: 3, bar: 'baz', baz: 'ignored'}), redis.print);
173 |     client.set('QN60', JSON.stringify({foo: 4, bar: 'baz', baz: 'ignored', qux: 'required'}), redis.print);
174 |     client.unref();
175 |     assert.end();
176 | });
177 | 
178 | tape('redisProps.keep', function(assert) {
179 |     new TestSource(null, function(err, testSource) {
180 |         assert.ifError(err);
181 |         var options = {
182 |             key: 'NTACode',
183 |             source: testSource,
184 |             sourceProps: {keep: ['BoroCode', 'NTACode']},
185 |             redisProps: {keep: ['foo', 'bar', 'qux']}
186 |         };
187 |         new TileliveDecorator(options, function(err, source) {
188 |             assert.ifError(err);
189 |             source.getTile(14, 4831, 6159, function(err, tile) {
190 |                 assert.ifError(err);
191 |                 zlib.gunzip(tile, function(err, buffer) {
192 |                     assert.ifError(err);
193 |                     var tile = new VectorTile(new Protobuf(buffer));
194 |                     var layer = tile.layers.nycneighborhoods;
195 |                     var qn99, qn60;
196 | 
197 |                     for (var i = 0; i < layer.length; i++) {
198 |                         var ft = layer.feature(i);
199 |                         if (ft.properties.NTACode === 'QN99') qn99 = ft;
200 |                         if (ft.properties.NTACode === 'QN60') qn60 = ft;
201 |                     }
202 | 
203 |                     assert.deepEqual(qn60.properties, {
204 |                         BoroCode: 4,
205 |                         NTACode: 'QN60',
206 |                         bar: 'baz',
207 |                         foo: 4,
208 |                         qux: 'required'
209 |                     });
210 | 
211 |                     assert.deepEqual(qn99.properties, {
212 |                         BoroCode: 4,
213 |                         NTACode: 'QN99',
214 |                         bar: 'baz',
215 |                         foo: 3
216 |                     });
217 | 
218 |                     source.client.unref();
219 |                     assert.end();
220 |                 });
221 |             });
222 |         });
223 |     });
224 | });
225 | 
226 | tape('redisProps.required', function(assert) {
227 |     new TestSource(null, function(err, testSource) {
228 |         assert.ifError(err);
229 |         var options = {
230 |             key: 'NTACode',
231 |             source: testSource,
232 |             sourceProps: {keep: ['BoroCode', 'NTACode']},
233 |             redisProps: {
234 |                 keep: ['foo', 'bar', 'qux'],
235 |                 required: ['qux']
236 |             }
237 |         };
238 |         new TileliveDecorator(options, function(err, source) {
239 |             assert.ifError(err);
240 |             source.getTile(14, 4831, 6159, function(err, tile) {
241 |                 assert.ifError(err);
242 |                 zlib.gunzip(tile, function(err, buffer) {
243 |                     assert.ifError(err);
244 |                     var tile = new VectorTile(new Protobuf(buffer));
245 |                     var layer = tile.layers.nycneighborhoods;
246 |                     var qn99, qn60;
247 | 
248 |                     for (var i = 0; i < layer.length; i++) {
249 |                         var ft = layer.feature(i);
250 |                         if (ft.properties.NTACode === 'QN99') qn99 = ft;
251 |                         if (ft.properties.NTACode === 'QN60') qn60 = ft;
252 |                     }
253 | 
254 |                     assert.deepEqual(qn60.properties, {
255 |                         BoroCode: 4,
256 |                         NTACode: 'QN60',
257 |                         bar: 'baz',
258 |                         foo: 4,
259 |                         qux: 'required'
260 |                     }, 'QN60 is decorated - it has required key qux');
261 | 
262 |                     assert.deepEqual(qn99.properties, {
263 |                         BoroCode: 4,
264 |                         NTACode: 'QN99'
265 |                     }, 'QN99 isn\'t decorated - it doesn\'t have required key qux');
266 | 
267 |                     source.client.unref();
268 |                     assert.end();
269 |                 });
270 |             });
271 |         });
272 |     });
273 | });
274 | 
275 | 
276 | tape('outputProps.keep', function(assert) {
277 |     new TestSource(null, function(err, testSource) {
278 |         assert.ifError(err);
279 |         var options = {
280 |             key: 'NTACode',
281 |             source: testSource,
282 |             outputProps: {keep: ['NTACode', 'foo']}
283 |         };
284 |         new TileliveDecorator(options, function(err, source) {
285 |             assert.ifError(err);
286 |             source.getTile(14, 4831, 6159, function(err, tile) {
287 |                 assert.ifError(err);
288 |                 zlib.gunzip(tile, function(err, buffer) {
289 |                     assert.ifError(err);
290 |                     var tile = new VectorTile(new Protobuf(buffer));
291 |                     var layer = tile.layers.nycneighborhoods;
292 |                     var qn99, qn60;
293 | 
294 |                     for (var i = 0; i < layer.length; i++) {
295 |                         var ft = layer.feature(i);
296 |                         if (ft.properties.NTACode === 'QN99') qn99 = ft;
297 |                         if (ft.properties.NTACode === 'QN60') qn60 = ft;
298 |                     }
299 | 
300 |                     assert.deepEqual(qn60.properties, {
301 |                         NTACode: 'QN60',
302 |                         foo: 4
303 |                     });
304 | 
305 |                     assert.deepEqual(qn99.properties, {
306 |                         NTACode: 'QN99',
307 |                         foo: 3
308 |                     });
309 | 
310 |                     source.client.unref();
311 |                     assert.end();
312 |                 });
313 |             });
314 |         });
315 |     });
316 | });
317 | 
318 | 
319 | tape('outputProps.required', function(assert) {
320 |     new TestSource(null, function(err, testSource) {
321 |         assert.ifError(err);
322 |         var options = {
323 |             key: 'NTACode',
324 |             source: testSource,
325 |             outputProps: {keep: ['NTACode', 'foo'], required: 'qux'}
326 |         };
327 |         new TileliveDecorator(options, function(err, source) {
328 |             assert.ifError(err);
329 |             source.getTile(14, 4831, 6159, function(err, tile) {
330 |                 assert.ifError(err);
331 |                 zlib.gunzip(tile, function(err, buffer) {
332 |                     assert.ifError(err);
333 |                     var tile = new VectorTile(new Protobuf(buffer));
334 |                     var layer = tile.layers.nycneighborhoods;
335 |                     var qn99, qn60;
336 | 
337 |                     for (var i = 0; i < layer.length; i++) {
338 |                         var ft = layer.feature(i);
339 |                         if (ft.properties.NTACode === 'QN99') qn99 = ft;
340 |                         if (ft.properties.NTACode === 'QN60') qn60 = ft;
341 |                     }
342 | 
343 |                     assert.deepEqual(qn60.properties, {
344 |                         NTACode: 'QN60',
345 |                         foo: 4
346 |                     }, 'QN60 isn\'t filtered out - it has required output property qux');
347 |                     assert.notOk(qn99, 'QN99 is filtered out - it doesn\'t have required output property qux');
348 | 
349 |                     source.client.unref();
350 |                     assert.end();
351 |                 });
352 |             });
353 |         });
354 |     });
355 | });
356 | var cache = new LRU({max: 1000});
357 | 
358 | tape('lru setup', function(assert) {
359 |     client = redis.createClient();
360 |     client.mset(
361 |         '1', JSON.stringify({foo: 1}),
362 |         '2', JSON.stringify({foo: 2}),
363 |         '3', JSON.stringify({foo: 3}),
364 |         '4', JSON.stringify({foo: 4}),
365 |         assert.end
366 |     );
367 | });
368 | 
369 | tape('loadAttributes (cache miss)', function(assert) {
370 |     TileliveDecorator.loadAttributes(false, ['1', '2'], client, cache, function(err, replies, loaded) {
371 |         assert.ifError(err);
372 |         assert.deepEqual(replies, ['{"foo":1}', '{"foo":2}'], 'loads');
373 |         assert.equal(cache.get('1'), '{"foo":1}', 'sets item 1 in cache');
374 |         assert.equal(cache.get('2'), '{"foo":2}', 'sets item 2 in cache');
375 |         assert.equal(loaded, 2, '2 items loaded from redis');
376 |         assert.end();
377 |     });
378 | });
379 | 
380 | tape('loadAttributes (cache hit)', function(assert) {
381 |     TileliveDecorator.loadAttributes(false, ['1', '2'], client, cache, function(err, replies, loaded) {
382 |         assert.ifError(err);
383 |         assert.deepEqual(replies, ['{"foo":1}', '{"foo":2}'], 'loads');
384 |         assert.equal(loaded, 0, '0 items loaded from redis');
385 |         assert.end();
386 |     });
387 | });
388 | 
389 | tape('loadAttributes (cache mixed)', function(assert) {
390 |     TileliveDecorator.loadAttributes(false, ['1', '3', '2', '4'], client, cache, function(err, replies, loaded) {
391 |         assert.ifError(err);
392 |         assert.deepEqual(replies, ['{"foo":1}', '{"foo":3}', '{"foo":2}', '{"foo":4}'], 'loads');
393 |         assert.equal(cache.get('1'), '{"foo":1}', 'sets item 1 in cache');
394 |         assert.equal(cache.get('2'), '{"foo":2}', 'sets item 2 in cache');
395 |         assert.equal(cache.get('3'), '{"foo":3}', 'sets item 3 in cache');
396 |         assert.equal(cache.get('4'), '{"foo":4}', 'sets item 4 in cache');
397 |         assert.equal(loaded, 2, '2 items loaded from redis');
398 |         assert.end();
399 |     });
400 | });
401 | 
402 | tape('lru teardown', function(assert) {
403 |     cache.reset();
404 |     client.del('1', '2', '3', '4', function() {
405 |         client.unref();
406 |         assert.end();
407 |     });
408 | });
409 | 
410 | 
411 | tape('lru setup', function(assert) {
412 |     client = redis.createClient();
413 |     var multi = client.multi();
414 |     multi.hset('1', 'foo', 1);
415 |     multi.hset('2', 'foo', 2);
416 |     multi.hset('3', 'foo', 3);
417 |     multi.hset('4', 'foo', 4);
418 |     multi.exec(assert.end);
419 | });
420 | 
421 | tape('loadAttributes (using hashes)', function(assert) {
422 |     TileliveDecorator.loadAttributes(true, ['1', '2'], client, cache, function(err, replies, loaded) {
423 |         assert.ifError(err);
424 |         assert.deepEqual(replies, [{foo: '1'}, {foo: '2'}], 'loads');
425 |         assert.equal(loaded, 2, '2 items loaded from redis');
426 |         assert.end();
427 |     });
428 | });
429 | 
430 | tape('lru teardown', function(assert) {
431 |     cache.reset();
432 |     client.del('1', '2', '3', '4', function() {
433 |         client.unref();
434 |         assert.end();
435 |     });
436 | });
437 | 
438 | 


--------------------------------------------------------------------------------