├── .gitignore ├── README.md ├── package.json ├── index.js └── test └── resolver.js /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | node_modules 3 | 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # resolve 2 | 3 | Resolve URLs and cache results to Redis 4 | 5 | ## Usage 6 | 7 | ```javascript 8 | var createResolver = require("./resolver").createResolver; 9 | 10 | var resolver = createResolver(options); 11 | 12 | resolver.resolve(url, function(err, resolved, cached){}); 13 | ``` 14 | 15 | Where 16 | 17 | **options** include the following properties 18 | 19 | * **redis** is an object `{host:, port:, db:}` 20 | * **cacheTTL** is the time resolved URLs will be stored 21 | * **userAgent** is the User Agent header value for retrieving urls 22 | 23 | **resolved** is the resolved URL or false if not found 24 | 25 | **cached** is `true` if the data was loaded from db 26 | 27 | ## License 28 | 29 | **MIT** -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cached-resolver", 3 | "description": "Resolve URLs and cache results to Redis", 4 | "version": "0.1.0", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "nodeunit test" 8 | }, 9 | "author": "Andris Reinman", 10 | "dependencies": { 11 | "redis": "~0.10.0", 12 | "hiredis": "~0.1.16", 13 | "resolver": "~0.1.11" 14 | }, 15 | "devDependencies": { 16 | "nodeunit": "~0.8.2" 17 | }, 18 | "license": "MIT", 19 | "directories": { 20 | "test": "test" 21 | }, 22 | "repository": { 23 | "type": "git", 24 | "url": "git@github.com:andris9/cached-resolver.git" 25 | }, 26 | "bugs": { 27 | "url": "https://github.com/andris9/cached-resolver/issues" 28 | }, 29 | "homepage": "https://github.com/andris9/cached-resolver" 30 | } 31 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | var resolver = require("resolver"), 4 | crypto = require("crypto"), 5 | redis = require("redis"); 6 | 7 | module.exports.createResolver = function(config){ 8 | var resolver = new Resolver(config); 9 | return resolver; 10 | }; 11 | 12 | function Resolver(config){ 13 | config = config || {}; 14 | this.redisClient = redis.createClient(config.redis.port, config.redis.host); 15 | this.db = config.redis.db; 16 | this.userAgent = config.userAgent || "Resolver"; 17 | this.cacheTTL = config.cacheTTL || 30 * 24 * 3600; 18 | this.debug = config.debug; 19 | } 20 | 21 | Resolver.prototype.close = function(){ 22 | this.redisClient.end(); 23 | }; 24 | 25 | Resolver.prototype.resolve = function(url, callback){ 26 | this.resolveRedis(url, (function(err, resolvedUrl){ 27 | if(err){ 28 | if(this.debug){ 29 | console.log("Redis error for %s", url); 30 | console.log(err); 31 | } 32 | } 33 | 34 | if(resolvedUrl){ 35 | return callback(null, resolvedUrl, true); 36 | } 37 | 38 | this.resolveUrl(url, (function(err, resolvedUrl){ 39 | if(err){ 40 | if(this.debug){ 41 | console.log("Resolver error for %s", url); 42 | console.log(err); 43 | console.log(err.stack); 44 | } 45 | } 46 | return callback(null, resolvedUrl || false, false); 47 | }).bind(this)); 48 | }).bind(this)); 49 | }; 50 | 51 | Resolver.prototype.resolveRedis = function(url, callback){ 52 | var key = "l~" + crypto.createHash("md5").update(url).digest("hex"); 53 | this.redisClient.multi(). 54 | select(this.db). 55 | get(key). 56 | expire(key, this.cacheTTL). 57 | exec((function(err, replies){ 58 | if(err){ 59 | return callback(err); 60 | } 61 | return callback(null, replies && replies[1] || false); 62 | }).bind(this)); 63 | }; 64 | 65 | Resolver.prototype.resolveUrl = function(url, callback){ 66 | resolver.resolve(url, { 67 | removeParams: [/^utm_/, "ref", "rsscount"], 68 | userAgent: this.userAgent 69 | }, (function(err, resolvedUrl){ 70 | if(err){ 71 | return callback(err); 72 | } 73 | 74 | if(!resolvedUrl){ 75 | return callback(null, false); 76 | } 77 | 78 | var key = crypto.createHash("md5").update(url).digest("hex"); 79 | this.redisClient.multi(). 80 | select(this.db). 81 | set("l~" + key, resolvedUrl). 82 | expire("l~" + key, this.cacheTTL). 83 | exec((function(){ 84 | return callback(null, resolvedUrl); 85 | }).bind(this)); 86 | }).bind(this)); 87 | }; 88 | -------------------------------------------------------------------------------- /test/resolver.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | var redis = require("redis"), 4 | createResolver = require("../index").createResolver, 5 | http = require("http"), 6 | urllib = require("url"); 7 | 8 | var REDIS_HOST = "localhost"; 9 | var REDIS_PORT = false; 10 | var REDIS_DB = 11; 11 | 12 | var HTTP_PORT = 1367; 13 | 14 | module.exports = { 15 | setUp: function(next){ 16 | 17 | var redisClient = this.redisClient = redis.createClient(REDIS_PORT, REDIS_HOST); 18 | 19 | this.server = http.createServer(function (req, res) { 20 | var url = urllib.parse(req.url, true, true), 21 | headers; 22 | 23 | if(url.pathname.match(/^\/direct/)){ 24 | res.writeHead(200, {'Content-Type': 'text/plain'}); 25 | res.end('Hello World\n'); 26 | }else if(url.pathname == "/status"){ 27 | headers = {'Content-Type': 'text/plain'}; 28 | if(url.query.location){ 29 | headers.location = url.query.location; 30 | } 31 | res.writeHead(url.query.status, headers); 32 | res.end(url.query.message); 33 | }else if(url.pathname == "/ua"){ 34 | headers = { 35 | 'Content-Type': 'text/plain', 36 | location: "http://127.0.0.1:" + HTTP_PORT + "/direct/ua?ua=" + encodeURIComponent(req.headers["user-agent"]) 37 | }; 38 | 39 | res.writeHead(301, headers); 40 | res.end(); 41 | }else{ 42 | res.writeHead(404, {'Content-Type': 'text/plain'}); 43 | res.end("Not found"); 44 | } 45 | }); 46 | 47 | this.server.listen(HTTP_PORT, function(){ 48 | redisClient.multi(). 49 | select(REDIS_DB). 50 | flushdb(). 51 | exec(next); 52 | }); 53 | }, 54 | tearDown: function(next){ 55 | var redisClient = this.redisClient; 56 | this.server.close(function(){ 57 | redisClient.multi(). 58 | select(REDIS_DB). 59 | flushdb(). 60 | exec(function(){ 61 | redisClient.end(); 62 | next(); 63 | }); 64 | }); 65 | }, 66 | "Exact match": function(test){ 67 | var resolver = createResolver({ 68 | redis: { 69 | host: REDIS_HOST, 70 | port: REDIS_PORT, 71 | db: REDIS_DB 72 | }, 73 | userAgent: "testResolver" 74 | }), 75 | 76 | url = "http://127.0.0.1:" + HTTP_PORT+ "/direct?aaaa", 77 | expected = url; 78 | 79 | resolver.resolve(url, function(err, resolved, cached){ 80 | test.ifError(err); 81 | test.equal(resolved, expected); 82 | test.ok(!cached); 83 | resolver.resolve(url, function(err, resolved, cached){ 84 | test.ifError(err); 85 | test.equal(resolved, expected); 86 | test.ok(cached); 87 | resolver.close(); 88 | test.done(); 89 | }); 90 | }); 91 | }, 92 | 93 | "301": function(test){ 94 | var resolver = createResolver({ 95 | redis: { 96 | host: REDIS_HOST, 97 | port: REDIS_PORT, 98 | db: REDIS_DB 99 | }, 100 | userAgent: "testResolver" 101 | }), 102 | 103 | expected = "http://127.0.0.1:" + HTTP_PORT+ "/direct/302", 104 | url = "http://127.0.0.1:" + HTTP_PORT+ "/status?status=301&location="+encodeURIComponent(expected); 105 | 106 | resolver.resolve(url, function(err, resolved, cached){ 107 | test.ifError(err); 108 | test.equal(resolved, expected); 109 | test.ok(!cached); 110 | resolver.resolve(url, function(err, resolved, cached){ 111 | test.ifError(err); 112 | test.equal(resolved, expected); 113 | test.ok(cached); 114 | resolver.close(); 115 | test.done(); 116 | }); 117 | }); 118 | }, 119 | 120 | "User-Agent": function(test){ 121 | var userAgent = "mytest", 122 | resolver = createResolver({ 123 | redis: { 124 | host: REDIS_HOST, 125 | port: REDIS_PORT, 126 | db: REDIS_DB 127 | }, 128 | userAgent: userAgent 129 | }), 130 | 131 | expected = "http://127.0.0.1:" + HTTP_PORT+ "/direct/ua?ua=" + encodeURIComponent(userAgent), 132 | url = "http://127.0.0.1:" + HTTP_PORT+ "/ua"; 133 | 134 | resolver.resolve(url, function(err, resolved, cached){ 135 | test.ifError(err); 136 | test.equal(resolved, expected); 137 | test.ok(!cached); 138 | resolver.resolve(url, function(err, resolved, cached){ 139 | test.ifError(err); 140 | test.equal(resolved, expected); 141 | test.ok(cached); 142 | resolver.close(); 143 | test.done(); 144 | }); 145 | }); 146 | }, 147 | 148 | "404": function(test){ 149 | var resolver = createResolver({ 150 | redis: { 151 | host: REDIS_HOST, 152 | port: REDIS_PORT, 153 | db: REDIS_DB 154 | }, 155 | userAgent: "testResolver" 156 | }), 157 | 158 | url = "http://127.0.0.1:" + HTTP_PORT+ "/404"; 159 | 160 | resolver.resolve(url, function(err, resolved, cached){ 161 | test.ifError(err); 162 | test.ok(!resolved); 163 | test.ok(!cached); 164 | resolver.resolve(url, function(err, resolved, cached){ 165 | test.ifError(err); 166 | test.ok(!resolved); 167 | test.ok(!cached); 168 | resolver.close(); 169 | test.done(); 170 | }); 171 | }); 172 | }, 173 | 174 | "non-resolving": function(test){ 175 | var resolver = createResolver({ 176 | redis: { 177 | host: REDIS_HOST, 178 | port: REDIS_PORT, 179 | db: REDIS_DB 180 | }, 181 | userAgent: "testResolver" 182 | }), 183 | 184 | url = "http://non-exitant-server/"; 185 | 186 | resolver.resolve(url, function(err, resolved, cached){ 187 | test.ifError(err); 188 | test.ok(!resolved); 189 | test.ok(!cached); 190 | resolver.resolve(url, function(err, resolved, cached){ 191 | test.ifError(err); 192 | test.ok(!resolved); 193 | test.ok(!cached); 194 | resolver.close(); 195 | test.done(); 196 | }); 197 | }); 198 | }, 199 | 200 | "Set Expire TTL": function(test){ 201 | var resolver = createResolver({ 202 | redis: { 203 | host: REDIS_HOST, 204 | port: REDIS_PORT, 205 | db: REDIS_DB 206 | }, 207 | userAgent: "testResolver", 208 | cacheTTL: 2 209 | }), 210 | 211 | url = "http://127.0.0.1:" + HTTP_PORT+ "/direct?aaaa", 212 | expected = url; 213 | 214 | resolver.resolve(url, function(err, resolved, cached){ 215 | test.ifError(err); 216 | test.equal(resolved, expected); 217 | test.ok(!cached); 218 | setTimeout(function(){ 219 | resolver.resolve(url, function(err, resolved, cached){ 220 | test.ifError(err); 221 | test.equal(resolved, expected); 222 | test.ok(cached); 223 | setTimeout(function(){ 224 | resolver.resolve(url, function(err, resolved, cached){ 225 | test.ifError(err); 226 | test.equal(resolved, expected); 227 | test.ok(!cached); 228 | resolver.close(); 229 | test.done(); 230 | }); 231 | }, 3000); 232 | 233 | }); 234 | }, 1000); 235 | }); 236 | }, 237 | "Remove unneeded query params": function(test){ 238 | var resolver = createResolver({ 239 | redis: { 240 | host: REDIS_HOST, 241 | port: REDIS_PORT, 242 | db: REDIS_DB 243 | }, 244 | userAgent: "testResolver" 245 | }), 246 | 247 | url = "http://127.0.0.1:" + HTTP_PORT+ "/direct?utm_test?=1&yep=2&rsscount=4", 248 | expected = "http://127.0.0.1:" + HTTP_PORT+ "/direct?yep=2"; 249 | 250 | resolver.resolve(url, function(err, resolved, cached){ 251 | test.ifError(err); 252 | test.equal(resolved, expected); 253 | test.ok(!cached); 254 | resolver.resolve(url, function(err, resolved, cached){ 255 | test.ifError(err); 256 | test.equal(resolved, expected); 257 | test.ok(cached); 258 | resolver.close(); 259 | test.done(); 260 | }); 261 | }); 262 | } 263 | }; 264 | --------------------------------------------------------------------------------