├── .eslintrc.js ├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── UPGRADE.md ├── index.js ├── lib ├── normalizer │ └── default.js └── repo │ └── default.js ├── normalizer.js ├── package.json ├── repo.js └── test ├── defaultnormalizer.test.js ├── defaultrepo.test.js ├── fixtures ├── normalizer.json └── test_case.json └── integration └── test.js /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | "env": { 3 | "es6": true, 4 | "node": true, 5 | "mocha": true, 6 | 7 | }, 8 | "extends": "eslint:recommended", 9 | "parserOptions": { 10 | "sourceType": "module" 11 | }, 12 | "rules": { 13 | "indent": [ 14 | "error", 15 | "tab" 16 | ], 17 | "linebreak-style": [ 18 | "error", 19 | "unix" 20 | ], 21 | "quotes": [ 22 | "error", 23 | "single" 24 | ], 25 | "semi": [ 26 | "error", 27 | "always" 28 | ], 29 | "no-console":[ 30 | "off" 31 | ] 32 | } 33 | }; 34 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Unix tmp file 2 | *~ 3 | 4 | # Logs 5 | logs 6 | *.log 7 | 8 | # Runtime data 9 | pids 10 | *.pid 11 | *.seed 12 | 13 | # Directory for instrumented libs generated by jscoverage/JSCover 14 | lib-cov 15 | 16 | # Coverage directory used by tools like istanbul 17 | coverage 18 | 19 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 20 | .grunt 21 | 22 | # node-waf configuration 23 | .lock-wscript 24 | 25 | # Compiled binary addons (http://nodejs.org/api/addons.html) 26 | build/Release 27 | 28 | # Dependency directory 29 | # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git 30 | node_modules 31 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - 6 4 | - 8 5 | - 10 6 | os: 7 | - linux 8 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Mike Chen 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![NPM](https://nodei.co/npm/seenreq.png?downloads=true&downloadRank=true&stars=true)](https://nodei.co/npm/seenreq/) 2 | 3 | [![build status](https://secure.travis-ci.org/mike442144/seenreq.png)](https://travis-ci.org/mike442144/seenreq) 4 | [![Dependency Status](https://david-dm.org/mike442144/seenreq/status.svg)](https://david-dm.org/mike442144/seenreq) 5 | [![NPM download][download-image]][download-url] 6 | [![NPM quality][quality-image]][quality-url] 7 | 8 | [quality-image]: http://npm.packagequality.com/shield/seenreq.svg?style=flat-square 9 | [quality-url]: http://packagequality.com/#?package=seenreq 10 | [download-image]: https://img.shields.io/npm/dm/seenreq.svg?style=flat-square 11 | [download-url]: https://npmjs.org/package/seenreq 12 | 13 | # seenreq 14 | A library to test if a url/request is crawled, usually used in a web crawler. Compatible with [request](https://github.com/request/request) and [node-crawler](https://github.com/bda-research/node-crawler). The 1.x or newer version has quite different APIs and is not compatible with 0.x versions. Please read the [upgrade guide](./UPGRADE.md) document. 15 | 16 | # Table of Contents 17 | 18 | * [Quick Start](#quick-start) 19 | * [Installation](#installation) 20 | * [Basic Usage](#basic-usage) 21 | * [Use Redis](#use-redis) 22 | * [Use Mongodb](#use-mongodb) 23 | * [Class:seenreq](#classseenreq) 24 | * [seen.initialize()](#seeninitialize) 25 | * [seen.normalize(uri|option[,options])](#seennormalizeurioptionoptions) 26 | * [seen.exists(uri|option|array[,options])](#seenexistsurioptionarrayoptions) 27 | * [seen.dispose()](#seen_dispose) 28 | * [Options](#options) 29 | 30 | ## Quick Start 31 | 32 | ### Installation 33 | 34 | $ npm install seenreq --save 35 | 36 | ### Basic Usage 37 | 38 | ```javascript 39 | const seenreq = require('seenreq') 40 | , seen = new seenreq(); 41 | 42 | //url to be normalized 43 | let url = "http://www.GOOGLE.com"; 44 | console.log(seen.normalize(url));//{ sign: "GET http://www.google.com/\r\n", options: {} } 45 | 46 | //request options to be normalized 47 | let option = { 48 | uri: 'http://www.GOOGLE.com', 49 | rupdate: false 50 | }; 51 | 52 | console.log(seen.normalize(option));//{sign: "GET http://www.google.com/\r\n", options:{rupdate: false} } 53 | 54 | seen.initialize().then(()=>{ 55 | return seen.exists(url); 56 | }).then( (rst) => { 57 | console.log(rst[0]);//false if ask for a `request` never see 58 | return seen.exists(opt); 59 | }).then( (rst) => { 60 | console.log(rst[0]);//true if got same `request` 61 | }).catch(e){ 62 | console.error(e); 63 | }; 64 | ``` 65 | When you call `exists`, the module will do normalization itself first and then check if exists. 66 | 67 | ### Use Redis 68 | `seenreq` stores keys in memory by default, memory usage will soar as number of keys increases. Redis will solve this problem. Because seenreq uses `ioredis` as redis client, all `ioredis`' [options](https://github.com/luin/ioredis/blob/master/API.md) are recived and supported. You should first install: 69 | 70 | ```javascript 71 | npm install seenreq-repo-redis --save 72 | ``` 73 | and then set repo to `redis`: 74 | 75 | ```javascript 76 | const seenreq = require('seenreq') 77 | let seen = new seenreq({ 78 | repo:'redis',// use redis instead of memory 79 | host:'127.0.0.1', 80 | port:6379, 81 | clearOnQuit:false // clear redis cache or don't when calling dispose(), default true. 82 | }); 83 | 84 | seen.initialize().then(()=>{ 85 | //do stuff... 86 | }).catch(e){ 87 | console.error(e); 88 | } 89 | ``` 90 | 91 | ### Use mongodb 92 | It is similar with redis above: 93 | 94 | ```javascript 95 | npm install seenreq-repo-mongo --save 96 | ``` 97 | 98 | ```javascript 99 | const seenreq = require('seenreq') 100 | let seen = new seenreq({ 101 | repo:'mongo', 102 | url:'mongodb://xxx/seenreq', 103 | collection: 'foor' 104 | }); 105 | ``` 106 | 107 | 108 | ## Class:seenreq 109 | 110 | Instance of seenreq 111 | 112 | ### __seen.initialize()__ 113 | Initialize the repo, returns a promise. 114 | 115 | ### __seen.normalize(uri|option[,options])__ 116 | * `uri` String, `option` is Option of [request](https://github.com/request/request) or [node-crawler](https://github.com/bda-research/node-crawler) 117 | * [options](#options) 118 | 119 | Returns normalized Object: {sign,options}. 120 | 121 | ### __seen.exists(uri|option|array[,options])__ 122 | * uri|option 123 | * [options](#options) 124 | 125 | Returns a promise with an Boolean array, e.g. [true, false, true, false, false]. 126 | 127 | ### __seen.dispose()__ 128 | 129 | Dispose resources of repo. If you are using repo other than memory, like Redis you should call `dispose` to release connection. Returns a promise. 130 | 131 | ## Options 132 | 133 | * removeKeys: Array, Ignore specified keys when doing normalization. For instance, there is a `ts` property in the url like `http://www.xxx.com/index?ts=1442382602504` which is timestamp and it should be same whenever you visit. 134 | * stripFragment: Boolean, Remove the fragment at the end of the URL (Default true). 135 | * rupdate: Boolean, it is short for `repo update`. Store in repo so that `seenreq` can hit the same `req` next time (Default true). 136 | 137 | # RoadMap 138 | * add `mysql` repo to persist keys to disk. 139 | * add keys life time management. 140 | -------------------------------------------------------------------------------- /UPGRADE.md: -------------------------------------------------------------------------------- 1 | # Upgrade guide 2 | ## api changes from version below 1.0 to 1.x 3 | * `exists` are changed to node-style arguments with callback. In previous version it uses memory as default repo to store keys, you can get result by return value but you can't in new version. seenreq uses `process.nextTick` to produce asynchronous callback. So be careful to change your code to get result in callback even if you use defualt memory repo. 4 | * `normalize` return value is an object now, it looks like: `{sign:"GET http://www.google.com\r\n",options:{key1:"val"}}`, the sign is same as the returned string by normalize before. 5 | * `options.update` is changed to `options.rupdate` to avoid duplicate, so you can place `rupdate` in `request`, e.g. `{uri:"http://www.google.com", rupdate:false}`. It also takes effect to place it in `options`. 6 | * Use `initialize` before use `exists` 7 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | 2 | 'use strict'; 3 | 4 | const URL = require('node-url-utils'); 5 | 6 | /* 7 | * 8 | * 9 | */ 10 | 11 | function seenreq(options) { 12 | let Repo = null; 13 | const Normalizers = []; 14 | 15 | options = options || {}; 16 | if(!options.repo || options.repo==='default' || options.repo==='memory'){ 17 | Repo = require('./lib/repo/default.js'); 18 | }else{ 19 | const moduleName = `seenreq-repo-${options.repo}`; 20 | try{ 21 | Repo = require(moduleName); 22 | }catch(e){ 23 | console.error(`\nCannot load module ${moduleName}, please run 'npm install ${moduleName}' and retry\n`); 24 | throw e; 25 | } 26 | } 27 | 28 | this.repo = new Repo(options); 29 | 30 | if(!options.normalizer){ 31 | Normalizers.push(require('./lib/normalizer/default.js')); 32 | }else{ 33 | let moduleNames = null; 34 | if(typeof options.normalizer === 'string'){ 35 | moduleNames = [options.normalizer]; 36 | }else{ 37 | moduleNames = options.normalizer; 38 | } 39 | 40 | moduleNames.map(moduleName=>{ 41 | moduleName = `seenreq-nmlz-${moduleName}`; 42 | try{ 43 | Normalizers.push(require(moduleName)); 44 | }catch(e){ 45 | console.error(`Cannot load module ${moduleName}, please run 'npm install ${moduleName}' and retry`); 46 | } 47 | }); 48 | } 49 | 50 | this.normalizers = Normalizers.map(ctor => new ctor(options)); 51 | this.globalOptions = options; 52 | } 53 | 54 | /* Initialize repo 55 | * - callback 56 | * @return Promise if there is no callback 57 | */ 58 | seenreq.prototype.initialize = function(){ 59 | return this.repo.initialize(); 60 | }; 61 | 62 | /* Generate method + full uri + body string. 63 | * - req, String|Object 64 | * - [options], Object 65 | * @return, Object. e.g {sign, options} 66 | */ 67 | seenreq.prototype.normalize = function(req, options) { 68 | if(!req){ 69 | throw new Error('Argument req is required.'); 70 | } 71 | 72 | const opt = { 73 | method: 'GET', 74 | body: null 75 | }; 76 | 77 | options = Object.assign({},this.globalOptions,options); 78 | 79 | if (typeof req === 'string') { 80 | opt.uri = req; 81 | }else if(typeof req === 'object'){ 82 | Object.assign(opt, req); 83 | opt.uri = opt.uri || opt.url; 84 | } 85 | 86 | /* A normalizedRequest is an object of request with some modified keys and values */ 87 | const normalizedRequest = this.normalizers.reduce((r, cur) => cur.normalize(r,options), opt); 88 | const sign = [ 89 | [normalizedRequest.method, URL.normalize(normalizedRequest.uri, options)].join(' '), normalizedRequest.body 90 | ].join('\r\n'); 91 | 92 | const requestArgsSet = new Set(['uri','url','qs','method','headers','body','form','json','multipart','followRedirect','followAllRedirects', 'maxRedirects','encoding','pool','timeout','proxy','auth','oauth','strictSSL','jar','aws','gzip','time','tunnel','proxyHeaderWhiteList','proxyHeaderExclusiveList','localAddress','forever']); 93 | 94 | Object.keys(normalizedRequest).filter(key => !requestArgsSet.has(key) ).forEach(key=>options[key]=normalizedRequest[key]); 95 | return {sign,options}; 96 | }; 97 | 98 | seenreq.prototype.exists = function(req, options) { 99 | if(!req){ 100 | throw new Error('Argument req is required.'); 101 | } 102 | 103 | if (!(req instanceof Array)) { 104 | req = [req]; 105 | } 106 | 107 | const rs = req.map(r=>this.normalize(r,options)); 108 | return this.repo.exists(rs, options).then( rst => rst.length == 1 ? rst[0] : rst); 109 | }; 110 | 111 | seenreq.prototype.dispose = function() { 112 | return this.repo.dispose(); 113 | }; 114 | 115 | module.exports = seenreq; 116 | -------------------------------------------------------------------------------- /lib/normalizer/default.js: -------------------------------------------------------------------------------- 1 | 2 | 'use strict'; 3 | 4 | const qs = require('querystring'); 5 | const URL = require('node-url-utils'); 6 | const Normalizer = require('../../normalizer.js'); 7 | 8 | class DefaultNormalizer extends Normalizer{ 9 | constructor(options){ 10 | super(options); 11 | this.globalOptions = options || {}; 12 | } 13 | 14 | /* 15 | * Generate method + full uri + body string. 16 | * - req, Object 17 | * - [options], Object 18 | */ 19 | 20 | normalize(req, options) { 21 | options = options || {}; 22 | 23 | if (!URL.parse(req.uri).search && req.qs) { 24 | req.uri = [req.uri, qs.stringify(req.qs) ].join('?'); 25 | } 26 | 27 | if (req.method === 'POST') { 28 | if (req.json && typeof req.body === 'object') { //only support one level Object 29 | const sorted = Object.keys(req.body).map(k => [k, req.body[k]]).sort(function(a, b) { 30 | return a[0] === b[0] ? a[1] > b[1] : a[0] > b[0]; 31 | }).reduce(function(pre, cur) { 32 | pre[cur[0]] = cur[1]; 33 | return pre; 34 | }, Object.create(null)); 35 | req.body = JSON.stringify(sorted); 36 | } else if (typeof req.form === 'object') { 37 | req.body = Object.keys(req.form).map(function(k) { 38 | return [k, req.form[k]].join('='); 39 | }).sort().join('&'); 40 | } else if (typeof req.form === 'string') { 41 | req.body = req.form.split('&').sort().join('&'); 42 | } 43 | } 44 | 45 | const opts = Object.assign({},this.globalOptions, options); 46 | if (opts.stripFragment !== false) { 47 | req.uri = req.uri.replace(/#.*/g, ''); 48 | } 49 | 50 | return req; 51 | } 52 | } 53 | 54 | module.exports = DefaultNormalizer; 55 | -------------------------------------------------------------------------------- /lib/repo/default.js: -------------------------------------------------------------------------------- 1 | 2 | 'use strict'; 3 | 4 | const Repo = require('../../repo'); 5 | 6 | class MemRepo extends Repo{ 7 | constructor(){ 8 | super(); 9 | this.cache = new Set(); 10 | } 11 | 12 | getByKeys(keys){ 13 | if(!keys || !(keys instanceof Array) ) 14 | return Promise.reject(new Error('Arguments required!') ); 15 | 16 | const rst = keys.map(this.cache.has, this.cache); 17 | 18 | return Promise.resolve(rst); 19 | } 20 | 21 | setByKeys(keys){ 22 | if(!keys || !(keys instanceof Array) ) 23 | return Promise.reject(new Error('Arguments required') ); 24 | 25 | keys.forEach(this.cache.add, this.cache); 26 | return Promise.resolve(); 27 | } 28 | 29 | dispose(callback) { 30 | this.cache = null; 31 | if(callback){ 32 | callback(); 33 | }else{ 34 | return Promise.resolve(); 35 | } 36 | } 37 | } 38 | 39 | module.exports = MemRepo; 40 | -------------------------------------------------------------------------------- /normalizer.js: -------------------------------------------------------------------------------- 1 | 2 | 'use strict'; 3 | 4 | class Normalizer{ 5 | normalize(){ 6 | throw new Error('not implemented'); 7 | } 8 | } 9 | 10 | module.exports = Normalizer; 11 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "seenreq", 3 | "version": "3.0.0", 4 | "description": "A library to test if a url(request) is crawled, usually used in a web crawler. Compatible with `request` and `node-crawler`", 5 | "main": "index.js", 6 | "scripts": { 7 | "hint": "eslint ./lib/*/*.js ./test/*.js", 8 | "test": "mocha --timeout=15000 ./test/*test.js", 9 | "cover": "istanbul cover _mocha --report lcovonly -- --timeout=15000 --reporter spec test/*.test.js" 10 | }, 11 | "repository": { 12 | "type": "git", 13 | "url": "https://github.com/mike442144/seenreq" 14 | }, 15 | "keywords": [ 16 | "nodejs", 17 | "url seen test", 18 | "reomve duplicate url", 19 | "request normalize" 20 | ], 21 | "author": "Mike Chen", 22 | "license": "MIT", 23 | "bugs": { 24 | "url": "https://github.com/mike442144/seenreq/issues" 25 | }, 26 | "homepage": "https://github.com/mike442144/seenreq", 27 | "dependencies": { 28 | "node-url-utils": "^0.4.0" 29 | }, 30 | "devDependencies": { 31 | "chai": "^4.1.2", 32 | "eslint": "^5.2.0", 33 | "istanbul": "^0.4.5", 34 | "mocha": "^5.2.0", 35 | "rewire": "^4.0.1", 36 | "sinon": "^6.1.1" 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /repo.js: -------------------------------------------------------------------------------- 1 | 2 | 'use strict'; 3 | 4 | const crypto = require('crypto'); 5 | 6 | /* Repo is an abstract class 7 | * 8 | * 9 | */ 10 | class Repo{ 11 | initialize(){ 12 | return Promise.resolve(); 13 | } 14 | 15 | /* 16 | * - normalizedReq, Array 17 | * - sign, String 18 | * - options, Object, this is options for one request 19 | * - options, Object, this is options for all requests 20 | * - callback, Function 21 | * 22 | * Priority of two options : normalizedReq.options > options 23 | */ 24 | exists(normalizedReq, options){ 25 | const req = normalizedReq; 26 | const slots = {}; 27 | const uniq = []; 28 | const keysToInsert = {}; 29 | const rst = new Array(req.length); 30 | 31 | for (let i = 0; i < req.length; i++) { 32 | const reqOptions = Object.assign({},options,req[i].options); 33 | const key = this.transformKey(req[i].sign); 34 | if (key in slots) { 35 | rst[i] = true; 36 | } else { 37 | rst[i] = false; 38 | slots[key] = i; 39 | uniq.push(key); 40 | keysToInsert[key] = null; 41 | 42 | if (reqOptions.rupdate === false) { 43 | delete keysToInsert[key]; 44 | } 45 | } 46 | } 47 | 48 | return this.getByKeys(uniq).then( (result) => { 49 | const ifTruthy = (key) => key==='1' || key===1 || key==='true' || key===true ; 50 | 51 | for (let j = 0; j < uniq.length; j++) { 52 | if (ifTruthy(result[j])) { 53 | rst[slots[uniq[j]]] = true; 54 | delete keysToInsert[uniq[j]]; 55 | } else { 56 | rst[slots[uniq[j]]] = false; 57 | } 58 | } 59 | 60 | return this.setByKeys(Object.keys(keysToInsert)); 61 | }).then( () => { 62 | return rst; 63 | }); 64 | } 65 | 66 | /* 67 | * 68 | * @return Array, transformed keys 69 | * 70 | */ 71 | transformKey(key){ 72 | const hash = (str) => { 73 | const hashFn = crypto.createHash('md5'); 74 | hashFn.update(str); 75 | return hashFn.digest('hex'); 76 | }; 77 | 78 | return hash(key); 79 | } 80 | 81 | dispose(){ 82 | throw new Error('`dispose` not implemented'); 83 | } 84 | } 85 | 86 | module.exports = Repo; 87 | -------------------------------------------------------------------------------- /test/defaultnormalizer.test.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | const expect = require('chai').expect; 3 | const DefaultNormalizer = require('../lib/normalizer/default.js'); 4 | 5 | describe('default normalizer', ()=>{ 6 | let ctx = require('./fixtures/normalizer.json'); 7 | let normalizer; 8 | beforeEach(()=>{ 9 | normalizer = new DefaultNormalizer(); 10 | }); 11 | 12 | describe('constructor()', () => { 13 | it('should create an object', ()=>{ 14 | expect(normalizer.globalOptions).to.eql({}); 15 | normalizer = new DefaultNormalizer(ctx.opts[0]); 16 | expect(normalizer.globalOptions).to.eql(ctx.opts[0]); 17 | }); 18 | 19 | }); 20 | 21 | describe('normalize()', () => { 22 | it('should compose querystring from qs ', () => { 23 | let res = normalizer.normalize(ctx.reqs[0]); 24 | expect(res).to.eql({ 25 | uri: 'http://www.GOOGLE.com?q=querysomething', 26 | qs:{ 27 | q:'querysomething', 28 | } 29 | }); 30 | }); 31 | 32 | it('should use local options to override global options', () => { 33 | normalizer = new DefaultNormalizer(ctx.opts[1]); 34 | let res = normalizer.normalize(ctx.reqs[1],ctx.opts[2]); 35 | expect(res).to.eql({ 36 | uri: 'http://www.GOOGLE.com', 37 | } 38 | ); 39 | }); 40 | 41 | it('should compose querystrings for get method', () => { 42 | normalizer = new DefaultNormalizer(ctx.opts[0]); 43 | let res = normalizer.normalize(ctx.reqs[2]); 44 | expect(res).to.eql({ 45 | method: 'GET', 46 | uri: 'http://www.google.com?q=querysomething&gfe_rd=cr&ei=qg5QVYyVBcrC8Afz6ICoDw&gws_rd=ssl&safe=strict', 47 | qs:{ 48 | q: 'querysomething',gfe_rd: 'cr',ei: 'qg5QVYyVBcrC8Afz6ICoDw',gws_rd: 'ssl',safe: 'strict' 49 | } 50 | }); 51 | }); 52 | 53 | it('should convert object form to string body for post method', () => { 54 | let res = normalizer.normalize(ctx.reqs[3]); 55 | expect(res.body).to.not.an('undefined'); 56 | expect(res).to.eql({ 57 | method: 'POST', 58 | uri: 'https://github.com/logout', 59 | form: { 60 | utf8: '✓', 61 | authenticity_token: 'R1d7nfjekS+a5/h8+L2DrSy02gt7GCxRLFla5JBjwMrYQRDRrGPaTFz/tHTQKaqYfMeZIYlYMhfBrnMwDDz+cg==' 62 | }, 63 | body: 'authenticity_token=R1d7nfjekS+a5/h8+L2DrSy02gt7GCxRLFla5JBjwMrYQRDRrGPaTFz/tHTQKaqYfMeZIYlYMhfBrnMwDDz+cg==&utf8=✓' 64 | }); 65 | }); 66 | 67 | it('should convert string form to string body for post method', () => { 68 | let res = normalizer.normalize(ctx.reqs[4]); 69 | expect(res.body).to.not.an('undefined'); 70 | expect(res).to.eql({ 71 | method: 'POST', 72 | uri: 'https://github.com/logout', 73 | form: 'utf8=✓&authenticity_token=R1d7nfjekS+a5/h8+L2DrSy02gt7GCxRLFla5JBjwMrYQRDRrGPaTFz/tHTQKaqYfMeZIYlYMhfBrnMwDDz+cg==', 74 | body: 'authenticity_token=R1d7nfjekS+a5/h8+L2DrSy02gt7GCxRLFla5JBjwMrYQRDRrGPaTFz/tHTQKaqYfMeZIYlYMhfBrnMwDDz+cg==&utf8=✓' 75 | }); 76 | }); 77 | 78 | it('should normalize json body for post method', () => { 79 | let res = normalizer.normalize(ctx.reqs[5]); 80 | expect(res.body).to.not.an('undefined'); 81 | expect(res.body).to.eql(JSON.stringify({ 82 | authenticity_token: 'R1d7nfjekS+a5/h8+L2DrSy02gt7GCxRLFla5JBjwMrYQRDRrGPaTFz/tHTQKaqYfMeZIYlYMhfBrnMwDDz+cg==', 83 | utf8:'✓' 84 | })); 85 | }); 86 | }); 87 | }); -------------------------------------------------------------------------------- /test/defaultrepo.test.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | const expect = require('chai').expect; 3 | const DefaultMemRepo= require('../lib/repo/default.js'); 4 | 5 | describe('default repo', () => { 6 | let memRepo; 7 | beforeEach(()=>{ 8 | memRepo = new DefaultMemRepo(); 9 | }); 10 | 11 | describe('constructor()', ()=>{ 12 | it('should create an object and initialize cache property', ()=>{ 13 | expect(memRepo.cache).to.be.an.instanceof(Set); 14 | }); 15 | }); 16 | 17 | describe('getByKeys()', () => { 18 | it('should get cache value', (done) => { 19 | memRepo.cache = new Set(['key1','key2','key3']); 20 | memRepo.getByKeys(['key1', 'key4']).then( ([key1, key4]) => { 21 | expect(key1).to.be.true; 22 | expect(key4).to.be.false; 23 | done(); 24 | }); 25 | }); 26 | }); 27 | 28 | describe('setByKeys()', ()=>{ 29 | it('should set cache value to null', (done)=>{ 30 | memRepo.cache = new Set(['key1','key2','key3']); 31 | memRepo.setByKeys(['key2','key4']).then( () => { 32 | expect(memRepo.cache.has('key4')).to.be.true; 33 | expect(memRepo.cache.has('key2')).to.be.true; 34 | done(); 35 | }); 36 | }); 37 | }); 38 | 39 | describe('dispose()', ()=>{ 40 | it('should clear cache', (done)=>{ 41 | memRepo.cache = new Set(['key1','key2','key3']); 42 | memRepo.getByKeys(['key0', 'key3']).then( ([key0, key3]) => { 43 | expect(key0).to.be.false; 44 | expect(key3).to.be.true; 45 | return memRepo.setByKeys(['key1','key2']); 46 | }).then( () => { 47 | expect(memRepo.cache.has('key1')).to.be.true; 48 | expect(memRepo.cache.has('key2')).to.be.true; 49 | return memRepo.dispose(); 50 | }).then(() => { 51 | expect(memRepo.cache).to.be.a('null'); 52 | done(); 53 | }); 54 | }); 55 | }); 56 | }); 57 | -------------------------------------------------------------------------------- /test/fixtures/normalizer.json: -------------------------------------------------------------------------------- 1 | { 2 | "reqs": [{ 3 | "uri": "http://www.GOOGLE.com", 4 | "qs":{ 5 | "q":"querysomething" 6 | } 7 | }, 8 | { 9 | "uri":"http://www.GOOGLE.com#hashpart" 10 | }, 11 | { 12 | "method":"GET", 13 | "uri":"http://www.google.com", 14 | "qs":{ 15 | "q":"querysomething", 16 | "gfe_rd":"cr", 17 | "ei":"qg5QVYyVBcrC8Afz6ICoDw", 18 | "gws_rd":"ssl", 19 | "safe":"strict" 20 | } 21 | }, 22 | { 23 | "method":"POST", 24 | "uri":"https://github.com/logout", 25 | "form":{ 26 | "utf8" :"✓", 27 | "authenticity_token" :"R1d7nfjekS+a5/h8+L2DrSy02gt7GCxRLFla5JBjwMrYQRDRrGPaTFz/tHTQKaqYfMeZIYlYMhfBrnMwDDz+cg==" 28 | } 29 | }, 30 | { 31 | "method":"POST", 32 | "uri":"https://github.com/logout", 33 | "form":"utf8=✓&authenticity_token=R1d7nfjekS+a5/h8+L2DrSy02gt7GCxRLFla5JBjwMrYQRDRrGPaTFz/tHTQKaqYfMeZIYlYMhfBrnMwDDz+cg==" 34 | 35 | }, 36 | { 37 | "method":"POST", 38 | "uri":"https://github.com/logout", 39 | "body":{ 40 | "utf8": "✓", 41 | "authenticity_token": "R1d7nfjekS+a5/h8+L2DrSy02gt7GCxRLFla5JBjwMrYQRDRrGPaTFz/tHTQKaqYfMeZIYlYMhfBrnMwDDz+cg==" 42 | }, 43 | "json": true 44 | } 45 | ], 46 | "opts": [ 47 | {}, 48 | {"stripFragment": false}, 49 | {"stripFragment": true} 50 | ] 51 | 52 | } -------------------------------------------------------------------------------- /test/fixtures/test_case.json: -------------------------------------------------------------------------------- 1 | { 2 | "opts": [ 3 | "http://www.GOOGLE.com", 4 | { 5 | "uri": "http://www.GOOGLE.com", 6 | "rupdate": false 7 | }, 8 | { 9 | "uri": "http://baoming.xdf.cn/ShoppingCart/Handlers/getCartVoucherHandler.ashx", 10 | "method": "POST", 11 | "form": { 12 | "s": "a" 13 | }, 14 | "headers": { 15 | "Cookie": "Xdf.WebPay.V4.Cart=" 16 | }, 17 | "jQuery": false 18 | }, 19 | { 20 | "uri": "http://baoming.xdf.cn/ShoppingCart/Handlers/getCartVoucherHandler.ashx", 21 | "method": "POST", 22 | "form": { 23 | "s": "b" 24 | }, 25 | "headers": { 26 | "Cookie": "Xdf.WebPay.V4.Cart=" 27 | }, 28 | "jQuery": false 29 | }, 30 | { 31 | "uri": "http://mall.autohome.com.cn/list/0-110100-0-0-0-0-0-0-0-2.html", 32 | "rupdate": false 33 | }, 34 | "http://mall.autohome.com.cn/list/0-110100-0-0-0-0-0-0-0-2.html", 35 | { 36 | "uri": "http://mall.autohome.com.cn/list/0-110100-0-0-0-0-0-0-0-2.html", 37 | "rupdate": true 38 | }, 39 | ["http://www.twitter.com", "http://www.google.com.hk", "http://www.twitter.com"] 40 | ] 41 | } -------------------------------------------------------------------------------- /test/integration/test.js: -------------------------------------------------------------------------------- 1 | 2 | 'use strict'; 3 | 4 | const seenreq = require('../../'); 5 | const expect = require('chai').expect; 6 | 7 | describe('seenreq integration testing', ()=>{ 8 | let ctx = require('../fixtures/test_case.json'); 9 | let seen; 10 | 11 | describe('basic usage', ()=>{ 12 | beforeEach((done)=>{ 13 | seen = new seenreq(); 14 | seen.initialize().then(done()).catch( (e) => { 15 | console.error(e); 16 | done(); 17 | }); 18 | }); 19 | 20 | it('should normalize and find duplicate requests ',(done)=>{ 21 | //url to be normalized 22 | expect(seen.normalize(ctx.opts[0])).to.eql({ 23 | sign: 'GET http://www.google.com/\r\n', 24 | options: {} 25 | }); 26 | 27 | //request options to be normalized 28 | expect(seen.normalize(ctx.opts[1])).to.eql({ 29 | sign: 'GET http://www.google.com/\r\n', 30 | options: {rupdate: false} 31 | }); 32 | 33 | seen.initialize() 34 | .then( ()=> seen.exists(ctx.opts[0]) ) 35 | .then( (rst)=>{ 36 | //false if ask for a `request` never see 37 | expect(rst).to.be.false; 38 | return seen.exists(ctx.opts[1]); 39 | }).then( (rst)=>{ 40 | //true if got same `request` 41 | expect(rst).to.be.true; 42 | //true if ask for a duplicate `request` 43 | return seen.exists(ctx.opts[0]); 44 | }).then( (rst)=>{ 45 | expect(rst).to.be.true; 46 | return seen.exists(ctx.opts[2]); 47 | }).then( (rst)=>{ 48 | expect(rst).to.be.false; 49 | return seen.exists(ctx.opts[3]); 50 | }).then( (rst)=>{ 51 | expect(rst).to.be.false; 52 | return seen.exists(ctx.opts[4]); 53 | }).then( (rst)=>{ 54 | expect(rst).to.be.false; 55 | return seen.exists(ctx.opts[6]); 56 | }).then( (rst)=>{ 57 | expect(rst).to.be.false; 58 | return seen.exists(ctx.opts[5]); 59 | }).then( (rst)=>{ 60 | expect(rst).to.be.true; 61 | return seen.exists(ctx.opts[4]); 62 | }).then((rst) => { 63 | expect(rst).to.be.true; 64 | return seen.exists([ctx.opts[0], ctx.opts[3], ctx.opts[6], ctx.opts[4]]); 65 | }).then((rst)=>{ 66 | expect(rst).to.eqls([true, true, true, true]); 67 | done(); 68 | }).catch(function (e){ 69 | console.error(e); 70 | expect(e).to.be.false; 71 | done(); 72 | }); 73 | }); 74 | }); 75 | }); 76 | --------------------------------------------------------------------------------