├── .gitignore ├── .npmignore ├── .babelrc ├── .eslintrc.json ├── src ├── connectors │ ├── connectorARINrr.js │ ├── connectorRIPE.js │ ├── connectorAFRINIC.js │ ├── connectorLACNICrr.js │ ├── connectorLACNICrir.js │ ├── connectorARIN.js │ ├── connectorLACNIC.js │ ├── connectorAPNIC.js │ ├── connector.js │ └── connectorARINrir.js └── index.js ├── LICENSE ├── README.md └── package.json /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | node_modules/ 3 | .DS_Store 4 | .npmrc 5 | .env 6 | .cache/ 7 | dist/ -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | node_modules/ 3 | .DS_Store 4 | .npmrc 5 | .env 6 | src/ 7 | .cache/ 8 | .gitignore 9 | .npmignore 10 | .babelrc 11 | .eslintrc.json 12 | -------------------------------------------------------------------------------- /.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": [ 3 | "@babel/preset-env" 4 | ], 5 | "plugins": [ 6 | "@babel/plugin-proposal-class-properties", 7 | "@babel/plugin-transform-async-to-generator", 8 | "@babel/plugin-proposal-object-rest-spread" 9 | ], 10 | 11 | "ignore": [ 12 | "./node_modules", 13 | "./tests", 14 | "./logs", 15 | "./dist" 16 | ] 17 | } 18 | -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "parser": "babel-eslint", 3 | "ecmaFeatures": { 4 | "modules": true, 5 | "spread" : true, 6 | "restParams" : true 7 | }, 8 | "env" : { 9 | "browser" : false, 10 | "node" : true, 11 | "es6" : true, 12 | "mocha": true 13 | }, 14 | "parserOptions": { 15 | "ecmaVersion": 6, 16 | "sourceType": "module", 17 | "ecmaFeatures": { 18 | "jsx": false 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/connectors/connectorARINrr.js: -------------------------------------------------------------------------------- 1 | import Connector from "./connector"; 2 | import fs from "fs"; 3 | 4 | export default class ConnectorARINrr extends Connector { 5 | constructor(params) { 6 | super(params) 7 | 8 | this.connectorName = "arin-rr"; 9 | this.cacheDir += this.connectorName + "/"; 10 | this.dumpUrl = this.params.dumpUrl || "https://ftp.arin.net/pub/rr/arin.db.gz"; 11 | this.cacheFile = [this.cacheDir, "arin.db.gz"].join("/").replace("//", "/"); 12 | this.daysWhoisCache = this.params.defaultCacheDays || 2; 13 | 14 | if (!fs.existsSync(this.cacheDir)) { 15 | fs.mkdirSync(this.cacheDir, { recursive: true }); 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/connectors/connectorRIPE.js: -------------------------------------------------------------------------------- 1 | import Connector from "./connector"; 2 | import fs from "fs"; 3 | 4 | export default class ConnectorRIPE extends Connector { 5 | constructor(params) { 6 | super(params) 7 | 8 | this.connectorName = "ripe"; 9 | this.cacheDir += this.connectorName + "/"; 10 | this.dumpUrl = this.params.dumpUrl || "https://ftp.ripe.net/ripe/dbase/ripe.db.gz"; 11 | this.cacheFile = [this.cacheDir, "ripe.db.gz"].join("/").replace("//", "/"); 12 | 13 | this.daysWhoisCache = this.params.defaultCacheDays || 2; 14 | 15 | if (!fs.existsSync(this.cacheDir)) { 16 | fs.mkdirSync(this.cacheDir, { recursive: true }); 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/connectors/connectorAFRINIC.js: -------------------------------------------------------------------------------- 1 | import Connector from "./connector"; 2 | import fs from "fs"; 3 | 4 | export default class ConnectorAFRINIC extends Connector { 5 | constructor(params) { 6 | super(params) 7 | 8 | this.connectorName = "afrinic"; 9 | this.cacheDir += this.connectorName + "/"; 10 | this.dumpUrl = this.params.dumpUrl || "http://ftp.afrinic.net/dbase/afrinic.db.gz"; 11 | this.cacheFile = [this.cacheDir, "afrinic.db.gz"].join("/").replace("//", "/"); 12 | this.daysWhoisCache = this.params.defaultCacheDays || 2; 13 | 14 | if (!fs.existsSync(this.cacheDir)) { 15 | fs.mkdirSync(this.cacheDir, { recursive: true }); 16 | } 17 | 18 | } 19 | } -------------------------------------------------------------------------------- /src/connectors/connectorLACNICrr.js: -------------------------------------------------------------------------------- 1 | import Connector from "./connector"; 2 | import fs from "fs"; 3 | 4 | export default class ConnectorLACNICrr extends Connector { 5 | constructor(params) { 6 | super(params) 7 | 8 | this.connectorName = "lacnic-rr"; 9 | this.cacheDir += this.connectorName + "/"; 10 | this.dumpUrl = this.params.dumpUrl || "http://ftp.lacnic.net/lacnic/irr/lacnic.db.gz"; 11 | this.cacheFile = [this.cacheDir, "lacnic-rr.db.gz"].join("/").replace("//", "/"); 12 | this.daysWhoisCache = this.params.defaultCacheDays || 2; 13 | 14 | if (!fs.existsSync(this.cacheDir)) { 15 | fs.mkdirSync(this.cacheDir, { recursive: true }); 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/connectors/connectorLACNICrir.js: -------------------------------------------------------------------------------- 1 | import Connector from "./connector"; 2 | import fs from "fs"; 3 | 4 | export default class ConnectorLACNICrir extends Connector { 5 | constructor(params) { 6 | super(params) 7 | 8 | this.connectorName = "lacnic-rir"; 9 | this.cacheDir += this.connectorName + "/"; 10 | this.dumpUrl = this.params.dumpUrl || "http://ftp.lacnic.net/lacnic/dbase/lacnic.db.gz"; 11 | this.cacheFile = [this.cacheDir, "lacnic-rir.db.gz"].join("/").replace("//", "/"); 12 | this.daysWhoisCache = this.params.defaultCacheDays || 2; 13 | 14 | if (!fs.existsSync(this.cacheDir)) { 15 | fs.mkdirSync(this.cacheDir, { recursive: true }); 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/connectors/connectorARIN.js: -------------------------------------------------------------------------------- 1 | import Connector from "./connector"; 2 | import ConnectorARINrir from "./connectorARINrir"; 3 | import ConnectorARINrr from "./connectorARINrr"; 4 | import batchPromises from "batch-promises"; 5 | 6 | export default class ConnectorARIN extends Connector { 7 | constructor(params) { 8 | super(params); 9 | 10 | this.rir = new ConnectorARINrir(params); 11 | this.rr = new ConnectorARINrr(params); 12 | 13 | }; 14 | 15 | _getCorrectConnector = (type, filterFunction, fields, forEachFunction) => { 16 | if (["inetnum", "inet6num"].includes(type)) { 17 | return this.rir.getObjects([type], filterFunction, fields, forEachFunction); 18 | } else { 19 | return this.rr.getObjects([type], filterFunction, fields, forEachFunction); 20 | } 21 | }; 22 | 23 | 24 | getObjects = (types, filterFunction, fields, forEachFunction) => { 25 | fields = fields || []; 26 | const objects = []; 27 | 28 | return batchPromises(1, types, type => { 29 | return this._getCorrectConnector(type, filterFunction, fields, forEachFunction) 30 | .then(data => objects.push(data)) 31 | }) 32 | .then(() => objects.flat()); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /src/connectors/connectorLACNIC.js: -------------------------------------------------------------------------------- 1 | import Connector from "./connector"; 2 | import ConnectorLACNICrir from "./connectorLACNICrir"; 3 | import ConnectorLACNICrr from "./connectorLACNICrr"; 4 | import batchPromises from "batch-promises"; 5 | 6 | export default class ConnectorLACNIC extends Connector { 7 | constructor(params) { 8 | super(params) 9 | 10 | this.rir = new ConnectorLACNICrir(params); 11 | this.rr = new ConnectorLACNICrr(params); 12 | } 13 | 14 | 15 | _getCorrectConnector = (type, filterFunction, fields, forEachFunction) => { 16 | if (["route", "route6", "as-set", "aut-num", "mntner", "person"].includes(type)) { 17 | return this.rr.getObjects([type], filterFunction, fields, forEachFunction); 18 | } else { 19 | return this.rir.getObjects([type], filterFunction, fields, forEachFunction); 20 | } 21 | }; 22 | 23 | 24 | getObjects = (types, filterFunction, fields, forEachFunction) => { 25 | fields = fields || []; 26 | const objects = []; 27 | 28 | return batchPromises(1, types, type => { 29 | return this._getCorrectConnector(type, filterFunction, fields, forEachFunction) 30 | .then(data => objects.push(data)) 31 | }) 32 | .then(() => objects.flat()); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2020, Massimo Candela 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | 2. Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | 3. Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | import ConnectorRIPE from "./connectors/connectorRIPE"; 2 | import ConnectorAFRINIC from "./connectors/connectorAFRINIC"; 3 | import ConnectorLACNIC from "./connectors/connectorLACNIC"; 4 | import ConnectorAPNIC from "./connectors/connectorAPNIC"; 5 | import ConnectorARIN from "./connectors/connectorARIN"; 6 | import batchPromises from 'batch-promises'; 7 | 8 | export default class WhoisParser { 9 | constructor(params) { 10 | this.params = params || { 11 | userAgent: "bulk-whois-parser" 12 | }; 13 | this.cacheDir = this.params.cacheDir || ".cache/"; 14 | 15 | this.connectors = {}; 16 | 17 | const connectors = { 18 | "ripe": ConnectorRIPE, 19 | "afrinic": ConnectorAFRINIC, 20 | "apnic": ConnectorAPNIC, 21 | "arin": ConnectorARIN, 22 | "lacnic": ConnectorLACNIC 23 | }; 24 | 25 | if (this.params.repos) { 26 | for (let repo of this.params.repos) { 27 | this.connectors[repo] = new connectors[repo](this.params); 28 | } 29 | } else { 30 | for (let repo in connectors) { 31 | this.connectors[repo] = new connectors[repo](this.params); 32 | } 33 | } 34 | 35 | }; 36 | 37 | _getObjects = (types, filterFunction, fields, forEachFunction) => { 38 | fields = fields || []; 39 | 40 | const objects = []; 41 | return batchPromises(3, Object.keys(this.connectors), connector => { 42 | return this.connectors[connector] 43 | .getObjects(types, filterFunction, fields, forEachFunction) 44 | .then(results => objects.concat(results.flat())) 45 | .catch(console.log) 46 | }); 47 | }; 48 | 49 | getObjects = (types, filterFunction, fields) => { 50 | return this._getObjects(types, filterFunction, fields, null); 51 | }; 52 | 53 | forEachObject = (types, filterFunction, fields, forEachFunction) => { 54 | return this._getObjects(types, filterFunction, fields, forEachFunction) 55 | .then(() => null); 56 | }; 57 | } 58 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # bulk-whois-parser 2 | Bulk whois data parser. 3 | It automatically downloads and caches bulk whois data. 4 | It parses and filters the data, and returns it as JavaScript objects. It also removes some format differences across the various whois servers. 5 | 6 | Install: 7 | 8 | ```bash 9 | npm install bulk-whois-parser 10 | ``` 11 | 12 | Import: 13 | 14 | ```js 15 | import WhoisParser from "bulk-whois-parser"; 16 | ``` 17 | 18 | 19 | Usage example: 20 | 21 | ```javascript 22 | 23 | const filterFunction = (object) => { 24 | // A function that returns true or false, used to filter the results 25 | // Make this function as selective as possible, since the amount of whois data 26 | // can be overwhelming. 27 | } 28 | 29 | // The fields you want to see in the object. Use null to get all the fields 30 | const fields = ["inetnum", "inet6num", "remarks"]; 31 | 32 | 33 | new WhoisParser({ repos: ["ripe", "lacnic", "apnic", "afrinic", "arin"] }) 34 | .getObjects(["inetnum", "inet6num"], filterFunction, fields) 35 | .then(objects => { 36 | // Do something with the objects (array) 37 | }); 38 | ``` 39 | 40 | > You don't have to pass any file or anything, the library will automatically download the data. 41 | 42 | Result example: 43 | 44 | ```js 45 | [ 46 | { 47 | inet6num: '2001:67c:370::/48', 48 | netname: 'ietf-ipv6-meeting-network', 49 | country: 'CH', 50 | org: 'ORG-IS136-RIPE', 51 | 'admin-c': 'DUMY-RIPE', 52 | 'tech-c': 'DUMY-RIPE', 53 | status: 'ASSIGNED PI', 54 | notify: 'ripedb-updates@noc.ietf.org', 55 | 'mnt-by': [ 'RIPE-NCC-END-MNT', 'IETF-MNT', 'netnod-mnt' ], 56 | 'mnt-routes': 'IETF-MNT', 57 | 'mnt-domains': 'ietf-MNT', 58 | created: '2010-11-18T17:16:42Z', 59 | 'last-modified': '2020-09-14T13:46:23Z', 60 | source: 'RIPE', 61 | 'sponsoring-org': 'ORG-NIE1-RIPE', 62 | remarks: [ 63 | 'Geofeed https://noc.ietf.org/geo/google.csv', 64 | '****************************', 65 | '* THIS OBJECT IS MODIFIED', 66 | '* Please note that all data that is generally regarded as personal', 67 | '* data has been removed from this object.', 68 | '* To view the original object, please query the RIPE Database at:', 69 | '* http://www.ripe.net/whois', 70 | '****************************' 71 | ] 72 | }, 73 | ... 74 | ] 75 | ``` 76 | 77 | Enjoy! 78 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bulk-whois-parser", 3 | "version": "1.4.8", 4 | "description": "A lib to parse bulk whois data", 5 | "main": "dist/index.js", 6 | "bin": { 7 | "bulk-whois-parser": "dist/index.js" 8 | }, 9 | "scripts": { 10 | "babel": "./node_modules/.bin/babel", 11 | "release": "dotenv release-it", 12 | "compile": "babel src -d dist", 13 | "serve": "babel-node src/index.js", 14 | "test": "echo \"Error: no test specified\" && exit 1" 15 | }, 16 | "repository": { 17 | "type": "git", 18 | "url": "git+https://github.com/massimocandela/bulk-whois-parser.git" 19 | }, 20 | "keywords": [ 21 | "rpsl", 22 | "whois", 23 | "bulk", 24 | "parser" 25 | ], 26 | "author": { 27 | "name": "Massimo Candela", 28 | "url": "https://massimocandela.com" 29 | }, 30 | "license": "BSD-3-Clause", 31 | "bugs": { 32 | "url": "https://github.com/massimocandela/bulk-whois-parser/issues" 33 | }, 34 | "homepage": "https://github.com/massimocandela/bulk-whois-parser#readme", 35 | "devDependencies": { 36 | "@babel/cli": "^7.23.9", 37 | "@babel/core": "^7.24.0", 38 | "@babel/node": "^7.23.9", 39 | "@babel/plugin-proposal-class-properties": "^7.18.6", 40 | "@babel/plugin-proposal-object-rest-spread": "^7.20.7", 41 | "@babel/preset-env": "^7.24.0", 42 | "dotenv-cli": "^7.4.0", 43 | "release-it": "^17.1.1" 44 | }, 45 | "dependencies": { 46 | "batch-promises": "0.0.3", 47 | "cli-progress": "^3.12.0", 48 | "ip-sub": "^1.5.2", 49 | "md5": "^2.3.0", 50 | "moment": "^2.30.1", 51 | "readline": "^1.3.0", 52 | "whois": "^2.14.0" 53 | }, 54 | "release-it": { 55 | "hooks": { 56 | "before:init": [ 57 | "npm ci" 58 | ], 59 | "after:bump": "npm run compile", 60 | "after:release": [ 61 | "echo Successfully released ${name} v${version} to ${repo.repository}.", 62 | "rm -r dist/" 63 | ] 64 | }, 65 | "git": { 66 | "changelog": "git log --pretty=format:\"* %s (%h)\" ${from}...${to}", 67 | "requireCleanWorkingDir": true, 68 | "requireBranch": "main", 69 | "requireUpstream": true, 70 | "requireCommits": false, 71 | "addUntrackedFiles": false, 72 | "commit": true, 73 | "commitMessage": "Release v${version}", 74 | "commitArgs": [], 75 | "tag": true, 76 | "tagName": null, 77 | "tagAnnotation": "Release v${version}", 78 | "tagArgs": [], 79 | "push": true, 80 | "pushArgs": [ 81 | "--follow-tags" 82 | ], 83 | "pushRepo": "" 84 | }, 85 | "gitlab": { 86 | "release": false 87 | }, 88 | "npm": { 89 | "publish": true 90 | }, 91 | "github": { 92 | "release": true, 93 | "releaseName": "v${version}", 94 | "tokenRef": "GITHUB_TOKEN", 95 | "origin": null, 96 | "skipChecks": false 97 | } 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /src/connectors/connectorAPNIC.js: -------------------------------------------------------------------------------- 1 | import Connector from "./connector"; 2 | import fs from "fs"; 3 | import moment from "moment"; 4 | 5 | export default class ConnectorAPNIC extends Connector { 6 | constructor(params) { 7 | super(params) 8 | 9 | this.connectorName = "apnic"; 10 | this.cacheDir += this.connectorName + "/"; 11 | this.dumpUrls = this.params.dumpUrls || [ 12 | "http://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz", 13 | "http://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz", 14 | "http://ftp.apnic.net/apnic/whois/apnic.db.as-block.gz", 15 | "http://ftp.apnic.net/apnic/whois/apnic.db.as-set.gz", 16 | "http://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz", 17 | "http://ftp.apnic.net/apnic/whois/apnic.db.filter-set.gz", 18 | "http://ftp.apnic.net/apnic/whois/apnic.db.inet-rtr.gz", 19 | "http://ftp.apnic.net/apnic/whois/apnic.db.irt.gz", 20 | "http://ftp.apnic.net/apnic/whois/apnic.db.key-cert.gz", 21 | "http://ftp.apnic.net/apnic/whois/apnic.db.mntner.gz", 22 | "http://ftp.apnic.net/apnic/whois/apnic.db.organisation.gz", 23 | "http://ftp.apnic.net/apnic/whois/apnic.db.peering-set.gz", 24 | "http://ftp.apnic.net/apnic/whois/apnic.db.rtr-set.gz", 25 | "http://ftp.apnic.net/apnic/whois/apnic.db.role.gz", 26 | "http://ftp.apnic.net/apnic/whois/apnic.db.route-set.gz", 27 | "http://ftp.apnic.net/apnic/whois/apnic.db.route.gz", 28 | "http://ftp.apnic.net/apnic/whois/apnic.db.route6.gz", 29 | "http://ftp.apnic.net/apnic/whois/apnic.db.rtr-set.gz", 30 | ]; 31 | 32 | this.cacheFiles = this.dumpUrls.map(this.getCacheFileName); 33 | this.daysWhoisCache = this.params.defaultCacheDays || 2; 34 | 35 | if (!fs.existsSync(this.cacheDir)) { 36 | fs.mkdirSync(this.cacheDir, { recursive: true }); 37 | } 38 | } 39 | 40 | _getDumpFile = (url) => { 41 | const cacheFile = this.getCacheFileName(url); 42 | 43 | return this._downloadFile(url, cacheFile); 44 | } 45 | 46 | 47 | _multiReadLines = (files, type, filterFunction, fields = [], forEachFunction) => { 48 | return Promise 49 | .all(files.map(file => this._readLines(file, type, filterFunction, fields, forEachFunction))) 50 | .then(objects => objects.flat()); 51 | }; 52 | 53 | _isCacheValid = () => { 54 | return this.cacheFiles 55 | .every(file => { 56 | if (fs.existsSync(file)) { 57 | const stats = fs.statSync(file); 58 | const lastDownloaded = moment(stats.ctime); 59 | 60 | if (moment(moment()).diff(lastDownloaded, 'days') <= this.daysWhoisCache){ 61 | return true; 62 | } 63 | } 64 | 65 | return false; 66 | }); 67 | }; 68 | 69 | _getDump = () => { 70 | 71 | if (this._isCacheValid()) { 72 | console.log("[apnic] Using cached whois data"); 73 | return Promise.resolve(this.cacheFiles); 74 | } else { 75 | console.log("[apnic] Downloading whois data"); 76 | 77 | return Promise 78 | .all(this.dumpUrls.map(this._getDumpFile)); 79 | } 80 | } 81 | 82 | 83 | getObjects = (types, filterFunction, fields, forEachFunction) => { 84 | 85 | return this._getDump() 86 | .then(file => { 87 | console.log(`[${this.connectorName}] Parsing whois data`); 88 | return Promise.all(types.map(type => this._multiReadLines(file, type, filterFunction, fields, forEachFunction))) 89 | .then(objects => objects.flat()); 90 | }); 91 | } 92 | 93 | } -------------------------------------------------------------------------------- /src/connectors/connector.js: -------------------------------------------------------------------------------- 1 | import md5 from "md5"; 2 | import fs from "fs"; 3 | import readline from "readline"; 4 | import zlib from "zlib"; 5 | import moment from "moment"; 6 | const urlParser = require('url'); 7 | const https = require('https'); 8 | const http = require('http'); 9 | 10 | const agentOptions = { 11 | family: 4, 12 | keepAlive: true, 13 | maxSockets: 240, 14 | keepAliveMsecs: 50000000, 15 | maxFreeSockets: 256, 16 | rejectUnauthorized: false 17 | }; 18 | 19 | const proto = { 20 | https: { 21 | fetch: https, 22 | agent: new https.Agent(agentOptions) 23 | }, 24 | http: { 25 | fetch: http, 26 | agent: new http.Agent(agentOptions) 27 | }, 28 | } 29 | 30 | export default class Connector { 31 | constructor(params) { 32 | this.params = params || {}; 33 | this.userAgent = this.params.userAgent || "bulk-whois-parser"; 34 | this.connectorName = "connector"; 35 | this.cacheDir = this.params.cacheDir || ".cache/"; 36 | this.cacheFile = null; 37 | this.dumpUrl = null; 38 | this.daysWhoisCache = this.params.defaultCacheDays || 1; 39 | 40 | if (!fs.existsSync(this.cacheDir)) { 41 | fs.mkdirSync(this.cacheDir, { recursive: true }); 42 | } 43 | } 44 | 45 | _readLines = (compressedFile, type, filterFunction, fields = [], forEachFunction=null) => { 46 | return new Promise((resolve, reject) => { 47 | 48 | if (!filterFunction) { 49 | reject(new Error("You MUST specify a filter function")); 50 | } 51 | 52 | let lastObject = null; 53 | const objects = []; 54 | 55 | let lineReader = readline.createInterface({ 56 | input: fs.createReadStream(compressedFile) 57 | .pipe(zlib.createGunzip()) 58 | .on("error", (error) => { 59 | console.log(error); 60 | console.log(`ERROR: Delete the cache file ${compressedFile}`); 61 | }) 62 | }); 63 | 64 | lineReader 65 | .on('line', (line) => { 66 | if (!lastObject && line.startsWith(type + ":")) { // new object 67 | const [key, value] = this.getStandardFormat(this.getKeyValue(line)); 68 | lastObject = { 69 | [key]: value 70 | }; 71 | } else if (lastObject && line.length === 0) { // end of object 72 | 73 | const objTmp = this.getStandardObject(lastObject); 74 | if (filterFunction(objTmp)) { 75 | if (!!forEachFunction) { 76 | forEachFunction(objTmp); 77 | } else { 78 | objects.push(objTmp); 79 | } 80 | } 81 | lastObject = null; 82 | 83 | } else if (lastObject) { 84 | const [key, value] = this.getStandardFormat(this.getKeyValue(line)); 85 | if (key && (!fields.length || fields.includes(key))) { 86 | if (lastObject[key]) { 87 | if (!Array.isArray(lastObject[key])) { 88 | lastObject[key] = [lastObject[key]]; 89 | } 90 | lastObject[key].push(value); 91 | } else { 92 | lastObject[key] = value; 93 | } 94 | } 95 | 96 | } 97 | }) 98 | .on("error", (error) => { 99 | if (this.params.deleteCorruptedCacheFile) { 100 | try { 101 | fs.unlinkSync(compressedFile); 102 | } catch (error) { 103 | console.log(`Corrupted file ${compressedFile} already deleted`); 104 | } 105 | } 106 | 107 | return reject(error, `Delete the cache file ${compressedFile}`); 108 | }) 109 | .on("close", () => { 110 | resolve(objects); 111 | }) 112 | 113 | }); 114 | 115 | } 116 | 117 | getKeyValue = (line) => { 118 | return line.split(/:(.+)/).filter(i => i.length).map(i => i.trim()); 119 | } 120 | 121 | getStandardFormat = ([key, value]) => { 122 | if (key && value && !key.startsWith("#") && !key.startsWith("%")) { 123 | return [key, value]; 124 | } 125 | 126 | return [null, null]; 127 | }; 128 | 129 | getStandardObject = (object) => { 130 | if (object.remarks) { 131 | if (!Array.isArray(object.remarks)) { 132 | object.remarks = [object.remarks]; 133 | } 134 | } 135 | 136 | if (object.members) { 137 | if (!Array.isArray(object.members)) { 138 | object.members = [object.members]; 139 | } 140 | } 141 | 142 | return object; 143 | }; 144 | 145 | getCacheFileName = (originalName) => { 146 | return [this.cacheDir, md5(originalName)] 147 | .join("/") 148 | .replace("//", "/"); 149 | }; 150 | 151 | _isCacheValid = (file, days) => { 152 | file = file ?? this.cacheFile; 153 | days = days ?? this.daysWhoisCache; 154 | 155 | if (fs.existsSync(file)) { 156 | const stats = fs.statSync(file); 157 | const lastDownloaded = moment(stats.ctime); 158 | 159 | if (moment(moment()).diff(lastDownloaded, 'days') <= days){ 160 | return true; 161 | } 162 | } 163 | 164 | return false; 165 | }; 166 | 167 | _writeFile = (file, data) => { 168 | if (typeof(data) === "object") { 169 | fs.writeFileSync(file, JSON.stringify(data)); 170 | } else { 171 | fs.writeFileSync(file, data); 172 | } 173 | 174 | return Promise.resolve(data); 175 | } 176 | 177 | _readFile = (file, json) => { 178 | return new Promise((resolve, reject) => { 179 | try { 180 | let content = fs.readFileSync(file, 'utf-8'); 181 | 182 | if (json) { 183 | content = JSON.parse(content); 184 | } 185 | 186 | resolve(content); 187 | 188 | } catch (error) { 189 | reject(error); 190 | } 191 | }); 192 | } 193 | 194 | _downloadAndReadFile = (url, file, days=1, json=true) => { 195 | if (!this._isCacheValid(file, days)) { 196 | return this._downloadFile(url, file) 197 | .then(() => this._readFile(file, json)); 198 | } else { 199 | return this._readFile(file, json) 200 | .catch(() => this._downloadFile(url, file) 201 | .then(() => this._readFile(file, json))); 202 | } 203 | } 204 | 205 | _downloadFile = (url, file) => { 206 | return new Promise((resolve, reject) => { 207 | const fileStream = fs.createWriteStream(file); 208 | const protocol = url.toLowerCase().split(":")[0]; 209 | 210 | const options = urlParser.parse(url); 211 | options.agent = proto[protocol].agent; 212 | options.method = 'GET'; 213 | options.gzip = true; 214 | options.timeout = 20000; 215 | options.keepAliveTimeout = 600000000; 216 | 217 | proto[protocol].fetch 218 | .get(options, response => { 219 | response.pipe(fileStream); 220 | 221 | fileStream.on('finish', _ => { 222 | resolve(file); 223 | }); 224 | 225 | fileStream.on('error', error => { 226 | reject(error); 227 | }); 228 | }) 229 | .on('error', error => { 230 | reject(error); 231 | }); 232 | }); 233 | } 234 | 235 | _getDump = () => { 236 | 237 | if (this._isCacheValid()) { 238 | console.log(`[${this.connectorName}] Using cached whois data`); 239 | return Promise.resolve(this.cacheFile); 240 | } else { 241 | console.log(`[${this.connectorName}] Downloading whois data`); 242 | 243 | return this._downloadFile(this.dumpUrl, this.cacheFile) 244 | .catch(error => { 245 | console.log(`Delete the cache file ${this.cacheFile}`); 246 | 247 | return Promise.reject(error); 248 | }); 249 | } 250 | } 251 | 252 | getObjects = (types, filterFunction, fields, forEachFunction) => { 253 | fields = fields || []; 254 | return this._getDump() 255 | .then(file => { 256 | console.log(`[${this.connectorName}] Parsing whois data: ${types}`); 257 | return Promise.all(types.map(type => this._readLines(file, type, filterFunction, fields, forEachFunction))) 258 | .then(objects => objects.flat()); 259 | }); 260 | }; 261 | } 262 | -------------------------------------------------------------------------------- /src/connectors/connectorARINrir.js: -------------------------------------------------------------------------------- 1 | import Connector from "./connector"; 2 | import fs from "fs"; 3 | import ipUtils from "ip-sub"; 4 | import cliProgress from "cli-progress"; 5 | import batchPromises from "batch-promises"; 6 | import webWhois from "whois"; 7 | import md5 from 'md5'; 8 | import moment from 'moment/moment'; 9 | 10 | export default class ConnectorARIN extends Connector { 11 | constructor(params) { 12 | super(params) 13 | 14 | this.connectorName = "arin-rir"; 15 | this.cacheDir += this.connectorName + "/"; 16 | this.statFile = `http://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest`; 17 | this.cacheFile = [this.cacheDir, "arin.inetnums"].join("/").replace("//", "/"); 18 | this.daysWhoisCache = this.params.defaultCacheDays || 7; 19 | this.daysWhoisSuballocationsCache = this.params.daysWhoisSuballocationsCache || 30; 20 | this.skipSuballocations = !!this.params.skipSuballocations; 21 | this.compileSuballocationLocally = !!this.params.compileSuballocationLocally; 22 | 23 | if (this.daysWhoisSuballocationsCache < 7) { 24 | console.log("Sub allocations in ARIN cannot be fetched more than once every 7 days. Using 7 days."); 25 | this.daysWhoisSuballocationsCache = 7; 26 | } 27 | 28 | if (this.daysWhoisSuballocationsCache < this.daysWhoisCache) { 29 | console.log(`Sub allocations in ARIN cannot be fetched more than once every ${this.daysWhoisCache} days. Using ${this.daysWhoisCache} days.`); 30 | this.daysWhoisSuballocationsCache = this.daysWhoisCache; 31 | } 32 | 33 | if (this.daysWhoisCache < 3) { 34 | console.log("NetRanges in ARIN cannot be fetched more than once every 3 days. Using 3 days."); 35 | this.daysWhoisCache = 3; 36 | } 37 | 38 | if (!fs.existsSync(this.cacheDir)) { 39 | fs.mkdirSync(this.cacheDir, { recursive: true }); 40 | } 41 | 42 | } 43 | 44 | _getStatFile = () => { 45 | console.log(`[arin] Downloading stat file`); 46 | 47 | const cacheFile = `${this.cacheDir}arin-stat-file`; 48 | 49 | return this._downloadAndReadFile(this.statFile, cacheFile, this.daysWhoisCache, false); 50 | }; 51 | 52 | _toPrefix = (firstIp, hosts) => { 53 | const af = ipUtils.getAddressFamily(firstIp); 54 | let bits = (af === 4) ? 32 - Math.log2(hosts) : hosts; 55 | 56 | return `${firstIp}/${bits}`; 57 | }; 58 | 59 | 60 | _addSubAllocations = (stats) => { 61 | if (this.skipSuballocations) { 62 | console.log(`[arin] Skipping sub allocations`); 63 | 64 | return stats; 65 | } else { 66 | const v4File = [this.cacheDir, `arin-stat-file-v4.json`].join("/").replace("//", "/"); 67 | const v6File = [this.cacheDir, `arin-stat-file-v6.json`].join("/").replace("//", "/"); 68 | 69 | return this._addSubAllocationsByType(stats, "ipv4") 70 | .then(v4 => this._writeFile(v4File, v4)) 71 | .then(v4 => { 72 | return this._addSubAllocationsByType(stats, "ipv6") 73 | .then(v6 => this._writeFile(v6File, v6)) 74 | .then(v6 => [...v4, ...v6]); 75 | }); 76 | } 77 | } 78 | 79 | _getRemoteSuballocationStatFile = (type) => { 80 | 81 | const file = `https://geofeeds.packetvis.com/geolocatemuch/arin-stat-file-${type}.json`; 82 | const cacheFile = this.getCacheFileName(file); 83 | 84 | return this._downloadAndReadFile(file, cacheFile, this.daysWhoisCache, true) 85 | .then(response => { 86 | if (response && response.length > 8000) { 87 | return response; 88 | } else { 89 | return Promise.reject("Empty remote sub allocation file"); 90 | } 91 | }); 92 | } 93 | 94 | _addSubAllocationByTypeLocally = (stats, type) => { 95 | stats = stats.filter(i => i.type === type && i.status === "allocated"); 96 | const out = stats; 97 | 98 | const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic); 99 | progressBar.start(stats.length, 0); 100 | 101 | return batchPromises(1, stats, item => { 102 | 103 | return this._whois(item.prefix) 104 | .then(data => { 105 | 106 | progressBar.increment(); 107 | 108 | try { 109 | const ips = data 110 | .map(i => i.data) 111 | .flat() 112 | .join("") 113 | .split("\n") 114 | .map(i => i.trim().split(" ")) 115 | .filter(i => i.length >= 5) 116 | .map(i => i.filter(n => ipUtils.isValidIP(n))) 117 | .filter(i => i.length === 2) 118 | 119 | for (let [firstIp, lastIp] of [...new Set(ips)]) { 120 | 121 | const prefixes = ipUtils.ipRangeToCidr(firstIp, lastIp); 122 | 123 | for (let prefix of prefixes) { 124 | 125 | out.push({ 126 | rir: "arin", 127 | type, 128 | prefix, 129 | firstIp, 130 | status: "allocated" 131 | }); 132 | } 133 | } 134 | 135 | } catch (error) { 136 | } 137 | }) 138 | .catch(console.log); 139 | 140 | }) 141 | .catch(console.log) 142 | .then(() => { 143 | 144 | progressBar.stop(); 145 | const index = {}; 146 | for (let i of out) { 147 | index[i.firstIp] = i; 148 | } 149 | 150 | return Object.values(index); 151 | }); 152 | } 153 | 154 | _addSubAllocationsByType = (stats, type) => { 155 | console.log(`[arin] Fetching sub allocations ${type}`); 156 | 157 | if (this.compileSuballocationLocally) { 158 | 159 | return this._addSubAllocationByTypeLocally(stats, type); 160 | } else { 161 | return this._getRemoteSuballocationStatFile(type) 162 | .catch(() => { 163 | 164 | console.log(`[arin] It was not possible to download precompiled sub allocations ${type}, I will try to compile them from whois instead`); 165 | 166 | return this._addSubAllocationByTypeLocally(stats, type); 167 | }); 168 | } 169 | } 170 | 171 | _whois = (prefix) => { 172 | const file = this.getCacheFileName(`whois-prefix-${prefix}`); 173 | 174 | if (this._isCacheValid(file, this._getDistributedCacheTime())) { 175 | return this._readFile(file, true); 176 | } else { 177 | 178 | return new Promise((resolve, reject) => { 179 | webWhois.lookup(`r > ${prefix}`, { 180 | follow: 0, 181 | verbose: true, 182 | timeout: 5000, 183 | returnPartialOnTimeout: true, 184 | server: "whois.arin.net" 185 | }, (error, data) => { 186 | if (error) { 187 | reject(error) 188 | } else { 189 | this._writeFile(file, data).then(resolve); 190 | } 191 | }) 192 | }); 193 | } 194 | } 195 | 196 | _compileNetRangesListLocally = () => { 197 | return this._getStatFile() 198 | .then(data => { 199 | const structuredData = data 200 | .split("\n") 201 | .filter(line => line.includes("ipv4") || line.includes("ipv6") ) 202 | .map(line => line.split("|")) 203 | .map(([rir, cc, type, firstIpUp, hosts, date, status, hash]) => { 204 | const firstIp = firstIpUp.toLowerCase(); 205 | return { 206 | rir, 207 | type, 208 | prefix: this._toPrefix(firstIp, hosts), 209 | firstIp, 210 | hosts, 211 | date, 212 | status 213 | }; 214 | }) 215 | .filter(i => i.rir === "arin" && 216 | ["ipv4", "ipv6"].includes(i.type) && 217 | ["allocated", "assigned"].includes(i.status)); 218 | 219 | return structuredData.reverse(); 220 | }) 221 | .then(this._addSubAllocations); 222 | } 223 | 224 | _createWhoisDump = (types) => { 225 | if (this._isCacheValid(this.cacheFile, 1)) { 226 | console.log(`[arin] Using cached whois data: ${types}`); 227 | return Promise.resolve(JSON.parse(fs.readFileSync(this.cacheFile, 'utf-8'))); 228 | } else { 229 | return this.getRemotePreFilteredNetRanges() 230 | .catch(() => this._compileNetRangesListLocally()) 231 | .then(this._toStandardFormat) 232 | .then(inetnums => inetnums.filter(i => !!i)) 233 | .then(inetnums => this._writeFile(this.cacheFile, inetnums)) 234 | } 235 | }; 236 | 237 | _getDistributedCacheTime = () => { 238 | const rndInt = Math.floor(Math.random() * parseInt(this.daysWhoisSuballocationsCache/2)) + 1; 239 | return Math.max(this.daysWhoisCache, this.daysWhoisSuballocationsCache - rndInt); 240 | } 241 | 242 | _getRdapQuery = (prefix) => { 243 | const url = `https://rdap.arin.net/registry/ip/${prefix}`; 244 | const file = this.getCacheFileName(url); 245 | 246 | return this._downloadAndReadFile(url, file, this._getDistributedCacheTime(), true) 247 | .catch(error => { 248 | console.log(`Cannot retrieve ${prefix}: ${error}`); 249 | return null; 250 | }); 251 | }; 252 | 253 | _toStandardFormat = (items) => { 254 | console.log(`[arin] Fetching NetRanges`); 255 | 256 | const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic); 257 | progressBar.start(items.length, 0); 258 | 259 | const singleBatch = (items) => { 260 | return batchPromises(4, items, item => { 261 | return this._getRdapQuery(item.firstIp) 262 | .then(data => { 263 | progressBar.increment(); 264 | if (data) { 265 | const { startAddress, endAddress, remarks, events } = data; 266 | const inetnum = {}; 267 | 268 | if (remarks) { 269 | const remarksArray = remarks.map(remark => (remark.description || [])); 270 | const cleanRemarks = remarksArray.flat().filter(i => i.toLowerCase().includes("geofeed")); 271 | 272 | if (cleanRemarks?.length) { 273 | 274 | const af = ipUtils.getAddressFamily(startAddress); 275 | if (af === 4) { 276 | inetnum.inetnum = `${startAddress} - ${endAddress}`; 277 | inetnum.type = "inetnum"; 278 | } else { 279 | inetnum.inet6num = item.prefix; 280 | inetnum.type = "inet6num"; 281 | } 282 | const lastChanges = (events || []) 283 | .filter(i => i.eventAction === "last changed") 284 | .pop(); 285 | inetnum["last-modified"] = lastChanges ? lastChanges.eventDate : null; 286 | 287 | inetnum.remarks = cleanRemarks; 288 | 289 | for (let prop in data) { 290 | if (typeof (data[prop]) === "string" && !inetnum[prop]) { 291 | inetnum[prop] = data[prop]; 292 | } 293 | } 294 | 295 | return inetnum; 296 | } 297 | } 298 | } 299 | 300 | return null; 301 | }); 302 | }) 303 | } 304 | 305 | const halfList = Math.ceil(items.length/2); 306 | return Promise 307 | .all([ 308 | singleBatch(items.slice(0, halfList)), 309 | singleBatch(items.slice(halfList)) 310 | ]) 311 | .then(inetnums => { 312 | progressBar.stop(); 313 | 314 | return inetnums.flat(); 315 | }) 316 | }; 317 | 318 | getRemotePreFilteredNetRanges = () => { 319 | if (this.compileSuballocationLocally) { 320 | return Promise.reject("Compile locally"); 321 | } 322 | 323 | const url = "https://geofeeds.packetvis.com/geolocatemuch/arin.inetnums"; 324 | const metadataUrl = "https://geofeeds.packetvis.com/geolocatemuch/metadata.json" 325 | const file = this.getCacheFileName(url); 326 | const metadataFile = this.getCacheFileName(metadataUrl); 327 | 328 | return this._downloadAndReadFile(metadataUrl, metadataFile, 1, true) 329 | .then(metadata => { 330 | const lastUpdate = moment.unix(metadata.lastUpdate); 331 | if (moment(moment()).diff(lastUpdate, 'days') <= 10){ 332 | 333 | return this._downloadAndReadFile(url, file, 1, true); 334 | } else { 335 | return Promise.reject("Remote file is too old"); 336 | } 337 | }) 338 | .then(data => { 339 | console.log("[arin] Using pre filtered inetnums"); 340 | 341 | return data 342 | .map(({startAddress, ipVersion, inet6num}) => { 343 | return { 344 | firstIp: startAddress, 345 | prefix: ipVersion === "v6" ? inet6num : null 346 | }; 347 | }); 348 | }) 349 | } 350 | 351 | getObjects = (types, filterFunction, fields, forEachFunction) => { 352 | if (this.params.arinBulk) { 353 | console.log("[arin] bulk whois data not yet supported"); 354 | return Promise.resolve([]); 355 | } else { 356 | return this._createWhoisDump(types) 357 | .then(data => { 358 | const filtered = data.filter(i => types.includes(i.type) && filterFunction(i)); 359 | if (fields && fields.length) { 360 | return filtered 361 | .map(item => { 362 | const out = {}; 363 | for (let k in item) { 364 | if (fields.includes(k)) { 365 | out[k] = item[k]; 366 | } 367 | } 368 | 369 | return out; 370 | }); 371 | } else { 372 | return filtered; 373 | } 374 | }) 375 | .then(data => { 376 | if (!!forEachFunction) { 377 | for (let i of data) { 378 | forEachFunction(i); 379 | } 380 | 381 | return []; 382 | } 383 | 384 | return data; 385 | }); 386 | } 387 | } 388 | } --------------------------------------------------------------------------------