├── .gitignore ├── Gemfile ├── LICENSE.txt ├── README.md ├── Rakefile ├── bin └── pdnstool ├── lib └── passivedns │ ├── client.rb │ └── client │ ├── cli.rb │ ├── passivedb.rb │ ├── provider │ ├── circl.rb │ ├── dnsdb.rb │ ├── osc.rb │ ├── passivetotal.rb │ ├── riskiq.rb │ └── virustotal.rb │ ├── state.rb │ └── version.rb ├── passivedns-client.gemspec └── test ├── helper.rb ├── test_cli.rb └── test_passivedns-client.rb /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | *.rbc 3 | .bundle 4 | .config 5 | .yardoc 6 | Gemfile.lock 7 | InstalledFiles 8 | _yardoc 9 | coverage 10 | doc/ 11 | html/ 12 | lib/bundler/man 13 | pkg 14 | rdoc 15 | spec/reports 16 | test/tmp 17 | test/version_tmp 18 | tmp 19 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # Specify your gem's dependencies in passivedns-client.gemspec 4 | gemspec 5 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013 chrislee35 2 | 3 | MIT License 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PassiveDNS::Client 2 | 3 | This rubygem queries the following Passive DNS databases: 4 | 5 | * CIRCL 6 | * DNSDB (FarSight) 7 | * OpenSource Context (OSC) 8 | * PassiveTotal 9 | * RiskIQ 10 | * VirusTotal 11 | 12 | Passive DNS is a technique where IP to hostname mappings are made by recording the answers of other people's queries. 13 | 14 | There is a tool included, pdnstool, that wraps a lot of the functionality that you would need. 15 | 16 | Please note that use of any passive DNS database is subject to the terms of use of that passive DNS database. Use of this script in violation of their terms is strongly discouraged. Also, please do not add any obfuscation to try to work around their terms of service. If you need special services, ask the providers for help/permission. Remember, these passive DNS operators are my friends. I don't want to have a row with them because some jerk used this library to abuse them. 17 | 18 | If you like this library, please buy the Passive DNS operators a round of beers. 19 | 20 | ## Installation 21 | 22 | Add this line to your application's Gemfile: 23 | 24 | gem 'passivedns-client' 25 | 26 | And then execute: 27 | 28 | $ bundle 29 | 30 | Or install it yourself as: 31 | 32 | $ gem install passivedns-client 33 | 34 | ## Configuration 35 | 36 | From version 2.0.0 on, all configuration keys for passive DNS providers are in one configuration file. By default the location of the file is $HOME/.passivedns-client . The syntax of this file is as follows: 37 | 38 | [dnsdb] 39 | APIKEY = 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef 40 | [virustotal] 41 | APIKEY = 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef 42 | [passivetotal] 43 | USERNAME = tom@example.com 44 | APIKEY = 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef 45 | [circl] 46 | USERNAME = circl_user 47 | PASSWORD = circl_pass 48 | [riskiq] 49 | API_TOKEN = 0123456789abcdef 50 | API_PRIVATE_KEY = 01234567890abcdefghijklmnopqrstu 51 | [osc] 52 | APIKEY = 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef 53 | 54 | CIRCL also can use and authorization token. In that case, you should drop the USERNAME and PASSWORD options and change the section to something like the following: 55 | 56 | [circl] 57 | AUTH_TOKEN = 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef 58 | 59 | ## Getting Access 60 | * CIRCL : https://www.circl.lu/services/passive-dns/ 61 | * DNSDB (Farsight Security) : https://api.dnsdb.info/ 62 | * OSC: https://oscontext.com/ 63 | * PassiveTotal : https://www.passivetotal.org 64 | * RiskIQ : https://github.com/RiskIQ/python_api/blob/master/LICENSE 65 | * VirusTotal : https://www.virustotal.com 66 | 67 | ## Usage 68 | 69 | require 'passivedns/client' 70 | 71 | c = PassiveDNS::Client.new(['riskiq','dnsdb']) 72 | results = c.query("example.com") 73 | 74 | 75 | Or use the included tool... 76 | 77 | Usage: bin/pdnstool [-d [cdprv]] [-g|-v|-m|-c|-x|-y|-j|-t] [-os ] [-f ] [-r#|-w#|-v] [-l ] [--config ] 78 | Passive DNS Providers 79 | -dcdprv uses all of the available passive dns database 80 | -dc use CIRCL 81 | -dd use DNSDB 82 | -dp use PassiveTotal 83 | -dr use RiskIQ 84 | -dv use VirusTotal 85 | -dvr uses VirusTotal and RiskIQ (for example) 86 | 87 | Output Formatting 88 | -g link-nodal GDF visualization definition 89 | -z link-nodal graphviz visualization definition 90 | -m link-nodal graphml visualization definition 91 | -c CSV 92 | -x XML 93 | -y YAML 94 | -j JSON 95 | -t ASCII text (default) 96 | -s specifies a field separator for text output, default is tab 97 | 98 | State and Recursion 99 | -f[file] specifies a sqlite3 database used to read the current state - useful for large result sets and generating graphs of previous runs. 100 | -r# specifies the levels of recursion to pull. **WARNING** This is quite taxing on the pDNS servers, so use judiciously (never more than 3 or so) or find yourself blocked! 101 | -w# specifies the amount of time to wait, in seconds, between queries (Default: 0) 102 | -l limits the number of records returned per passive dns database queried. 103 | 104 | Specifying a Configuration File 105 | --config specifies a config file. default: /home/chris/.passivedns-client 106 | 107 | Getting Help 108 | -h hello there. This option produces this helpful help information on how to access help. 109 | -v debugging information 110 | 111 | ## Writing Your Own Database Adaptor 112 | 113 | module PassiveDNS #:nodoc: don't document this 114 | # The Provider module contains all the Passive DNS provider client code 115 | module Provider 116 | # Queries OSContext's passive DNS database 117 | class MyDatabaseAdaptor < PassiveDB 118 | # Sets the modules self-reported name to "OSC" 119 | def self.name 120 | "MyPerfectDNS" # short, proper label 121 | end 122 | #override 123 | def self.config_section_name 124 | "perfect" # very short label to use in the configuration file 125 | end 126 | #override 127 | def self.option_letter 128 | "p" # single letter to specify the option for the command line tool 129 | end 130 | 131 | attr_accessor :debug 132 | 133 | def initialize(options={}) 134 | @debug = options[:debug] || false 135 | # please include a way to change the base URL, HOST, etc., so that people can test 136 | # against a test/alternate version of your service 137 | @base = options["URL"] || "http://myperfectdns.example.com/pdns.cgi?query=" 138 | @apikey = options["APIKEY"] || raise("APIKEY option required for #{self.class}") 139 | end 140 | 141 | # override 142 | def lookup(label, limit=nil) 143 | $stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug 144 | recs = [] 145 | Timeout::timeout(240) { 146 | t1 = Time.now 147 | # TODO: your code goes here to fetch the data from your service 148 | # TODO: don't forget to impose the limit either during the fetch or during the parse phase 149 | response_time = Time.now - t1 150 | # TODO: parse your data and add PDNSResult objects to recs array 151 | recs << PDNSResult.new(self.class.name, response_time, rrname , 152 | rdata, rrtype, ttl, first_seen, last_seen, count ) 153 | } 154 | recs 155 | rescue Timeout::Error => e # using the implied "begin/try" from the beginning of the function 156 | $stderr.puts "#{self.class.name} lookup timed out: #{label}" 157 | end 158 | end 159 | end 160 | end 161 | 162 | ## Passive DNS - Common Output Format 163 | 164 | There is an RFC, Passive DNS - Common Output Format, and a proof of concept implementation, pdns-qof-server, that describes a recommened JSON format for passive DNS data. passivedns-client is very close to supporting it, but since I've never enteracted with a true implementation of this RFC, I can't attest that I could correctly parse it. I think they way that they can encode multiple results into one record would actually break what I have right now. 165 | 166 | Right now, I'm in a wait and see mode with how this progresses before I start supporting yet another format or request that other providers start to adhere to a common output format. If you have thoughts on the matter, I would love to discuss. 167 | 168 | ## Contributing 169 | 170 | 1. Fork it 171 | 2. Create your feature branch (`git checkout -b my-new-feature`) 172 | 3. Commit your changes (`git commit -am 'Add some feature'`) 173 | 4. Push to the branch (`git push origin my-new-feature`) 174 | 5. Create new Pull Request 175 | 176 | 177 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env rake 2 | require "bundler/gem_tasks" 3 | 4 | require 'rake/testtask' 5 | require 'rdoc/task' 6 | 7 | Rake::TestTask.new do |t| 8 | t.libs << 'lib' 9 | t.test_files = FileList['test/test_*.rb'] 10 | t.verbose = true 11 | end 12 | 13 | RDoc::Task.new do |rd| 14 | rd.main = "README.doc" 15 | rd.rdoc_files.include("README.md", "lib/**/*.rb") 16 | rd.options << "--all" 17 | rd.options << "--verbose" 18 | end 19 | 20 | task :default => :test -------------------------------------------------------------------------------- /bin/pdnstool: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | require 'passivedns/client' 3 | require 'passivedns/client/cli' 4 | 5 | puts PassiveDNS::CLI.run(ARGV) -------------------------------------------------------------------------------- /lib/passivedns/client.rb: -------------------------------------------------------------------------------- 1 | require "passivedns/client/version" 2 | # DESCRIPTION: queries passive DNS databases 3 | # This code is released under the LGPL: http://www.gnu.org/licenses/lgpl-3.0.txt 4 | # Please note that use of any passive dns database is subject to the terms of use of that passive dns database. Use of this script in violation of their terms is not encouraged in any way. Also, please do not add any obfuscation to try to work around their terms of service. If you need special services, ask the providers for help/permission. 5 | # Remember, these passive DNS operators are my friends. I don't want to have a row with them because some asshat used this library to abuse them. 6 | require 'passivedns/client/state' 7 | require 'passivedns/client/passivedb' 8 | 9 | # load all the providers 10 | $passivedns_providers = Array.new 11 | provider_path = File.dirname(__FILE__)+"/client/provider/*.rb" 12 | Dir.glob(provider_path).each do |provider| 13 | name = File.basename(provider, '.rb') 14 | require "passivedns/client/provider/#{name}.rb" 15 | $passivedns_providers << name 16 | end 17 | 18 | require 'configparser' 19 | 20 | module PassiveDNS # :nodoc: 21 | 22 | class SecurityControl 23 | def allow(user_level) 24 | raise "unimplemented" 25 | end 26 | end 27 | 28 | class TLPSecurityControl < SecurityControl 29 | LEVELS = ['white','green','yellow','red'] 30 | 31 | def initialize(tlp) 32 | if tlp =~ /(white|green|yellow|red)/i 33 | @tlp = tlp.downcase 34 | @tlp_level = LEVELS.index(@tlp) 35 | else 36 | raise "Unknown TLP setting, #{tlp}" 37 | end 38 | end 39 | 40 | def allow(user_level) 41 | user_level = LEVELS.index(user_level.downcase) 42 | if user_level == nil 43 | raise "Invalid user level, #{user_level}" 44 | end 45 | return(user_level >= @tlp_level) 46 | end 47 | 48 | def to_s() 49 | @tlp 50 | end 51 | end 52 | 53 | # struct to contain the results from a PassiveDNS lookup 54 | class PDNSResult < Struct.new(:source, :response_time, :query, :answer, :rrtype, :ttl, :firstseen, :lastseen, :count, :security); end 55 | 56 | # coodinates the lookups accross all configured PassiveDNS providers 57 | class Client 58 | 59 | # instantiate and configure all specified PassiveDNS providers 60 | # pdns array of passivedns provider names, e.g., ["dnsdb","virustotal"] 61 | # configfile filename of the passivedns-client configuration (this should probably be abstracted) 62 | def initialize(pdns=$passivedns_providers, configfile="#{ENV['HOME']}/.passivedns-client") 63 | cp = {} 64 | if File.exist?(configfile) 65 | cp = ConfigParser.new(configfile) 66 | else 67 | $stderr.puts "Could not find config file at #{configfile}. Using a blank configuration." 68 | end 69 | # this creates a map of all the PassiveDNS provider names and their classes 70 | class_map = {} 71 | PassiveDNS::Provider.constants.each do |const| 72 | if PassiveDNS::Provider.const_get(const).is_a?(Class) and PassiveDNS::Provider.const_get(const).superclass == PassiveDNS::PassiveDB 73 | class_map[PassiveDNS::Provider.const_get(const).config_section_name] = PassiveDNS::Provider.const_get(const) 74 | end 75 | end 76 | 77 | @pdnsdbs = [] 78 | pdns.uniq.each do |pd| 79 | if class_map[pd] 80 | @pdnsdbs << class_map[pd].new(cp[pd] || {}) 81 | else 82 | raise "Unknown Passive DNS provider: #{pd}" 83 | end 84 | end 85 | 86 | end #initialize 87 | 88 | # set the debug flag 89 | def debug=(d) 90 | @pdnsdbs.each do |pdnsdb| 91 | pdnsdb.debug = d 92 | end 93 | end 94 | 95 | def timeout=(t) 96 | @pdnsdbs.each do |pdnsdb| 97 | pdnsdb.timeout = t 98 | end 99 | end 100 | 101 | # perform the query lookup accross all configured PassiveDNS providers 102 | def query(item, limit=nil) 103 | threads = [] 104 | @pdnsdbs.each do |pdnsdb| 105 | threads << Thread.new(item) do |q| 106 | pdnsdb.lookup(q, limit) 107 | end 108 | end 109 | 110 | results = [] 111 | threads.each do |thr| 112 | rv = thr.join.value 113 | if rv 114 | rv.each do |r| 115 | if ["A","AAAA","NS","CNAME","PTR","SOA"].index(r.rrtype) 116 | results << r 117 | end 118 | end 119 | end 120 | end 121 | 122 | return results 123 | end #query 124 | 125 | end # Client 126 | end # PassiveDNS 127 | -------------------------------------------------------------------------------- /lib/passivedns/client/cli.rb: -------------------------------------------------------------------------------- 1 | require 'getoptlong' 2 | require 'structformatter' 3 | require 'getoptlong' 4 | require 'yaml' 5 | require 'pp' 6 | 7 | module PassiveDNS # :nodoc: 8 | # Handles all the command-line parsing, state tracking, and dispatching queries to the PassiveDNS::Client instance 9 | # CLInterface is aliased by CLI 10 | class CLInterface 11 | # generates a mapping between the option letter for each PassiveDNS provider and the class 12 | def self.get_letter_map 13 | letter_map = {} 14 | mod = PassiveDNS::Provider 15 | mod.constants.each do |const| 16 | if mod.const_get(const).is_a?(Class) and mod.const_get(const).superclass == PassiveDNS::PassiveDB 17 | letter = mod.const_get(const).option_letter 18 | name = mod.const_get(const).name 19 | config_section_name = mod.const_get(const).config_section_name 20 | letter_map[letter] = [name, config_section_name] 21 | end 22 | end 23 | letter_map 24 | end 25 | 26 | # parses the command line and yields an options hash 27 | # === Default Options 28 | # options = { 29 | # :pdnsdbs => [], # passive dns providers to query 30 | # :format => "text", # output format 31 | # :sep => "\t", # field separator for text format 32 | # :recursedepth => 1, # recursion depth 33 | # :wait => 0, # wait period between recursions 34 | # :res => nil, # unused. I don't remember why this is here. 35 | # :debug => false, # debug flag 36 | # :sqlitedb => nil, # filename for maintaining state in an sqlite3 db 37 | # :limit => nil, # number of results per provider per recursion 38 | # :help => false # display the usage text 39 | # } 40 | def self.parse_command_line(args) 41 | origARGV = ARGV.dup 42 | ARGV.replace(args) 43 | opts = GetoptLong.new( 44 | [ '--help', '-h', GetoptLong::NO_ARGUMENT ], 45 | [ '--debug', '-v', GetoptLong::NO_ARGUMENT ], 46 | [ '--database', '-d', GetoptLong::REQUIRED_ARGUMENT ], 47 | 48 | [ '--gdf', '-g', GetoptLong::NO_ARGUMENT ], 49 | [ '--graphviz', '-z', GetoptLong::NO_ARGUMENT ], 50 | [ '--graphml', '-m', GetoptLong::NO_ARGUMENT ], 51 | [ '--csv', '-c', GetoptLong::NO_ARGUMENT ], 52 | [ '--xml', '-x', GetoptLong::NO_ARGUMENT ], 53 | [ '--yaml', '-y', GetoptLong::NO_ARGUMENT ], 54 | [ '--json', '-j', GetoptLong::NO_ARGUMENT ], 55 | [ '--text', '-t', GetoptLong::NO_ARGUMENT ], 56 | [ '--sep', '-s', GetoptLong::REQUIRED_ARGUMENT ], 57 | 58 | [ '--sqlite3', '-f', GetoptLong::REQUIRED_ARGUMENT ], 59 | [ '--recurse', '-r', GetoptLong::REQUIRED_ARGUMENT ], 60 | [ '--wait', '-w', GetoptLong::REQUIRED_ARGUMENT ], 61 | [ '--limit', '-l', GetoptLong::REQUIRED_ARGUMENT ], 62 | [ '--config', GetoptLong::REQUIRED_ARGUMENT ] 63 | ) 64 | 65 | letter_map = get_letter_map 66 | 67 | # sets the default search methods 68 | options = { 69 | :pdnsdbs => [], 70 | :format => "text", 71 | :sep => "\t", 72 | :recursedepth => 1, 73 | :wait => 0, 74 | :res => nil, 75 | :debug => false, 76 | :sqlitedb => nil, 77 | :limit => nil, 78 | :help => false, 79 | :configfile => "#{ENV['HOME']}/.passivedns-client" 80 | } 81 | 82 | opts.each do |opt, arg| 83 | case opt 84 | when '--help' 85 | options[:help] = true 86 | when '--debug' 87 | options[:debug] = true 88 | when '--database' 89 | arg.split(//).each do |c| 90 | if c == ',' 91 | next 92 | elsif letter_map[c] 93 | options[:pdnsdbs] << letter_map[c][1] 94 | else 95 | $stderr.puts "ERROR: Unknown passive DNS database identifier: #{c}." 96 | usage(letter_map) 97 | end 98 | end 99 | when '--gdf' 100 | options[:format] = 'gdf' 101 | when '--graphviz' 102 | options[:format] = 'graphviz' 103 | when '--graphml' 104 | options[:format] = 'graphml' 105 | when '--csv' 106 | options[:format] = 'text' 107 | options[:sep] = ',' 108 | when '--yaml' 109 | options[:format] = 'yaml' 110 | when '--xml' 111 | options[:format] = 'xml' 112 | when '--json' 113 | options[:format] = 'json' 114 | when '--text' 115 | options[:format] = 'text' 116 | when '--sep' 117 | options[:sep] = arg 118 | when '--recurse' 119 | options[:recursedepth] = arg.to_i 120 | when '--wait' 121 | options[:wait] = arg.to_i 122 | when '--sqlite3' 123 | options[:sqlitedb] = arg 124 | when '--limit' 125 | options[:limit] = arg.to_i 126 | when '--config' 127 | options[:configfile] = arg 128 | else 129 | options[:help] = true 130 | end 131 | end 132 | args = ARGV.dup 133 | ARGV.replace(origARGV) 134 | 135 | if options[:pdnsdbs].length == 0 136 | options[:pdnsdbs] << "virustotal" 137 | end 138 | 139 | if options[:debug] 140 | $stderr.puts "Using the following databases: #{options[:pdnsdbs].join(", ")}" 141 | $stderr.puts "Recursions: #{options[:recursedepth]}, Wait time: #{options[:wait]}, Limit: #{options[:limit] or 'none'}" 142 | if options[:format] == "text" or options[:format] == "csv" 143 | $stderr.puts "Output format: #{options[:format]} (sep=\"#{options[:sep]}\")" 144 | else 145 | $stderr.puts "Output format: #{options[:format]}" 146 | end 147 | if ENV['http_proxy'] 148 | $stderr.puts "Using proxy settings: http_proxy=#{ENV['http_proxy']}, https_proxy=#{ENV['https_proxy']}" 149 | end 150 | end 151 | 152 | [options, args] 153 | end 154 | 155 | # returns a string containing the usage information 156 | # takes in a hash of letter to passive dns providers 157 | def self.usage(letter_map) 158 | databases = letter_map.keys.sort.join("") 159 | help_text = "" 160 | help_text << "Usage: #{$0} [-d [#{databases}]] [-g|-v|-m|-c|-x|-y|-j|-t] [-os ] [-f ] [-r#|-w#|-v] [-l ] [--config ] \n" 161 | help_text << "Passive DNS Providers\n" 162 | help_text << " -d#{databases} uses all of the available passive dns database\n" 163 | letter_map.keys.sort.each do |l| 164 | help_text << " -d#{l} use #{letter_map[l][0]}\n" 165 | end 166 | help_text << " -dvr uses VirusTotal and RiskIQ (for example)\n" 167 | help_text << "\n" 168 | help_text << "Output Formatting\n" 169 | help_text << " -g link-nodal GDF visualization definition\n" 170 | help_text << " -z link-nodal graphviz visualization definition\n" 171 | help_text << " -m link-nodal graphml visualization definition\n" 172 | help_text << " -c CSV\n" 173 | help_text << " -x XML\n" 174 | help_text << " -y YAML\n" 175 | help_text << " -j JSON\n" 176 | help_text << " -t ASCII text (default)\n" 177 | help_text << " -s specifies a field separator for text output, default is tab\n" 178 | help_text << "\n" 179 | help_text << "State and Recursion\n" 180 | help_text << " -f[file] specifies a sqlite3 database used to read the current state - useful for large result sets and generating graphs of previous runs.\n" 181 | help_text << " -r# specifies the levels of recursion to pull. **WARNING** This is quite taxing on the pDNS servers, so use judiciously (never more than 3 or so) or find yourself blocked!\n" 182 | help_text << " -w# specifies the amount of time to wait, in seconds, between queries (Default: 0)\n" 183 | help_text << " -l limits the number of records returned per passive dns database queried.\n" 184 | help_text << "\n" 185 | help_text << "Specifying a Configuration File\n" 186 | help_text << " --config specifies a config file. default: #{ENV['HOME']}/.passivedns-client\n" 187 | help_text << "\n" 188 | help_text << "Getting Help\n" 189 | help_text << " -h hello there. This option produces this helpful help information on how to access help.\n" 190 | help_text << " -v debugging information\n" 191 | 192 | help_text 193 | end 194 | 195 | # performs a stateful, recursive (if desired) passive DNS lookup against all specified providers 196 | def self.pdnslookup(state, pdnsclient, options) 197 | recursedepth = options[:recursedepth] 198 | wait = options[:wait] 199 | debug = options[:debug] 200 | limit = options[:limit] 201 | puts "pdnslookup: #{state.level} #{recursedepth}" if debug 202 | level = 0 203 | while level < recursedepth 204 | puts "pdnslookup: #{level} < #{recursedepth}" if debug 205 | state.each_query(recursedepth) do |q| 206 | rv = pdnsclient.query(q,limit) 207 | if rv 208 | rv.each do |r| 209 | if ["A","AAAA","NS","CNAME","PTR", "SOA"].index(r.rrtype) 210 | puts "pdnslookup: #{r.to_s}" if debug 211 | state.add_result(r) 212 | end 213 | end 214 | else 215 | state.update_query(rv,'failed') 216 | end 217 | sleep wait if level < recursedepth 218 | end 219 | level += 1 220 | end 221 | state 222 | end 223 | 224 | # returns a string transforming all the PassiveDNS::PDNSResult stored in the state object into text/xml/json/etc. 225 | def self.results_to_s(state,options) 226 | format = options[:format] 227 | sep = options[:sep] 228 | case format 229 | when 'text' 230 | PassiveDNS::PDNSResult.members.join(sep)+"\n"+state.to_s(sep) 231 | when 'yaml' 232 | state.to_yaml 233 | when 'xml' 234 | state.to_xml 235 | when 'json' 236 | state.to_json 237 | when 'gdf' 238 | state.to_gdf 239 | when 'graphviz' 240 | state.to_graphviz 241 | when 'graphml' 242 | state.to_graphml 243 | end 244 | end 245 | 246 | # create a state instance 247 | def self.create_state(sqlitedb=nil) 248 | state = nil 249 | if sqlitedb 250 | state = PassiveDNS::PDNSToolStateDB.new(sqlitedb) 251 | else 252 | state = PassiveDNS::PDNSToolState.new 253 | end 254 | state 255 | end 256 | 257 | # main method, takes command-line arguments and performs the desired queries and outputs 258 | def self.run(args) 259 | options, items = parse_command_line(args) 260 | if options[:help] 261 | return usage(get_letter_map) 262 | end 263 | if options[:recursedepth] > 3 264 | $stderr.puts "WARNING: a recursedepth of > 3 can be abusive, please reconsider: sleeping 60 seconds for sense to come to you (hint: hit CTRL-C)" 265 | sleep 60 266 | end 267 | state = create_state(options[:sqlitedb]) 268 | state.debug = options[:debug] 269 | 270 | pdnsclient = PassiveDNS::Client.new(options[:pdnsdbs], options[:configfile]) 271 | pdnsclient.debug = options[:debug] 272 | 273 | if items.length > 0 274 | items.each do |arg| 275 | state.add_query(arg,'pending',0) 276 | end 277 | else 278 | $stdin.each_line do |l| 279 | state.add_query(l.chomp,'pending',0) 280 | end 281 | end 282 | pdnslookup(state,pdnsclient,options) 283 | results_to_s(state,options) 284 | end 285 | end 286 | # Alias for the CLInterface class 287 | CLI = PassiveDNS::CLInterface 288 | end 289 | 290 | -------------------------------------------------------------------------------- /lib/passivedns/client/passivedb.rb: -------------------------------------------------------------------------------- 1 | module PassiveDNS #:nodoc: don't document this 2 | # abstract class that all PassiveDNS::Provider should subclass 3 | class PassiveDB 4 | # raises an exception that this should be implemented by the subclass 5 | def self.name 6 | raise "You should implement your own version of .name" 7 | end 8 | 9 | # raises an exception that this should be implemented by the subclass 10 | def self.config_section_name 11 | name 12 | end 13 | 14 | # raises an exception that this should be implemented by the subclass 15 | def self.option_letter 16 | raise "You should pick a unique letter to serve as your database option letter for the command line option -d" 17 | end 18 | 19 | # raises an exception that this should be implemented by the subclass 20 | def lookup(label, limit=nil) 21 | raise "You must implement the lookup function" 22 | end 23 | end 24 | end -------------------------------------------------------------------------------- /lib/passivedns/client/provider/circl.rb: -------------------------------------------------------------------------------- 1 | # DESCRIPTION: Module to query PassiveTotal's passive DNS repository 2 | 3 | require 'net/http' 4 | require 'net/https' 5 | require 'openssl' 6 | 7 | module PassiveDNS #:nodoc: don't document this 8 | # The Provider module contains all the Passive DNS provider client code 9 | module Provider 10 | # Queries CIRCL.LU's passive DNS database 11 | # Circl is aliased by CIRCL 12 | class Circl < PassiveDB 13 | # Sets the modules self-reported name to "CIRCL" 14 | def self.name 15 | "CIRCL" 16 | end 17 | # Sets the configuration section name to "circl" 18 | def self.config_section_name 19 | "circl" 20 | end 21 | # Sets the command line database argument to "c" 22 | def self.option_letter 23 | "c" 24 | end 25 | 26 | # :debug enables verbose logging to standard output 27 | attr_accessor :debug 28 | # === Options 29 | # * :debug Sets the debug flag for the module 30 | # * "USERNAME" User name associated with your CIRCL account 31 | # * "PASSWORD" Password associated with your CIRCL account 32 | # * "AUTH_TOKEN" Authorization token associated with your CIRCL account 33 | # * "URL" Alternate url for testing. Defaults to "https://www.circl.lu/pdns/query" 34 | 35 | # You should either have a username+password or an authorization token to use this service 36 | # 37 | # === Example Instantiation 38 | # 39 | # options = { 40 | # :debug => true, 41 | # "USERNAME" => "circl_user", 42 | # "PASSWORD" => "circl_pass", 43 | # "URL" => "https://www.circl.lu/pdns/query" 44 | # } 45 | # 46 | # PassiveDNS::Provider::CIRCL.new(options) 47 | # 48 | def initialize(options={}) 49 | @debug = options[:debug] || false 50 | @timeout = options[:timeout] || 20 51 | @username = options["USERNAME"] 52 | @password = options["PASSWORD"] 53 | @auth_token = options["AUTH_TOKEN"] 54 | @url = options["URL"] || "https://www.circl.lu/pdns/query" 55 | end 56 | 57 | # Takes a label (either a domain or an IP address) and returns 58 | # an array of PassiveDNS::PDNSResult instances with the answers to the query 59 | def lookup(label, limit=nil) 60 | $stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug 61 | recs = [] 62 | Timeout::timeout(@timeout) { 63 | url = @url+"/"+label 64 | $stderr.puts "DEBUG: #{self.class.name} url = #{url}" if @debug 65 | begin 66 | url = URI.parse url 67 | rescue URI::InvalidURIError 68 | $stderr.puts "ERROR: Invalid address: #{url}" 69 | return recs 70 | end 71 | http = Net::HTTP.new(url.host, url.port) 72 | http.use_ssl = (url.scheme == 'https') 73 | http.verify_mode = OpenSSL::SSL::VERIFY_NONE 74 | http.verify_depth = 5 75 | request = Net::HTTP::Get.new(url.request_uri) 76 | request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}") 77 | if @username 78 | request.basic_auth(@username, @password) 79 | end 80 | if @auth_token 81 | request.add_field("Authorization", @auth_token) 82 | end 83 | t1 = Time.now 84 | 0.upto(9) do 85 | response = http.request(request) 86 | body = response.body 87 | if body == "Rate Limit Exceeded" 88 | $stderr.puts "DEBUG: Rate Limit Exceeded. Retrying #{label}" if @debug 89 | else 90 | t2 = Time.now 91 | recs = parse_json(response.body, label, t2-t1) 92 | break 93 | end 94 | end 95 | if limit 96 | recs[0,limit] 97 | else 98 | recs 99 | end 100 | } 101 | rescue Timeout::Error 102 | $stderr.puts "#{self.class.name} lookup timed out: #{label}" 103 | recs 104 | end 105 | 106 | private 107 | 108 | # parses the response of circl's JSON reply to generate an array of PDNSResult 109 | def parse_json(page,query,response_time=0) 110 | res = [] 111 | page.split(/\n/).each do |line| 112 | row = JSON.parse(line) 113 | firstseen = Time.at(row['time_first'].to_i) 114 | lastseen = Time.at(row['time_last'].to_i) 115 | res << PDNSResult.new(self.class.name,response_time, 116 | row['rrname'], row['rdata'], row['rrtype'], 0, 117 | firstseen, lastseen, row['count'], 'yellow') 118 | end 119 | res 120 | rescue Exception => e 121 | $stderr.puts "#{self.class.name} Exception: #{e}" 122 | raise e 123 | end 124 | 125 | end 126 | CIRCL = PassiveDNS::Provider::Circl 127 | end 128 | end 129 | -------------------------------------------------------------------------------- /lib/passivedns/client/provider/dnsdb.rb: -------------------------------------------------------------------------------- 1 | # DESCRIPTION: this is a module for pdns.rb, primarily used by pdnstool.rb, to query the Farsight Security passive DNS database 2 | # details on the API are at https://api.dnsdb.info/ 3 | # to request an API key, please email dnsdb-api at farsightsecurity dot com. 4 | require 'net/http' 5 | require 'net/https' 6 | 7 | module PassiveDNS #:nodoc: don't document this 8 | # The Provider module contains all the Passive DNS provider client code 9 | module Provider 10 | # Queries FarSight's passive DNS database 11 | class DNSDB < PassiveDB 12 | # Sets the modules self-reported name to "DNSDB" 13 | def self.name 14 | "DNSDB" 15 | end 16 | # Sets the configuration section name to "dnsdb" 17 | def self.config_section_name 18 | "dnsdb" 19 | end 20 | # Sets the command line database argument to "d" 21 | def self.option_letter 22 | "d" 23 | end 24 | 25 | # :debug enables verbose logging to standard output 26 | attr_accessor :debug 27 | # === Options 28 | # * :debug Sets the debug flag for the module 29 | # * "APIKEY" REQUIRED: The API key associated with DNSDB 30 | # * "URL" Alternate url for testing. Defaults to "https://api.dnsdb.info/lookup" 31 | # 32 | # === Example Instantiation 33 | # 34 | # options = { 35 | # :debug => true, 36 | # "APIKEY" => "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", 37 | # "URL" => "https://api.dnsdb.info/lookup" 38 | # } 39 | # 40 | # PassiveDNS::Provider::DNSDB.new(options) 41 | # 42 | def initialize(options={}) 43 | @debug = options[:debug] || false 44 | @timeout = options[:timeout] || 20 45 | @key = options["APIKEY"] || raise("APIKEY option required for #{self.class}") 46 | @base = options["URL"] || "https://api.dnsdb.info/lookup" 47 | end 48 | 49 | # Takes a label (either a domain or an IP address) and returns 50 | # an array of PassiveDNS::PDNSResult instances with the answers to the query 51 | def lookup(label, limit=nil) 52 | $stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug 53 | Timeout::timeout(@timeout) { 54 | url = nil 55 | if label =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}(\/\d{1,2})?$/ 56 | label = label.gsub(/\//,',') 57 | url = "#{@base}/rdata/ip/#{label}" 58 | else 59 | url = "#{@base}/rrset/name/#{label}" 60 | end 61 | url = URI.parse url 62 | http = Net::HTTP.new(url.host, url.port) 63 | http.use_ssl = (url.scheme == 'https') 64 | http.verify_mode = OpenSSL::SSL::VERIFY_NONE 65 | http.verify_depth = 5 66 | path = url.path 67 | if limit 68 | path << "?limit=#{limit}" 69 | end 70 | request = Net::HTTP::Get.new(path) 71 | request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}") 72 | request.add_field("X-API-Key", @key) 73 | request.add_field("Accept", "application/json") 74 | t1 = Time.now 75 | response = http.request(request) 76 | if response.code.to_i == 404 77 | $stderr.puts "DEBUG: empty response from server" if @debug 78 | return 79 | end 80 | t2 = Time.now 81 | $stderr.puts response.body if @debug 82 | parse_json(response.body,t2-t1) 83 | } 84 | rescue Timeout::Error 85 | $stderr.puts "#{self.class.name} lookup timed out: #{label}" 86 | end 87 | 88 | private 89 | 90 | # parses the response of DNSDB's JSON reply to generate an array of PDNSResult 91 | def parse_json(page,response_time) 92 | res = [] 93 | raise "Error: unable to parse request" if page =~ /Error: unable to parse request/ 94 | rows = page.split(/\n/) 95 | rows.each do |row| 96 | record = JSON.parse(row) 97 | answers = record['rdata'] 98 | answers = [record['rdata']] if record['rdata'].class == String 99 | query = record['rrname'].gsub!(/\.$/,'') 100 | rrtype = record['rrtype'] 101 | firstseen = Time.at(record['time_first'].to_i) 102 | lastseen = Time.at(record['time_last'].to_i) 103 | count = record['count'] 104 | 105 | answers.each do |answer| 106 | answer.gsub!(/\.$/,'') 107 | if record['time_first'] 108 | res << PDNSResult.new(self.class.name,response_time,query,answer,rrtype, 109 | 0,firstseen,lastseen,count, 'yellow') 110 | else 111 | res << PDNSResult.new(self.class.name,response_time,query,answer,rrtype) 112 | end 113 | end 114 | end 115 | res 116 | rescue Exception => e 117 | $stderr.puts "#{self.class.name} Exception: #{e}" 118 | $stderr.puts page 119 | raise e 120 | end 121 | end 122 | end 123 | end 124 | -------------------------------------------------------------------------------- /lib/passivedns/client/provider/osc.rb: -------------------------------------------------------------------------------- 1 | # DESCRIPTION: this is a module for Open Source Context's PassiveDNS archive. 2 | require 'net/http' 3 | require 'net/https' 4 | 5 | module PassiveDNS #:nodoc: don't document this 6 | # The Provider module contains all the Passive DNS provider client code 7 | module Provider 8 | # Queries OSContext's passive DNS database 9 | class OSC < PassiveDB 10 | # Sets the modules self-reported name to "OSC" 11 | def self.name 12 | "OSC" 13 | end 14 | # Sets the configuration section name to "osc" 15 | def self.config_section_name 16 | "osc" 17 | end 18 | # Sets the command line database argument to "d" 19 | def self.option_letter 20 | "o" 21 | end 22 | 23 | # :debug enables verbose logging to standard output 24 | attr_accessor :debug 25 | # === Options 26 | # * :debug Sets the debug flag for the module 27 | # * "APIKEY" REQUIRED: The API key associated with OSC 28 | # * "URL" Alternate url for testing. Defaults to "https://api.oscontext.com/api/v2/domainsquery" 29 | # 30 | # === Example Instantiation 31 | # 32 | # options = { 33 | # :debug => true, 34 | # "APIKEY" => "0123456789abcdef0123456789abcdef01234567", 35 | # "URL" => "https://api.oscontext.com/api/v2/domainsquery" 36 | # } 37 | # 38 | # PassiveDNS::Provider::OSC.new(options) 39 | # 40 | def initialize(options={}) 41 | @debug = options[:debug] || false 42 | @timeout = options[:timeout] || 20 43 | @token = options["APIKEY"] || raise("APIKEY option required for #{self.class}") 44 | @url = options["URL"] || "https://api.oscontext.com/api/v2/domainsquery" 45 | end 46 | 47 | # Takes a label (either a domain or an IP address) and returns 48 | # an array of PassiveDNS::PDNSResult instances with the answers to the query 49 | def lookup(label, limit=nil) 50 | $stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug 51 | Timeout::timeout(@timeout) { 52 | if label =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}(\/\d{1,2})?$/ 53 | url = @url+"?q=value_ip:#{label}&size=250&token=#{@token}" 54 | else 55 | url = @url+"?q=qname%3A#{label}&size=250&token=#{@token}" 56 | end 57 | 58 | url = URI.parse url 59 | 60 | $stderr.puts "--DEBUG: #{self.class.name} url = #{url}" if @debug 61 | 62 | http = Net::HTTP.new(url.host, url.port) 63 | http.use_ssl = (url.scheme == 'https') 64 | http.verify_mode = OpenSSL::SSL::VERIFY_NONE 65 | http.verify_depth = 5 66 | 67 | request = Net::HTTP::Get.new(url) 68 | request.add_field("content-type","application/x-www-form-urlencoded") 69 | request.add_field("referer", "clitool") 70 | request.add_field("accept-encoding", "gzip") 71 | t1 = Time.now 72 | response = http.request(request) 73 | if response.code.to_i == 404 74 | $stderr.puts "DEBUG: empty response from server" if @debug 75 | return 76 | end 77 | t2 = Time.now 78 | #$stderr.puts response.body if @debug 79 | parse_json(response.body,t2-t1) 80 | } 81 | rescue Timeout::Error 82 | $stderr.puts "#{self.class.name} lookup timed out: #{label}" 83 | end 84 | 85 | private 86 | 87 | # parses the response of OSC's JSON reply to generate an array of PDNSResult 88 | def parse_json(page,response_time) 89 | res = [] 90 | raise "Error: unable to parse request" if page =~ /Error: unable to parse request/ 91 | 92 | data = JSON.parse(page) 93 | if data['results'] 94 | data['results'].each do |row| 95 | if row['qtype'].to_i == 1 96 | firstseen = Time.parse(row['date']) 97 | if row['last_seen'] 98 | lastseen = Time.parse(row['last_seen']) 99 | else 100 | lastseen = nil 101 | end 102 | res << PDNSResult.new( 103 | self.class.name, 104 | response_time, 105 | row['domain'], 106 | row['value'], 107 | 'A', 108 | nil, 109 | firstseen, 110 | lastseen, 111 | 'amber') 112 | elsif row['type'] == "soa_email" 113 | firstseen = Time.parse(row['date']) 114 | 115 | if row['last_seen'] 116 | lastseen = Time.parse(row['last_seen']) 117 | else 118 | lastseen = nil 119 | end 120 | 121 | res << PDNSResult.new( 122 | self.class.name, 123 | response_time, 124 | row['domain'], 125 | row['value'], 126 | 'SOA', 127 | nil, 128 | firstseen, 129 | lastseen, 130 | 'amber') 131 | 132 | 133 | end 134 | end 135 | end 136 | 137 | res 138 | rescue Exception => e 139 | $stderr.puts "#{self.class.name} Exception: #{e}" 140 | # $stderr.puts page 141 | raise e 142 | end 143 | end 144 | end 145 | end 146 | -------------------------------------------------------------------------------- /lib/passivedns/client/provider/passivetotal.rb: -------------------------------------------------------------------------------- 1 | # DESCRIPTION: Module to query PassiveTotal's passive DNS repository 2 | 3 | require 'net/http' 4 | require 'net/https' 5 | require 'openssl' 6 | require 'pp' 7 | 8 | module PassiveDNS #:nodoc: don't document this 9 | # The Provider module contains all the Passive DNS provider client code 10 | module Provider 11 | # Queries PassiveTotal's passive DNS database 12 | class PassiveTotal < PassiveDB 13 | # Sets the modules self-reported name to "PassiveTotal" 14 | def self.name 15 | "PassiveTotal" 16 | end 17 | # Sets the configuration section name to "passivetotal" 18 | def self.config_section_name 19 | "passivetotal" 20 | end 21 | # Sets the command line database argument to "p" 22 | def self.option_letter 23 | "p" 24 | end 25 | 26 | # :debug enables verbose logging to standard output 27 | attr_accessor :debug 28 | # === Options 29 | # * :debug Sets the debug flag for the module 30 | # * "USERNAME" REQUIRED: The username for the associated API key 31 | # * "APIKEY" REQUIRED: The API key associated with PassiveTotal 32 | # * "URL" Alternate url for testing. Defaults to "https://api.passivetotal.org/v2/dns/passive" 33 | # 34 | # === Example Instantiation 35 | # 36 | # options = { 37 | # :debug => true, 38 | # "USERNAME" => "tom@example.com", 39 | # "APIKEY" => "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", 40 | # "URL" => "https://api.passivetotal.org/v2/dns/passive" 41 | # } 42 | 43 | # or 44 | # 45 | # options = { 46 | # :debug => true, 47 | # "USERNAME" => "tom@example.com" 48 | # "APIKEY" => "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", 49 | # } 50 | # 51 | # then 52 | # 53 | # PassiveDNS::Provider::PassiveTotal.new(options) 54 | # 55 | def initialize(options={}) 56 | @debug = options[:debug] || false 57 | @timeout = options[:timeout] || 20 58 | @username = options["USERNAME"] || raise("#{self.class.name} requires a USERNAME") 59 | @apikey = options["APIKEY"] || raise("#{self.class.name} requires an APIKEY") 60 | @url = options["URL"] || "https://api.passivetotal.org/v2/dns/passive" 61 | end 62 | 63 | # Takes a label (either a domain or an IP address) and returns 64 | # an array of PassiveDNS::PDNSResult instances with the answers to the query 65 | def lookup(label, limit=nil) 66 | $stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug 67 | Timeout::timeout(@timeout) { 68 | url = @url+"?query=#{label}" 69 | $stderr.puts "DEBUG: #{self.class.name} url = #{url}" if @debug 70 | url = URI.parse url 71 | http = Net::HTTP.new(url.host, url.port) 72 | http.use_ssl = (url.scheme == 'https') 73 | http.verify_mode = OpenSSL::SSL::VERIFY_NONE 74 | http.verify_depth = 5 75 | request = Net::HTTP::Get.new(url.request_uri) 76 | request.basic_auth(@username, @apikey) 77 | request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}") 78 | #request.set_form_data({"api_key" => @apikey, "query" => label}) 79 | t1 = Time.now 80 | response = http.request(request) 81 | t2 = Time.now 82 | recs = parse_json(response.body, label, t2-t1) 83 | if limit 84 | recs[0,limit] 85 | else 86 | recs 87 | end 88 | } 89 | rescue Timeout::Error 90 | $stderr.puts "#{self.class.name} lookup timed out: #{label}" 91 | end 92 | 93 | private 94 | 95 | # parses the response of passivetotals's JSON reply to generate an array of PDNSResult 96 | def parse_json(page,query,response_time=0) 97 | res = [] 98 | data = JSON.parse(page) 99 | pp data 100 | if data['message'] 101 | raise "#{self.class.name} Error: #{data['message']}" 102 | end 103 | query = data['queryValue'] 104 | if data['results'] 105 | data['results'].each do |row| 106 | first_seen = (row['firstSeen'] == "None") ? nil : Time.parse(row['firstSeen']+" +0000") 107 | last_seen = (row['lastSeen'] == "None") ? nil : Time.parse(row['lastSeen']+" +0000") 108 | value = row['resolve'] 109 | source = row['source'].join(",") 110 | res << PDNSResult.new(self.class.name+"/"+source,response_time, 111 | query, value, "A", 0, first_seen, last_seen, 'yellow') 112 | end 113 | end 114 | res 115 | rescue Exception => e 116 | $stderr.puts "#{self.class.name} Exception: #{e}" 117 | raise e 118 | end 119 | end 120 | end 121 | end 122 | -------------------------------------------------------------------------------- /lib/passivedns/client/provider/riskiq.rb: -------------------------------------------------------------------------------- 1 | # DESCRIPTION: Module to query PassiveTotal's passive DNS repository 2 | 3 | require 'net/http' 4 | require 'net/https' 5 | require 'openssl' 6 | require 'pp' 7 | 8 | module PassiveDNS #:nodoc: don't document this 9 | # The Provider module contains all the Passive DNS provider client code 10 | module Provider 11 | # Queries RiskIQ's passive DNS database 12 | class RiskIQ < PassiveDB 13 | # Sets the modules self-reported name to "RiskIQ" 14 | def self.name 15 | "RiskIQ" 16 | end 17 | # Sets the configuration section name to "riskiq" 18 | def self.config_section_name 19 | "riskiq" 20 | end 21 | # Sets the command line database argument to "r" 22 | def self.option_letter 23 | "r" 24 | end 25 | 26 | # :debug enables verbose logging to standard output 27 | attr_accessor :debug 28 | # === Options 29 | # * :debug Sets the debug flag for the module 30 | # * "API_TOKEN" REQUIRED: User name associated with your RiskIQ account 31 | # * "API_PRIVATE_KEY" REQUIRED: Password associated with your RiskIQ account 32 | # * "API_SERVER" Alternate server for testing. Defaults to "ws.riskiq.net" 33 | # * "API_VERSION" Alternate version of the API to test. Defaults to "V1" 34 | # 35 | # === Example Instantiation 36 | # 37 | # options = { 38 | # :debug => true, 39 | # "API_TOKEN" => "riskiq_token", 40 | # "API_PRIVATE_KEY" => "riskiq_private_key", 41 | # "API_SERVER" => "ws.riskiq.net", 42 | # "API_VERSION" => "v1" 43 | # } 44 | # 45 | # PassiveDNS::Provider::RiskIQ.new(options) 46 | # 47 | def initialize(options={}) 48 | @debug = options[:debug] || false 49 | @timeout = options[:timeout] || 20 50 | @token = options["API_TOKEN"] || raise("#{self.class.name} requires an API_TOKEN") 51 | @privkey = options["API_PRIVATE_KEY"] || raise("#{self.class.name} requires an API_PRIVATE_KEY") 52 | @version = options["API_VERSION"] || "v1" 53 | @server = options["API_SERVER"] || api_settings[@version][:server] 54 | @url = "https://#{@server}/#{@version}" 55 | end 56 | 57 | # Takes a label (either a domain or an IP address) and returns 58 | # an array of PassiveDNS::PDNSResult instances with the answers to the query 59 | def lookup(label, limit=nil) 60 | $stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug 61 | Timeout::timeout(@timeout) { 62 | url = nil 63 | params = {"rrType" => "", "maxResults" => limit || 1000} 64 | 65 | if label =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/ 66 | url = @url+"/dns/data" 67 | params["ip"] = label 68 | else 69 | resource = api_settings[@version][:resource] 70 | param = api_settings[@version][:param] 71 | url = @url+"/dns/#{resource}" 72 | params[param] = label 73 | end 74 | url << "?" 75 | params.each do |k,v| 76 | url << "#{k}=#{v}&" 77 | end 78 | url.gsub!(/\&$/,"") 79 | 80 | $stderr.puts "DEBUG: #{self.class.name} url = #{url}" if @debug 81 | url = URI.parse url 82 | http = Net::HTTP.new(url.host, url.port) 83 | http.use_ssl = (url.scheme == 'https') 84 | http.verify_mode = OpenSSL::SSL::VERIFY_NONE 85 | http.verify_depth = 5 86 | request = Net::HTTP::Get.new(url.request_uri) 87 | request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}") 88 | request.add_field('Accept', 'Application/JSON') 89 | request.add_field('Content-Type', 'Application/JSON') 90 | request.basic_auth(@token, @privkey) 91 | t1 = Time.now 92 | response = http.request(request) 93 | t2 = Time.now 94 | recs = parse_json(response.body, label, t2-t1) 95 | if limit 96 | recs[0,limit] 97 | else 98 | recs 99 | end 100 | } 101 | rescue Timeout::Error 102 | $stderr.puts "#{self.class.name} lookup timed out: #{label}" 103 | end 104 | 105 | private 106 | 107 | def api_settings 108 | @api_settings ||= { 109 | 'v1' => { server: "ws.riskiq.net", resource: 'name', param: 'name' }, 110 | 'v2' => { server: "api.passivetotal.org", resource: 'passive', param: 'query' } 111 | } 112 | end 113 | 114 | # parses the response of riskiq's JSON reply to generate an array of PDNSResult 115 | def parse_json(page,query,response_time=0) 116 | res = [] 117 | data = JSON.parse(page) 118 | if data['message'] 119 | if data['message'] =~ /quota_exceeded/ 120 | $stderr.puts "ERROR: quota exceeded." 121 | return res 122 | end 123 | end 124 | if data['records'] 125 | data['records'].each do |record| 126 | name = record['name'].gsub!(/\.$/,'') 127 | type = record['rrtype'] 128 | last_seen = Time.parse(record['lastSeen']) 129 | first_seen = Time.parse(record['firstSeen']) 130 | count = record['count'] 131 | record['data'].each do |datum| 132 | datum.gsub!(/\.$/,'') 133 | res << PDNSResult.new(self.class.name,response_time, 134 | name, datum, type, 0, first_seen, last_seen, count, 'yellow') 135 | end 136 | end 137 | end 138 | res 139 | rescue Exception => e 140 | $stderr.puts "#{self.class.name} Exception: #{e}" 141 | raise e 142 | end 143 | 144 | end 145 | end 146 | end 147 | -------------------------------------------------------------------------------- /lib/passivedns/client/provider/virustotal.rb: -------------------------------------------------------------------------------- 1 | # DESCRIPTION: this is a module for pdns.rb, primarily used by pdnstool.rb, to query VirusTotal's passive DNS database 2 | require 'net/http' 3 | require 'net/https' 4 | require 'openssl' 5 | 6 | module PassiveDNS #:nodoc: don't document this 7 | # The Provider module contains all the Passive DNS provider client code 8 | module Provider 9 | # Queries VirusTotal's passive DNS database 10 | class VirusTotal < PassiveDB 11 | # Sets the modules self-reported name to "VirusTotal" 12 | def self.name 13 | "VirusTotal" 14 | end 15 | # Sets the configuration section name to "virustotal" 16 | def self.config_section_name 17 | "virustotal" 18 | end 19 | # Sets the command line database argument to "v" 20 | def self.option_letter 21 | "v" 22 | end 23 | 24 | # :debug enables verbose logging to standard output 25 | attr_accessor :debug 26 | 27 | # === Options 28 | # * :debug Sets the debug flag for the module 29 | # * "APIKEY" Mandatory. API Key associated with your VirusTotal account 30 | # * "URL" Alternate url for testing. Defaults to https://www.virustotal.com/vtapi/v2/ 31 | # 32 | # === Example Instantiation 33 | # 34 | # options = { 35 | # :debug => true, 36 | # "APIKEY" => "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", 37 | # "URL" => "https://www.virustotal.com/vtapi/v2/" 38 | # } 39 | # 40 | # PassiveDNS::Provider::VirusTotal.new(options) 41 | # 42 | def initialize(options={}) 43 | @debug = options[:debug] || false 44 | @timeout = options[:timeout] || 20 45 | @apikey = options["APIKEY"] || raise("#{self.class.name} requires an APIKEY. See README.md") 46 | @url = options["URL"] || "https://www.virustotal.com/vtapi/v2/" 47 | end 48 | 49 | # Takes a label (either a domain or an IP address) and returns 50 | # an array of PassiveDNS::PDNSResult instances with the answers to the query 51 | def lookup(label, limit=nil) 52 | $stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug 53 | Timeout::timeout(@timeout) { 54 | url = nil 55 | if label =~ /^[\d\.]+$/ 56 | url = "#{@url}ip-address/report?ip=#{label}&apikey=#{@apikey}" 57 | else 58 | url = "#{@url}domain/report?domain=#{label}&apikey=#{@apikey}" 59 | end 60 | $stderr.puts "DEBUG: #{self.class.name} url = #{url}" if @debug 61 | begin 62 | url = URI.parse url 63 | rescue URI::InvalidURIError 64 | $stderr.puts "ERROR: Invalid address: #{url}" 65 | return 66 | end 67 | http = Net::HTTP.new(url.host, url.port) 68 | http.use_ssl = (url.scheme == 'https') 69 | http.verify_mode = OpenSSL::SSL::VERIFY_NONE 70 | http.verify_depth = 5 71 | request = Net::HTTP::Get.new(url.path+"?"+url.query) 72 | request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}") 73 | t1 = Time.now 74 | response = http.request(request) 75 | if response.code.to_i == 204 76 | $stderr.puts "DEBUG: empty response from server" if @debug 77 | return 78 | end 79 | t2 = Time.now 80 | recs = parse_json(response.body, label, t2-t1) 81 | if limit 82 | recs[0,limit] 83 | else 84 | recs 85 | end 86 | } 87 | rescue Timeout::Error 88 | $stderr.puts "#{self.class.name} lookup timed out: #{label}" 89 | end 90 | 91 | private 92 | 93 | # parses the response of virustotal's JSON reply to generate an array of PDNSResult 94 | def parse_json(page,query,response_time=0) 95 | res = [] 96 | return res if !page 97 | data = JSON.parse(page) 98 | if data['resolutions'] 99 | data['resolutions'].each do |row| 100 | lastseen = Time.parse(row['last_resolved']+" +0000") 101 | if row['ip_address'] 102 | res << PDNSResult.new(self.class.name,response_time,query,row['ip_address'],'A',nil,nil,lastseen, 'yellow') 103 | elsif row['hostname'] 104 | res << PDNSResult.new(self.class.name,response_time,row['hostname'],query,'A',nil,nil,lastseen, 'yellow') 105 | end 106 | end 107 | end 108 | if data['response_code'] == 0 109 | $stderr.puts "DEBUG: server returned error: #{data['verbose_msg']}" if @debug 110 | end 111 | res 112 | rescue Exception => e 113 | $stderr.puts "VirusTotal Exception: #{e}" 114 | raise e 115 | end 116 | 117 | end 118 | end 119 | end 120 | -------------------------------------------------------------------------------- /lib/passivedns/client/state.rb: -------------------------------------------------------------------------------- 1 | require 'sqlite3' 2 | require 'yaml' 3 | require 'structformatter' 4 | 5 | module PassiveDNS # :nodoc: 6 | # struct to hold pending entries for query 7 | class PDNSQueueEntry < Struct.new(:query, :state, :level); end 8 | 9 | # holds state in memory of the queue to be queried, records returned, and the level of recursion 10 | class PDNSToolState 11 | # :debug enables verbose logging to standard output 12 | attr_accessor :debug 13 | # :level is the recursion depth 14 | attr_reader :level 15 | 16 | # creates a new, blank PDNSToolState instance 17 | def initialize 18 | @queue = [] 19 | @recs = [] 20 | @level = 0 21 | end 22 | 23 | # returns the next record 24 | def next_result 25 | @recs.each do |rec| 26 | yield rec 27 | end 28 | end 29 | 30 | # adds the record to the list of records received and tries to add the answer and query back to the queue for future query 31 | def add_result(res) 32 | @recs << res 33 | add_query(res.answer,'pending') 34 | add_query(res.query,'pending') 35 | end 36 | 37 | # sets the state of a given query 38 | def update_query(query,state) 39 | @queue.each do |q| 40 | if q.query == query 41 | puts "update_query: #{query} (#{q.state}) -> (#{state})" if @debug 42 | q.state = state 43 | break 44 | end 45 | end 46 | end 47 | 48 | # returns the state of a provided query 49 | def get_state(query) 50 | @queue.each do |q| 51 | if q.query == query 52 | return q.state 53 | end 54 | end 55 | false 56 | end 57 | 58 | # adding a query to the queue of things to be queried, but only if the query isn't already queued or answered 59 | def add_query(query,state,level=@level+1) 60 | if query =~ /^\d+ \w+\./ 61 | query = query.split(/ /,2)[1] 62 | end 63 | return if get_state(query) 64 | puts "Adding query: #{query}, #{state}, #{level}" if @debug 65 | @queue << PDNSQueueEntry.new(query,state,level) 66 | end 67 | 68 | # returns each query waiting on the queue 69 | def each_query(max_level=20) 70 | @queue.each do |q| 71 | if q.state == 'pending' or q.state == 'failed' 72 | @level = q.level 73 | q.state = 'queried' 74 | if q.level < max_level 75 | yield q.query 76 | end 77 | end 78 | end 79 | end 80 | 81 | # transforms a set of results into GDF syntax 82 | def to_gdf 83 | output = "nodedef> name,description VARCHAR(12),color,style\n" 84 | # IP "$node2,,white,1" 85 | # domain "$node2,,gray,2" 86 | # Struct.new(:query, :answer, :rrtype, :ttl, :firstseen, :lastseen) 87 | colors = {"MX" => "green", "A" => "blue", "CNAME" => "pink", "NS" => "red", "SOA" => "white", "PTR" => "purple", "TXT" => "brown"} 88 | nodes = {} 89 | edges = {} 90 | next_result do |i| 91 | if i 92 | nodes[i.query + ",,gray,2"] = true 93 | if i.answer =~ /[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/ then 94 | nodes[i.answer + ",,white,1"] = true 95 | else 96 | nodes[i.answer + ",,gray,2"] = true 97 | end 98 | color = colors[i.rrtype] 99 | color ||= "blue" 100 | edges[i.query + "," + i.answer + "," + color] = true 101 | end 102 | end 103 | nodes.each do |i,j| 104 | output += i+"\n" 105 | end 106 | output += "edgedef> node1,node2,color\n" 107 | edges.each do |i,j| 108 | output += i+"\n" 109 | end 110 | output 111 | end 112 | 113 | # transforms a set of results into graphviz syntax 114 | def to_graphviz 115 | colors = {"MX" => "green", "A" => "blue", "CNAME" => "pink", "NS" => "red", "SOA" => "white", "PTR" => "purple", "TXT" => "brown"} 116 | output = "graph pdns {\n" 117 | nodes = {} 118 | next_result do |l| 119 | if l 120 | unless nodes[l.query] 121 | output += " \"#{l.query}\" [shape=ellipse, style=filled, color=gray];\n" 122 | if l.answer =~ /^\d{3}\.\d{3}\.\d{3}\.\d{3}$/ 123 | output += " \"#{l.answer}\" [shape=box, style=filled, color=white];\n" 124 | else 125 | output += " \"#{l.answer}\" [shape=ellipse, style=filled, color=gray];\n" 126 | end 127 | nodes[l.query] = true 128 | end 129 | output += " \"#{l.query}\" -- \"#{l.answer}\" [color=#{colors[l.rrtype]}];\n" 130 | end 131 | end 132 | output += "}\n" 133 | end 134 | 135 | # transforms a set of results into graphml syntax 136 | def to_graphml 137 | output = ' 138 | 142 | 143 | ' 144 | nodes = {} 145 | edges = {} 146 | next_result do |r| 147 | if r 148 | output += " \n" unless nodes["#{r.query}"] 149 | nodes[r.query] = true 150 | output += " \n" unless nodes["#{r.answer}"] 151 | nodes[r.answer] = true 152 | output += " \n" unless edges["#{r.query}|#{r.answer}"] 153 | end 154 | end 155 | output += ''+"\n" 156 | end 157 | 158 | # transforms a set of results into XML 159 | def to_xml 160 | output = ''+"\n" 161 | output += "\n" 162 | output += " \n" 163 | next_result do |rec| 164 | output += " "+rec.to_xml+"\n" 165 | end 166 | output += " \n" 167 | output += "\n" 168 | end 169 | 170 | # transforms a set of results into YAML 171 | def to_yaml 172 | output = "" 173 | next_result do |rec| 174 | output += rec.to_yaml+"\n" 175 | end 176 | output 177 | end 178 | 179 | # transforms a set of results into JSON 180 | def to_json 181 | output = "[\n" 182 | sep = "" 183 | next_result do |rec| 184 | output += sep 185 | output += rec.to_json 186 | sep = ",\n" 187 | end 188 | output += "\n]\n" 189 | end 190 | 191 | # transforms a set of results into a text string 192 | def to_s(sep="\t") 193 | output = "" 194 | next_result do |rec| 195 | output += rec.to_s(sep)+"\n" 196 | end 197 | output 198 | end 199 | end # class PDNSToolState 200 | 201 | 202 | # creates persistence to the tool state by leveraging SQLite3 203 | class PDNSToolStateDB < PDNSToolState 204 | attr_reader :level 205 | # creates an SQLite3-based Passive DNS Client state 206 | # only argument is the filename of the sqlite3 database 207 | def initialize(sqlitedb=nil) 208 | @debug = false 209 | puts "PDNSToolState initialize #{sqlitedb}" if @debug 210 | @level = 0 211 | @sqlitedb = sqlitedb 212 | raise "Cannot use this class without a database file" unless @sqlitedb 213 | unless File.exist?(@sqlitedb) 214 | newdb = true 215 | end 216 | @sqlitedbh = SQLite3::Database.new(@sqlitedb) 217 | if newdb 218 | create_tables 219 | end 220 | res = @sqlitedbh.execute("select min(level) from queue where state = 'pending'") 221 | if res 222 | res.each do |row| 223 | @level = row[0].to_i 224 | puts "changed @level = #{@level}" if @debug 225 | end 226 | end 227 | end 228 | 229 | # creates the sqlite3 tables needed to track the state of this tool as itqueries and recurses 230 | def create_tables 231 | puts "creating tables" if @debug 232 | @sqlitedbh.execute("create table results (query, answer, rrtype, ttl, firstseen, lastseen, ts REAL)") 233 | @sqlitedbh.execute("create table queue (query, state, level INTEGER, ts REAL)") 234 | @sqlitedbh.execute("create index residx on results (ts)") 235 | @sqlitedbh.execute("create unique index queue_unique on queue (query)") 236 | @sqlitedbh.execute("create index queue_level_idx on queue (level)") 237 | @sqlitedbh.execute("create index queue_state_idx on queue (state)") 238 | end 239 | 240 | # returns the next record 241 | def next_result 242 | rows = @sqlitedbh.execute("select query, answer, rrtype, ttl, firstseen, lastseen from results order by ts") 243 | rows.each do |row| 244 | yield PDNSResult.new(*row) 245 | end 246 | end 247 | 248 | # adds the record to the list of records received and tries to add the answer and query back to the queue for future query 249 | def add_result(res) 250 | puts "adding result: #{res.to_s}" if @debug 251 | curtime = Time.now().to_f 252 | @sqlitedbh.execute("insert into results values ('#{res.query}','#{res.answer}','#{res.rrtype}','#{res.ttl}','#{res.firstseen}','#{res.lastseen}',#{curtime})") 253 | 254 | add_query(res.answer,'pending') 255 | add_query(res.query,'pending') 256 | end 257 | 258 | # adding a query to the queue of things to be queried, but only if the query isn't already queued or answered 259 | def add_query(query,state,level=@level+1) 260 | return if get_state(query) 261 | curtime = Time.now().to_f 262 | begin 263 | puts "add_query(#{query},#{state},level=#{level})" if @debug 264 | @sqlitedbh.execute("insert into queue values ('#{query}','#{state}',#{level},#{curtime})") 265 | rescue 266 | end 267 | end 268 | 269 | # sets the state of a given query 270 | def update_query(query,state) 271 | @sqlitedbh.execute("update queue set state = '#{state}' where query = '#{query}'") 272 | end 273 | 274 | # returns each query waiting on the queue 275 | def get_state(query) 276 | rows = @sqlitedbh.execute("select state from queue where query = '#{query}'") 277 | if rows 278 | rows.each do |row| 279 | return row[0] 280 | end 281 | end 282 | false 283 | end 284 | 285 | # returns each query waiting on the queue 286 | def each_query(max_level=20) 287 | puts "each_query max_level=#{max_level} curlevel=#{@level}" if @debug 288 | rows = @sqlitedbh.execute("select query, state, level from queue where state = 'failed' or state = 'pending' order by level limit 1") 289 | if rows 290 | rows.each do |row| 291 | query,state,level = row 292 | puts " #{query},#{state},#{level}" if @debug 293 | if level < max_level 294 | update_query(query,'queried') 295 | yield query 296 | end 297 | end 298 | end 299 | end 300 | end # class PDNSToolStateDB 301 | end 302 | -------------------------------------------------------------------------------- /lib/passivedns/client/version.rb: -------------------------------------------------------------------------------- 1 | module PassiveDNS # :nodoc: 2 | # coodinates the lookups accross all configured PassiveDNS providers 3 | class Client 4 | # version of PassiveDNS::Client 5 | VERSION = "2.1.14" 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /passivedns-client.gemspec: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | lib = File.expand_path('../lib', __FILE__) 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 4 | require 'passivedns/client/version' 5 | 6 | Gem::Specification.new do |spec| 7 | spec.name = "passivedns-client" 8 | spec.version = PassiveDNS::Client::VERSION 9 | spec.authors = ["chrislee35"] 10 | spec.email = ["rubygems@chrislee.dhs.org"] 11 | spec.description = %q{This provides interfaces to various passive DNS databases to do the query and to normalize the responses. The query tool also allows for recursive queries, using an SQLite3 database to keep state.} 12 | spec.summary = %q{Query passive DNS databases} 13 | spec.homepage = "https://github.com/chrislee35/passivedns-client" 14 | spec.license = "MIT" 15 | 16 | spec.files = `git ls-files`.split($/) 17 | spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) } 18 | spec.test_files = spec.files.grep(%r{^(test|spec|features)/}) 19 | spec.require_paths = ["lib"] 20 | 21 | spec.add_runtime_dependency 'json' 22 | spec.add_runtime_dependency 'sqlite3' 23 | spec.add_runtime_dependency 'structformatter' 24 | spec.add_runtime_dependency 'configparser' 25 | spec.add_development_dependency "minitest" 26 | spec.add_development_dependency "bundler" 27 | spec.add_development_dependency "rake" 28 | 29 | #spec.signing_key = "#{File.dirname(__FILE__)}/../gem-private_key.pem" 30 | #spec.cert_chain = ["#{File.dirname(__FILE__)}/../gem-public_cert.pem"] 31 | end 32 | -------------------------------------------------------------------------------- /test/helper.rb: -------------------------------------------------------------------------------- 1 | require 'minitest/autorun' 2 | require 'minitest/test' 3 | require 'minitest/unit' 4 | require 'minitest/pride' 5 | include MiniTest::Assertions 6 | require File.expand_path('../../lib/passivedns/client.rb', __FILE__) -------------------------------------------------------------------------------- /test/test_cli.rb: -------------------------------------------------------------------------------- 1 | unless Kernel.respond_to?(:require_relative) 2 | module Kernel 3 | def require_relative(path) 4 | require File.join(File.dirname(caller[0]), path.to_str) 5 | end 6 | end 7 | end 8 | 9 | require_relative 'helper' 10 | require 'configparser' 11 | require_relative '../lib/passivedns/client/cli.rb' 12 | 13 | class TestCLI < Minitest::Test 14 | def test_letter_map 15 | letter_map = PassiveDNS::CLI.get_letter_map 16 | assert_equal("cdoprv", letter_map.keys.sort.join("")) 17 | end 18 | 19 | def test_help_text 20 | helptext = PassiveDNS::CLI.run(["--help"]) 21 | helptext.gsub!(/Usage: .*?\[/, "Usage: [") 22 | assert_equal( 23 | "Usage: [-d [cdoprv]] [-g|-v|-m|-c|-x|-y|-j|-t] [-os ] [-f ] [-r#|-w#|-v] [-l ] [--config ] 24 | Passive DNS Providers 25 | -dcdoprv uses all of the available passive dns database 26 | -dc use CIRCL 27 | -dd use DNSDB 28 | -do use OSC 29 | -dp use PassiveTotal 30 | -dr use RiskIQ 31 | -dv use VirusTotal 32 | -dvr uses VirusTotal and RiskIQ (for example) 33 | 34 | Output Formatting 35 | -g link-nodal GDF visualization definition 36 | -z link-nodal graphviz visualization definition 37 | -m link-nodal graphml visualization definition 38 | -c CSV 39 | -x XML 40 | -y YAML 41 | -j JSON 42 | -t ASCII text (default) 43 | -s specifies a field separator for text output, default is tab 44 | 45 | State and Recursion 46 | -f[file] specifies a sqlite3 database used to read the current state - useful for large result sets and generating graphs of previous runs. 47 | -r# specifies the levels of recursion to pull. **WARNING** This is quite taxing on the pDNS servers, so use judiciously (never more than 3 or so) or find yourself blocked! 48 | -w# specifies the amount of time to wait, in seconds, between queries (Default: 0) 49 | -l limits the number of records returned per passive dns database queried. 50 | 51 | Specifying a Configuration File 52 | --config specifies a config file. default: #{ENV['HOME']}/.passivedns-client 53 | 54 | Getting Help 55 | -h hello there. This option produces this helpful help information on how to access help. 56 | -v debugging information 57 | ", helptext) 58 | end 59 | 60 | def test_provider_parsing 61 | options_target = { 62 | :pdnsdbs => ["virustotal"], 63 | :format => "text", 64 | :sep => "\t", 65 | :recursedepth => 1, 66 | :wait => 0, 67 | :res => nil, 68 | :debug => false, 69 | :sqlitedb => nil, 70 | :limit => nil, 71 | :help => false, 72 | :configfile => "#{ENV['HOME']}/.passivedns-client" 73 | } 74 | 75 | options, items = PassiveDNS::CLI.parse_command_line([]) 76 | assert_equal(options_target, options) 77 | assert_equal([], items) 78 | 79 | options_target[:pdnsdbs] = ["circl", "dnsdb", "riskiq"] 80 | options, items = PassiveDNS::CLI.parse_command_line(["-dcdr"]) 81 | assert_equal(options_target, options) 82 | assert_equal([], items) 83 | 84 | options_target[:pdnsdbs] = ["passivetotal", "virustotal"] 85 | options, items = PassiveDNS::CLI.parse_command_line(["-dpv"]) 86 | assert_equal(options_target, options) 87 | assert_equal([], items) 88 | 89 | end 90 | 91 | def test_output_parsing 92 | options_target = { 93 | :pdnsdbs => ["passivetotal", "virustotal"], 94 | :format => "text", 95 | :sep => "\t", 96 | :recursedepth => 1, 97 | :wait => 0, 98 | :res => nil, 99 | :debug => false, 100 | :sqlitedb => nil, 101 | :limit => nil, 102 | :help => false, 103 | :configfile => "#{ENV['HOME']}/.passivedns-client" 104 | } 105 | 106 | options_target[:sep] = "," 107 | options, items = PassiveDNS::CLI.parse_command_line(["-dpv", "-c", "8.8.8.8"]) 108 | assert_equal(options_target, options) 109 | assert_equal(["8.8.8.8"], items) 110 | 111 | options_target[:sep] = "|" 112 | options, items = PassiveDNS::CLI.parse_command_line(["-dpv", "-s", "|", "8.8.8.8"]) 113 | assert_equal(options_target, options) 114 | assert_equal(["8.8.8.8"], items) 115 | 116 | options_target[:sep] = "\t" 117 | 118 | options, items = PassiveDNS::CLI.parse_command_line(["-dpv", "-t", "8.8.8.8"]) 119 | assert_equal(options_target, options) 120 | assert_equal(["8.8.8.8"], items) 121 | 122 | options_target[:format] = "json" 123 | options, items = PassiveDNS::CLI.parse_command_line(["-dpv", "-j", "8.8.8.8"]) 124 | assert_equal(options_target, options) 125 | assert_equal(["8.8.8.8"], items) 126 | 127 | options_target[:format] = "xml" 128 | options, items = PassiveDNS::CLI.parse_command_line(["-dpv", "-x", "8.8.8.8"]) 129 | assert_equal(options_target, options) 130 | assert_equal(["8.8.8.8"], items) 131 | 132 | options_target[:format] = "yaml" 133 | options, items = PassiveDNS::CLI.parse_command_line(["-dpv", "-y", "8.8.8.8"]) 134 | assert_equal(options_target, options) 135 | assert_equal(["8.8.8.8"], items) 136 | 137 | options_target[:format] = "gdf" 138 | options, items = PassiveDNS::CLI.parse_command_line(["-dpv", "-g", "8.8.8.8"]) 139 | assert_equal(options_target, options) 140 | assert_equal(["8.8.8.8"], items) 141 | 142 | options_target[:format] = "graphviz" 143 | options, items = PassiveDNS::CLI.parse_command_line(["-dpv", "-z", "8.8.8.8"]) 144 | assert_equal(options_target, options) 145 | assert_equal(["8.8.8.8"], items) 146 | 147 | options_target[:format] = "graphml" 148 | options, items = PassiveDNS::CLI.parse_command_line(["-dpv", "-m", "8.8.8.8"]) 149 | assert_equal(options_target, options) 150 | assert_equal(["8.8.8.8"], items) 151 | 152 | options_target[:format] = "text" 153 | end 154 | 155 | def test_help_debug_parsing 156 | options_target = { 157 | :pdnsdbs => ["passivetotal", "virustotal"], 158 | :format => "text", 159 | :sep => "\t", 160 | :recursedepth => 1, 161 | :wait => 0, 162 | :res => nil, 163 | :debug => false, 164 | :sqlitedb => nil, 165 | :limit => nil, 166 | :help => true, 167 | :configfile => "#{ENV['HOME']}/.passivedns-client" 168 | } 169 | 170 | options, items = PassiveDNS::CLI.parse_command_line(["-dpv", "-h", "8.8.8.8"]) 171 | assert_equal(options_target, options) 172 | assert_equal(["8.8.8.8"], items) 173 | 174 | options_target[:debug] = true 175 | options, items = PassiveDNS::CLI.parse_command_line(["-dpv", "-h", "-v", "8.8.8.8"]) 176 | assert_equal(options_target, options) 177 | assert_equal(["8.8.8.8"], items) 178 | end 179 | 180 | def test_state_recursion_parsing 181 | options_target = { 182 | :pdnsdbs => ["passivetotal", "virustotal"], 183 | :format => "text", 184 | :sep => "\t", 185 | :recursedepth => 5, 186 | :wait => 30, 187 | :res => nil, 188 | :debug => false, 189 | :sqlitedb => "test.db", 190 | :limit => 10, 191 | :help => false, 192 | :configfile => "#{ENV['HOME']}/.passivedns-client" 193 | } 194 | 195 | options, items = PassiveDNS::CLI.parse_command_line(["-dpv", "-f", "test.db", "-r", "5", "-w", "30", "-l", "10", "8.8.8.8"]) 196 | assert_equal(options_target, options) 197 | assert_equal(["8.8.8.8"], items) 198 | end 199 | 200 | def test_configuration_file 201 | options_target = { 202 | :pdnsdbs => ["virustotal"], 203 | :format => "text", 204 | :sep => "\t", 205 | :recursedepth => 1, 206 | :wait => 0, 207 | :res => nil, 208 | :debug => false, 209 | :sqlitedb => nil, 210 | :limit => nil, 211 | :help => false, 212 | :configfile => "#{ENV['HOME']}/.passivedns-client" 213 | } 214 | 215 | options, items = PassiveDNS::CLI.parse_command_line(["--config", "#{ENV['HOME']}/.passivedns-client"]) 216 | assert_equal(options_target, options) 217 | assert_equal([], items) 218 | 219 | end 220 | end 221 | -------------------------------------------------------------------------------- /test/test_passivedns-client.rb: -------------------------------------------------------------------------------- 1 | unless Kernel.respond_to?(:require_relative) 2 | module Kernel 3 | def require_relative(path) 4 | require File.join(File.dirname(caller[0]), path.to_str) 5 | end 6 | end 7 | end 8 | 9 | require_relative 'helper' 10 | require 'configparser' 11 | 12 | class TestPassiveDnsQuery < Minitest::Test 13 | 14 | def setup 15 | configfile="#{ENV['HOME']}/.passivedns-client" 16 | @cp = ConfigParser.new(configfile) 17 | @class_map = {} 18 | PassiveDNS.constants.each do |const| 19 | if PassiveDNS.const_get(const).is_a?(Class) and PassiveDNS.const_get(const).superclass == PassiveDNS::PassiveDB 20 | @class_map[PassiveDNS.const_get(const).config_section_name] = PassiveDNS.const_get(const) 21 | end 22 | end 23 | end 24 | 25 | def test_instantiate_Nonexisting_Client 26 | assert_raises RuntimeError do 27 | PassiveDNS::Client.new(['doesnotexist']) 28 | end 29 | end 30 | 31 | def test_instantiate_All_Clients 32 | PassiveDNS::Client.new() 33 | end 34 | 35 | def test_instantiate_Passive_DNS_State 36 | refute_nil(PassiveDNS::PDNSToolState.new) 37 | end 38 | 39 | def test_instantiate_Passive_DNS_State_database 40 | if File.exist?("test/test.sqlite3") 41 | File.unlink("test/test.sqlite3") 42 | end 43 | refute_nil(PassiveDNS::PDNSToolStateDB.new("test/test.sqlite3")) 44 | if File.exist?("test/test.sqlite3") 45 | File.unlink("test/test.sqlite3") 46 | end 47 | end 48 | 49 | def test_DNSDB 50 | PassiveDNS::Client.new(['dnsdb']) 51 | d = PassiveDNS::Provider::DNSDB.new(@cp['dnsdb'] || {}) 52 | refute_nil(d) 53 | rows = d.lookup("example.org",3) 54 | refute_nil(rows) 55 | refute_nil(rows.to_s) 56 | refute_nil(rows.to_xml) 57 | refute_nil(rows.to_json) 58 | refute_nil(rows.to_yaml) 59 | assert_equal(3, rows.length) # this will fail since DNSDB has an off by one error 60 | rows = d.lookup("8.8.8.8") 61 | refute_nil(rows) 62 | refute_nil(rows.to_s) 63 | refute_nil(rows.to_xml) 64 | refute_nil(rows.to_json) 65 | refute_nil(rows.to_yaml) 66 | end 67 | 68 | def test_VirusTotal 69 | PassiveDNS::Client.new(['virustotal']) 70 | d = PassiveDNS::Provider::VirusTotal.new(@cp['virustotal'] || {}) 71 | refute_nil(d) 72 | rows = d.lookup("google.com",3) 73 | refute_nil(rows) 74 | refute_nil(rows.to_s) 75 | refute_nil(rows.to_xml) 76 | refute_nil(rows.to_json) 77 | refute_nil(rows.to_yaml) 78 | assert_equal(3, rows.length) 79 | rows = d.lookup("8.8.8.8") 80 | refute_nil(rows) 81 | refute_nil(rows.to_s) 82 | refute_nil(rows.to_xml) 83 | refute_nil(rows.to_json) 84 | refute_nil(rows.to_yaml) 85 | end 86 | 87 | def test_passivetotal 88 | PassiveDNS::Client.new(['passivetotal']) 89 | d = PassiveDNS::Provider::PassiveTotal.new(@cp['passivetotal'] || {}) 90 | refute_nil(d) 91 | rows = d.lookup("example.org") 92 | refute_nil(rows) 93 | refute_nil(rows.to_s) 94 | refute_nil(rows.to_xml) 95 | refute_nil(rows.to_json) 96 | refute_nil(rows.to_yaml) 97 | rows = d.lookup("example.org", 3) 98 | refute_nil(rows) 99 | refute_nil(rows.to_s) 100 | refute_nil(rows.to_xml) 101 | refute_nil(rows.to_json) 102 | refute_nil(rows.to_yaml) 103 | puts(rows) 104 | assert_equal(3, rows.length) 105 | rows = d.lookup("8.8.8.8") 106 | refute_nil(rows) 107 | refute_nil(rows.to_s) 108 | refute_nil(rows.to_xml) 109 | refute_nil(rows.to_json) 110 | refute_nil(rows.to_yaml) 111 | end 112 | 113 | def test_circl 114 | PassiveDNS::Client.new(['circl']) 115 | d = PassiveDNS::Provider::CIRCL.new(@cp['circl'] || {}) 116 | refute_nil(d) 117 | rows = d.lookup("example.org") 118 | refute_nil(rows) 119 | refute_nil(rows.to_s) 120 | refute_nil(rows.to_xml) 121 | refute_nil(rows.to_json) 122 | refute_nil(rows.to_yaml) 123 | rows = d.lookup("example.org",3) 124 | refute_nil(rows) 125 | refute_nil(rows.to_s) 126 | refute_nil(rows.to_xml) 127 | refute_nil(rows.to_json) 128 | refute_nil(rows.to_yaml) 129 | assert_equal(3, rows.length) 130 | rows = d.lookup("8.8.8.8") 131 | refute_nil(rows) 132 | refute_nil(rows.to_s) 133 | refute_nil(rows.to_xml) 134 | refute_nil(rows.to_json) 135 | refute_nil(rows.to_yaml) 136 | end 137 | 138 | def test_riskiq 139 | PassiveDNS::Client.new(['riskiq']) 140 | d = PassiveDNS::Provider::RiskIQ.new(@cp['riskiq'] || {}) 141 | refute_nil(d) 142 | rows = d.lookup("example.org") 143 | refute_nil(rows) 144 | refute_nil(rows.to_s) 145 | refute_nil(rows.to_xml) 146 | refute_nil(rows.to_json) 147 | refute_nil(rows.to_yaml) 148 | rows = d.lookup("example.org",3) 149 | refute_nil(rows) 150 | refute_nil(rows.to_s) 151 | refute_nil(rows.to_xml) 152 | refute_nil(rows.to_json) 153 | refute_nil(rows.to_yaml) 154 | assert_equal(3, rows.length) 155 | rows = d.lookup("8.8.8.8") 156 | refute_nil(rows) 157 | refute_nil(rows.to_s) 158 | refute_nil(rows.to_xml) 159 | refute_nil(rows.to_json) 160 | refute_nil(rows.to_yaml) 161 | end 162 | 163 | def test_osc 164 | PassiveDNS.Client.new(['osc']) 165 | d = PassiveDNS::Provider::OSC.new(@cp['osc'] || {}) 166 | refute_nil(d) 167 | rows = d.lookup("example.org") 168 | refute_nil(rows) 169 | puts rows 170 | end 171 | end 172 | --------------------------------------------------------------------------------