├── .gitignore ├── Gemfile ├── Gemfile.lock ├── README.md ├── sitediff.rb └── string.rb /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | *.rbc 3 | /.config 4 | /coverage/ 5 | /InstalledFiles 6 | /pkg/ 7 | /spec/reports/ 8 | /spec/examples.txt 9 | /test/tmp/ 10 | /test/version_tmp/ 11 | /tmp/ 12 | 13 | # Used by dotenv library to load environment variables. 14 | # .env 15 | 16 | ## Specific to RubyMotion: 17 | .dat* 18 | .repl_history 19 | build/ 20 | *.bridgesupport 21 | build-iPhoneOS/ 22 | build-iPhoneSimulator/ 23 | 24 | ## Specific to RubyMotion (use of CocoaPods): 25 | # 26 | # We recommend against adding the Pods directory to your .gitignore. However 27 | # you should judge for yourself, the pros and cons are mentioned at: 28 | # https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control 29 | # 30 | # vendor/Pods/ 31 | 32 | ## Documentation cache and generated files: 33 | /.yardoc/ 34 | /_yardoc/ 35 | /doc/ 36 | /rdoc/ 37 | 38 | ## Environment normalization: 39 | /.bundle/ 40 | /vendor/bundle 41 | /lib/bundler/man/ 42 | 43 | # for a library or gem, you might want to ignore these files since the code is 44 | # intended to run in multiple environments; otherwise, check them in: 45 | # Gemfile.lock 46 | # .ruby-version 47 | # .ruby-gemset 48 | 49 | # unless supporting rvm < 1.11.0 or doing something fancy, ignore this: 50 | .rvmrc 51 | 52 | .*swp 53 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | gem 'net-http-digest_auth' 3 | gem "filesize" 4 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: https://rubygems.org/ 3 | specs: 4 | filesize (0.1.1) 5 | net-http-digest_auth (1.4) 6 | 7 | PLATFORMS 8 | ruby 9 | 10 | DEPENDENCIES 11 | filesize 12 | net-http-digest_auth 13 | 14 | BUNDLED WITH 15 | 1.13.6 16 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Sitediff 2 | 3 | Imagine the scenario, you are testing a site running an open source package 4 | but not sure what version and need to find out. Sitediff can help you do just 5 | that, it takes a local directory of files and then requests each of them from 6 | the target site and reports back on what it finds. 7 | 8 | For a full write up see https://digi.ninja/projects/sitediff.php 9 | -------------------------------------------------------------------------------- /sitediff.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | # 4 | # Sitediff, a tool to compare local files with those served by a site. 5 | # 6 | # Find the latest version at https://github.com/digininja/sitediff 7 | # and a full write up at https://digi.ninja/projects/sitediff.php 8 | # 9 | # 10 | # Author:: Robin Wood (robin@digi.ninja) (https://digi.ninja) 11 | # Copyright:: Copyright (c) Robin Wood 2016 12 | # Licence:: CC-BY-SA 2.0 or GPL-3+ 13 | # 14 | 15 | VERSION = "1.0 (Lazy Day)" 16 | 17 | puts "sitediff #{VERSION} Robin Wood (robin@digi.ninja) (https://digi.ninja/)\n\n" 18 | 19 | begin 20 | require "filesize" 21 | require 'digest' 22 | require 'getoptlong' 23 | require 'net/http' 24 | require 'openssl' 25 | require_relative 'string' 26 | rescue LoadError => e 27 | # Catch error and provide feedback on installing gem 28 | if e.to_s =~ /cannot load such file -- (.*)/ 29 | missing_gem = $1 30 | puts "\nError: #{missing_gem} gem not installed\n" 31 | puts "\t Use: 'gem install #{missing_gem}' to install the required gem\n\n" 32 | puts "\t Or: bundle install\n\n" 33 | exit 2 34 | else 35 | puts "There was an error loading the gems:\n" 36 | puts e.to_s 37 | exit 2 38 | end 39 | end 40 | 41 | opts = GetoptLong.new( 42 | ['--help', '-h', GetoptLong::NO_ARGUMENT], 43 | ['--keep', '-k', GetoptLong::NO_ARGUMENT], # maybe in a future release 44 | ['--match-only', '-m', GetoptLong::NO_ARGUMENT], 45 | ['--path', "-p", GetoptLong::REQUIRED_ARGUMENT], 46 | ['--url', "-u", GetoptLong::REQUIRED_ARGUMENT], 47 | ['--ua', GetoptLong::REQUIRED_ARGUMENT], 48 | ['--auth_user', GetoptLong::REQUIRED_ARGUMENT], 49 | ['--auth_pass', GetoptLong::REQUIRED_ARGUMENT], 50 | ['--auth_type', GetoptLong::REQUIRED_ARGUMENT], 51 | ['--header', "-H", GetoptLong::REQUIRED_ARGUMENT], 52 | ['--proxy_host', GetoptLong::REQUIRED_ARGUMENT], 53 | ['--proxy_port', GetoptLong::REQUIRED_ARGUMENT], 54 | ['--proxy_username', GetoptLong::REQUIRED_ARGUMENT], 55 | ['--proxy_password', GetoptLong::REQUIRED_ARGUMENT], 56 | ["--verbose", "-v", GetoptLong::NO_ARGUMENT] 57 | ) 58 | 59 | # Display the usage 60 | def usage 61 | puts "Usage: sitediff [OPTION] 62 | --help, -h: show help 63 | --path, -p path: the path for the source 64 | --url, -u URL: the base URL 65 | --ua user-agent: user agent to send 66 | --match-only, -m: only show matches 67 | 68 | Authentication 69 | --auth_type: digest or basic 70 | --auth_user: authentication username 71 | --auth_pass: authentication password 72 | 73 | Proxy Support 74 | --proxy_host: proxy host 75 | --proxy_port: proxy port, default 8080 76 | --proxy_username: username for proxy, if required 77 | --proxy_password: password for proxy, if required 78 | 79 | Headers 80 | --header, -H: in format name:value - can pass multiple 81 | 82 | --verbose, -v: verbose 83 | 84 | " 85 | exit 0 86 | end 87 | 88 | debug = false 89 | verbose = false 90 | ua = "Sitediff spider #{VERSION} - https://digi.ninja/projects/sitediff.php" 91 | base_url = nil 92 | path = nil 93 | keep = false 94 | match_only = false 95 | 96 | auth_type = nil 97 | auth_user = nil 98 | auth_pass = nil 99 | 100 | proxy_host = nil 101 | proxy_port = nil 102 | proxy_username = nil 103 | proxy_password = nil 104 | 105 | # headers will be passed in in the format "header: value" 106 | # and there can be multiple 107 | headers = [] 108 | 109 | begin 110 | opts.each do |opt, arg| 111 | case opt 112 | when '--help' 113 | usage 114 | when "--path" 115 | if !File.directory?(arg) 116 | puts "#{arg} is not a directory\n" 117 | exit 1 118 | end 119 | 120 | path = arg 121 | when "--keep" 122 | keep = true 123 | when "--url" 124 | # Must have protocol 125 | base_url = arg 126 | base_url = "http://#{base_url}" unless base_url =~ /^http(s)?:\/\// 127 | base_url = base_url + "/" unless base_url =~ /.*\/$/ 128 | when '--match-only' 129 | match_only = true 130 | when '--ua' 131 | ua = arg 132 | when '--verbose' 133 | verbose = true 134 | when "--header" 135 | headers << arg 136 | when "--proxy_password" 137 | proxy_password = arg 138 | when "--proxy_username" 139 | proxy_username = arg 140 | when "--proxy_host" 141 | proxy_host = arg 142 | when "--proxy_port" 143 | proxy_port = arg.to_i 144 | when "--auth_pass" 145 | auth_pass = arg 146 | when "--auth_user" 147 | auth_user = arg 148 | when "--auth_type" 149 | if arg =~ /(digest|basic)/i 150 | auth_type = $1.downcase 151 | if auth_type == "digest" 152 | begin 153 | require "net/http/digest_auth" 154 | rescue LoadError => e 155 | # Catch error and provide feedback on installing gem 156 | puts "\nError: To use digest auth you require the net-http-digest_auth gem\n" 157 | puts "\t Use: 'gem install net-http-digest_auth'\n\n" 158 | exit 2 159 | end 160 | end 161 | else 162 | puts "\nInvalid authentication type, please specify either basic or digest\n\n" 163 | exit 1 164 | end 165 | end 166 | end 167 | rescue 168 | puts 169 | usage 170 | end 171 | 172 | if auth_type && (auth_user.nil? || auth_pass.nil?) 173 | puts "\nIf using basic or digest auth you must provide a username and password\n\n" 174 | exit 1 175 | end 176 | 177 | if auth_type.nil? && (!auth_user.nil? || !auth_pass.nil?) 178 | puts "\nAuthentication details provided but no mention of basic or digest\n\n" 179 | exit 1 180 | end 181 | 182 | if base_url.nil? 183 | puts "You must specify the URL to test (--url)\n" 184 | exit 1 185 | end 186 | 187 | if path.nil? 188 | puts "You must specify the path for the local files (--path)\n" 189 | exit 1 190 | end 191 | 192 | header_hash = {} 193 | 194 | if headers.length > 0 then 195 | headers.each do |header| 196 | header_split = header.split(":") 197 | if (header_split.count == 2) 198 | header_hash[header_split[0].strip] = header_split[1].strip 199 | else 200 | puts "Invalid header: " + header.inspect 201 | end 202 | end 203 | end 204 | 205 | unless ua.nil? 206 | header_hash['User-Agent'] = ua 207 | end 208 | 209 | match_count = 0 210 | total_count = 0 211 | 212 | Dir.chdir(path) 213 | Dir.glob('**/*').each do |f| 214 | if File.file?(f) 215 | local_file_size = File.size(f) 216 | url = "#{base_url}#{f}" 217 | puts "Testing: #{url}" if verbose 218 | 219 | uri = URI.parse(url) 220 | 221 | # Shortcut 222 | #response = Net::HTTP.get_response(uri) 223 | 224 | if proxy_host.nil? 225 | http = Net::HTTP.new(uri.host, uri.port) 226 | 227 | if uri.scheme == 'https' 228 | http.use_ssl = true 229 | http.verify_mode = OpenSSL::SSL::VERIFY_NONE 230 | end 231 | else 232 | proxy = Net::HTTP::Proxy(proxy_host, proxy_port, proxy_username, proxy_password) 233 | begin 234 | if uri.scheme == 'https' 235 | http = proxy.start(uri.host, uri.port, :use_ssl => true, :verify_mode => OpenSSL::SSL::VERIFY_NONE) 236 | else 237 | http = proxy.start(uri.host, uri.port) 238 | end 239 | rescue => e 240 | puts "\nFailed to connect to the proxy (#{proxy_host}:#{proxy_port})\n\n" 241 | exit 1 242 | end 243 | end 244 | 245 | request = Net::HTTP::Get.new(uri.request_uri) 246 | header_hash.each_pair do |header, value| 247 | request[header] = value 248 | end 249 | 250 | if auth_type 251 | case auth_type 252 | when "digest" 253 | uri.user = auth_user 254 | uri.password = auth_pass 255 | 256 | res = http.request request 257 | 258 | if res['www-authenticate'] 259 | digest_auth = Net::HTTP::DigestAuth.new 260 | auth = digest_auth.auth_header uri, res['www-authenticate'], 'GET' 261 | 262 | request = Net::HTTP::Get.new uri.request_uri 263 | request.add_field 'Authorization', auth 264 | end 265 | 266 | when "basic" 267 | request.basic_auth auth_user, auth_pass 268 | end 269 | end 270 | 271 | begin 272 | response = http.request request 273 | rescue => e 274 | puts "There was a problem connecting to the server, please check the URL." 275 | exit 1 276 | end 277 | 278 | total_count += 1 279 | 280 | puts "Response code: #{response.code}" if verbose 281 | response_code = response.code.to_i 282 | 283 | case response_code 284 | when 500 285 | puts("#{f}: Internal server error".negative) unless match_only 286 | when 404 287 | puts("#{f}: File not found on site".negative) unless match_only 288 | when 401 289 | puts("#{f}: Authentication required".negative) unless match_only 290 | when 301, 302 291 | puts("#{f}: Redirect found to #{response['location']}".negative) unless match_only 292 | when 200 293 | response_length = response.body.length 294 | puts "Local file size: #{local_file_size}" if verbose 295 | puts "Response length: #{response_length}" if verbose 296 | if local_file_size != response_length then 297 | puts("#{f}: File found but different file size - local #{Filesize.from(local_file_size.to_s + " B").pretty} vs remote #{Filesize.from(response_length.to_s + " B").pretty}".neutral) unless match_only 298 | else 299 | sha256 = Digest::MD5.file f 300 | local_md5 = sha256.hexdigest 301 | puts "Local MD5: #{local_md5}" if verbose 302 | 303 | remote_md5 = Digest::MD5.hexdigest response.body 304 | puts "Local MD5: #{remote_md5}" if verbose 305 | 306 | if local_md5 == remote_md5 then 307 | puts "#{f}: Matching file found - #{url}".positive 308 | match_count += 1 309 | else 310 | puts "#{f}: File exists and is same size but contents differ".neutral unless match_only 311 | end 312 | end 313 | else 314 | puts("#{f}: Response code #{response_code}".negative) unless match_only 315 | end 316 | end 317 | end 318 | puts 319 | puts "Summary:" 320 | puts "#{total_count} files checked" 321 | puts "#{match_count} files matched" 322 | puts 323 | -------------------------------------------------------------------------------- /string.rb: -------------------------------------------------------------------------------- 1 | class String 2 | def red; colorize(self, "\e[1m\e[31m"); end 3 | def green; colorize(self, "\e[1m\e[32m"); end 4 | def dark_green; colorize(self, "\e[32m"); end 5 | def yellow; colorize(self, "\e[1m\e[33m"); end 6 | def blue; colorize(self, "\e[1m\e[34m"); end 7 | def dark_blue; colorize(self, "\e[34m"); end 8 | def purple; colorize(self, "\e[35m"); end 9 | def dark_purple; colorize(self, "\e[1;35m"); end 10 | def cyan; colorize(self, "\e[1;36m"); end 11 | def dark_cyan; colorize(self, "\e[36m"); end 12 | def pure; colorize(self, "\e[1m\e[35m"); end 13 | def bold; colorize(self, "\e[1m"); end 14 | def underline; colorize(self, "\e[4m"); end 15 | def colorize(text, color_code) "#{color_code}#{text}\e[0m" end 16 | 17 | def is_numeric? 18 | Float(self) != nil rescue false 19 | end 20 | 21 | def positive 22 | "[+] #{self}".green 23 | end 24 | def negative 25 | "[-] #{self}".red 26 | end 27 | def neutral 28 | "[.] #{self}".yellow 29 | end 30 | end 31 | --------------------------------------------------------------------------------