├── .gitignore ├── MIT_LICENSE ├── README.markdown ├── Rakefile ├── bin └── doublecheck ├── doublecheck.gemspec ├── lib ├── check_url.rb ├── doublecheck.rb ├── doublecheck_command_line.rb └── sitemap.rb └── test ├── check_url_test.rb ├── fixtures └── sitemap.xml └── sitemap_test.rb /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | -------------------------------------------------------------------------------- /MIT_LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License 2 | 3 | Copyright (c) 2010 Nathan Humbert 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.markdown: -------------------------------------------------------------------------------- 1 | # doublecheck 2 | 3 | doublecheck is a gem that pulls a sitemap and checks all of the URLs listed in it. 4 | 5 | ## Install 6 | 7 | gem install rcov 8 | 9 | 10 | ## Usage 11 | 12 | doublecheck http://example.com/sitemap.xml 13 | 14 | 15 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'rake/testtask' 2 | task :default => :test 3 | 4 | desc 'Run tests' 5 | Rake::TestTask.new(:test) do |t| 6 | t.test_files = FileList['test/*_test.rb'] 7 | t.ruby_opts = ['-Itest'] 8 | t.ruby_opts << '-rubygems' if defined? Gem 9 | end 10 | -------------------------------------------------------------------------------- /bin/doublecheck: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require 'doublecheck' 4 | require 'doublecheck_command_line' 5 | 6 | if ARGV[0].nil? 7 | puts "USAGE: doublecheck http://example.com/sitemap.xml" 8 | else 9 | sitemap = DoubleCheck::CommandLine.new(ARGV[0]) 10 | sitemap.process 11 | data = sitemap.data 12 | data.each do |code, urls| 13 | puts "\nStatus Code: #{code}" 14 | puts "#{urls.size} urls" 15 | urls.each do |url| 16 | puts "\t#{url}" 17 | end 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /doublecheck.gemspec: -------------------------------------------------------------------------------- 1 | require 'rake' 2 | 3 | require 'rake' 4 | Gem::Specification.new do |spec| 5 | spec.name = 'doublecheck' 6 | spec.summary = 'Check URLs listed in a sitemap' 7 | spec.description = 'Check URLs in a sitemap to make sure you have not introduced any issues.' 8 | spec.homepage = 'https://github.com/nathanhumbert/doublecheck' 9 | spec.version = '0.3' 10 | spec.authors = ['Nathan Humbert'] 11 | spec.email = ['nathan.humbert@gmail.com'] 12 | spec.executables << 'doublecheck' 13 | spec.files = [ 14 | 'README.markdown', 15 | 'MIT_LICENSE', 16 | ] 17 | spec.files += FileList['lib/**/*.rb', 'bin/*'] 18 | spec.test_files = FileList['tests/**/*.rb'] 19 | spec.license = 'MIT' 20 | spec.add_dependency('nokogiri', '>= 1.4.3.1') 21 | spec.add_development_dependency('fakeweb') 22 | spec.add_development_dependency('mocha') 23 | 24 | end 25 | -------------------------------------------------------------------------------- /lib/check_url.rb: -------------------------------------------------------------------------------- 1 | class DoubleCheck::CheckUrl 2 | require 'uri' 3 | 4 | def self.get_response(url) 5 | uri = URI.parse(url) 6 | response = Net::HTTP.start(uri.host, uri.port) do |http| 7 | if uri.path.empty? 8 | path = '/' 9 | else 10 | path = uri.path 11 | end 12 | http.get(path) 13 | end 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /lib/doublecheck.rb: -------------------------------------------------------------------------------- 1 | module DoubleCheck 2 | require 'rubygems' 3 | require 'sitemap' 4 | require 'check_url' 5 | end 6 | -------------------------------------------------------------------------------- /lib/doublecheck_command_line.rb: -------------------------------------------------------------------------------- 1 | class DoubleCheck::CommandLine < DoubleCheck::Sitemap 2 | def record_result(url, response) 3 | super(url, response) 4 | $stdout.write_nonblock('.') 5 | 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /lib/sitemap.rb: -------------------------------------------------------------------------------- 1 | class DoubleCheck::Sitemap 2 | require 'open-uri' 3 | require 'nokogiri' 4 | 5 | def initialize(url) 6 | @sitemap_url = url 7 | @data = {} 8 | end 9 | 10 | def process 11 | doc = Nokogiri::XML(open(@sitemap_url)) 12 | doc.css('loc').each do |url| 13 | response = DoubleCheck::CheckUrl.get_response(url.content) 14 | record_result(url.content, response) 15 | end 16 | end 17 | 18 | def record_result(url, response) 19 | if @data[response.code].nil? 20 | @data[response.code] = [url] 21 | else 22 | @data[response.code].push(url) 23 | end 24 | end 25 | 26 | def data 27 | return @data 28 | end 29 | 30 | end 31 | -------------------------------------------------------------------------------- /test/check_url_test.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'doublecheck' 3 | require 'fakeweb' 4 | 5 | class CheckUrlTest < Test::Unit::TestCase 6 | def setup 7 | @body = 'This is a test' 8 | FakeWeb.register_uri(:get, 'http://www.example.com/', :body => @body, :status => ['200', 'OK']) 9 | FakeWeb.register_uri(:get, 'http://www.example.com/foo', :status => ['404', 'Not Found']) 10 | end 11 | 12 | 13 | def test_get_response 14 | response = DoubleCheck::CheckUrl.get_response('http://www.example.com/') 15 | assert_equal '200', response.code 16 | assert_equal @body, response.body 17 | end 18 | 19 | end 20 | -------------------------------------------------------------------------------- /test/fixtures/sitemap.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | http://www.example.com/ 5 | 6 | 7 | http://www.example.com/foo 8 | 9 | 10 | -------------------------------------------------------------------------------- /test/sitemap_test.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'doublecheck' 3 | require 'fakeweb' 4 | require 'mocha' 5 | 6 | class SitemapTest < Test::Unit::TestCase 7 | def setup 8 | @body = IO.read(File.join('test', 'fixtures', 'sitemap.xml')) 9 | @sitemap_url = 'http://www.example.com/sitemap.xml' 10 | FakeWeb.register_uri(:get, @sitemap_url, :body => @body) 11 | @sitemap = DoubleCheck::Sitemap.new(@sitemap_url) 12 | end 13 | 14 | def test_initialize_sets_url_instance_variable 15 | setup 16 | assert_equal @sitemap_url, @sitemap.instance_variable_get(:@sitemap_url) 17 | end 18 | 19 | def test_initialize_sets_data_instance_variable_to_empty_hash 20 | setup 21 | data = {} 22 | assert_equal data, @sitemap.instance_variable_get(:@data) 23 | end 24 | 25 | def test_process_puts_correct_data_in_data_instance_variable 26 | setup 27 | @sitemap.process 28 | data = {'404' => ['http://www.example.com/foo'], '200' => ['http://www.example.com/']} 29 | assert_equal data, @sitemap.instance_variable_get(:@data) 30 | end 31 | 32 | def test_record_result 33 | setup 34 | response = mock() 35 | response.stubs(:code).returns('100') 36 | @sitemap.record_result('test', response) 37 | data = {'100' => ['test']} 38 | assert_equal data, @sitemap.instance_variable_get(:@data) 39 | end 40 | 41 | def test_data_returns_data_instance_variable_contents 42 | setup 43 | @sitemap.instance_variable_set(:@data, 'foobar') 44 | assert_equal 'foobar', @sitemap.data 45 | end 46 | 47 | end 48 | --------------------------------------------------------------------------------