├── .github └── FUNDING.yml ├── .gitignore ├── .rspec ├── .travis.yml ├── .yardopts ├── Gemfile ├── Guardfile ├── MIT-LICENSE ├── README.md ├── Rakefile ├── lib ├── proxy_manager.rb └── proxy_manager │ └── proxy.rb ├── proxy_manager.gemspec └── spec ├── proxy_manager └── proxy_spec.rb ├── proxy_manager_spec.rb ├── spec_helper.rb └── support └── proxies.txt /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: [kirillplatonov] # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] 4 | patreon: # Replace with a single Patreon username 5 | open_collective: # Replace with a single Open Collective username 6 | ko_fi: # Replace with a single Ko-fi username 7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 9 | liberapay: # Replace with a single Liberapay username 10 | issuehunt: # Replace with a single IssueHunt username 11 | otechie: # Replace with a single Otechie username 12 | lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry 13 | custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .bundle/ 2 | log/*.log 3 | pkg/ 4 | Gemfile.lock 5 | tmp 6 | .yardoc/ 7 | *.gem 8 | doc/ 9 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --drb 2 | --format Fuubar 3 | --color -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | addons: 2 | code_climate: 3 | repo_token: 44136dcac9f72a39a472b5e8e838241afd7a5f315410d76649697a03e00d7423 4 | language: ruby 5 | rvm: 6 | - 1.9.3 7 | - 2.0.0 8 | - 2.1.1 9 | - rbx-2 10 | script: bundle exec rspec spec 11 | -------------------------------------------------------------------------------- /.yardopts: -------------------------------------------------------------------------------- 1 | --private 2 | --protected 3 | --readme README.md 4 | --charset utf-8 5 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | gemspec 4 | 5 | gem "codeclimate-test-reporter", group: :test, require: nil 6 | -------------------------------------------------------------------------------- /Guardfile: -------------------------------------------------------------------------------- 1 | # A sample Guardfile 2 | # More info at https://github.com/guard/guard#readme 3 | 4 | guard :rspec do 5 | watch(%r{^spec/.+_spec\.rb$}) 6 | watch(%r{^lib/(.+)\.rb$}) { |m| "spec/lib/#{m[1]}_spec.rb" } 7 | watch('spec/spec_helper.rb') { "spec" } 8 | 9 | # Rails example 10 | watch(%r{^app/(.+)\.rb$}) { |m| "spec/#{m[1]}_spec.rb" } 11 | watch(%r{^app/(.*)(\.erb|\.haml|\.slim)$}) { |m| "spec/#{m[1]}#{m[2]}_spec.rb" } 12 | watch(%r{^app/controllers/(.+)_(controller)\.rb$}) { |m| ["spec/routing/#{m[1]}_routing_spec.rb", "spec/#{m[2]}s/#{m[1]}_#{m[2]}_spec.rb", "spec/acceptance/#{m[1]}_spec.rb"] } 13 | watch(%r{^spec/support/(.+)\.rb$}) { "spec" } 14 | watch('config/routes.rb') { "spec/routing" } 15 | watch('app/controllers/application_controller.rb') { "spec/controllers" } 16 | 17 | # Capybara features specs 18 | watch(%r{^app/views/(.+)/.*\.(erb|haml|slim)$}) { |m| "spec/features/#{m[1]}_spec.rb" } 19 | 20 | # Turnip features and steps 21 | watch(%r{^spec/acceptance/(.+)\.feature$}) 22 | watch(%r{^spec/acceptance/steps/(.+)_steps\.rb$}) { |m| Dir[File.join("**/#{m[1]}.feature")][0] || 'spec/acceptance' } 23 | end 24 | 25 | notification :growl 26 | -------------------------------------------------------------------------------- /MIT-LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Kirill Platonov 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Ruby Proxy Manager 2 | 3 | [![Gem Version](https://badge.fury.io/rb/proxy_manager.svg)](http://badge.fury.io/rb/proxy_manager) 4 | [![Build Status](https://travis-ci.org/kirillplatonov/proxy_manager.svg?branch=master)](https://travis-ci.org/kirillplatonov/proxy_manager) 5 | [![Code Climate](https://codeclimate.com/github/kirillplatonov/proxy_manager.png)](https://codeclimate.com/github/kirillplatonov/proxy_manager) 6 | [![Coverage](https://codeclimate.com/github/kirillplatonov/proxy_manager/coverage.png)](https://codeclimate.com/github/kirillplatonov/proxy_manager) 7 | 8 | Easy manage proxy in your parsers/web-bots. 9 | 10 | 11 | ## Installation 12 | 13 | Add this line to your application's Gemfile: 14 | 15 | ```ruby 16 | gem 'proxy_manager' 17 | ``` 18 | 19 | And then execute: 20 | 21 | ```bash 22 | $ bundle 23 | ``` 24 | 25 | Or install it yourself as: 26 | 27 | ```bash 28 | $ gem install proxy_manager 29 | ``` 30 | 31 | ## Usage 32 | 33 | ### Load proxy list 34 | 35 | From array (IP:PORT) 36 | 37 | ```ruby 38 | proxy = ProxyManager::Proxy.new(['127.0.0.1:80', '127.0.0.1:8080']) 39 | ``` 40 | 41 | or from file 42 | 43 | ```ruby 44 | proxy = ProxyManager::Proxy.new('proxies.txt') 45 | ``` 46 | 47 | File with proxy list (in this case `proxies.txt`) should be readable and 48 | writable. 49 | Example of `proxies.txt` content: 50 | 51 | ``` 52 | 127.0.0.1:80 53 | 127.0.0.1:8080 54 | ... 55 | ``` 56 | 57 | ### Get proxy 58 | 59 | There is two methods to get proxy. 60 | 61 | First method return just next proxy, without any checking for availability: 62 | 63 | ```ruby 64 | proxy.get 65 | # => ["127.0.0.1", 80] 66 | ``` 67 | 68 | Band method return **only HTTP-available proxy**. It's perfect if you don't 69 | realy confidency on actuallity of your proxy list: 70 | 71 | ```ruby 72 | proxy.get! 73 | # => ["127.0.0.1", 8080] 74 | ``` 75 | 76 | You can also get for than one proxy per request by adding count for both 77 | methods like this: 78 | 79 | ```ruby 80 | proxy.get(2) 81 | # => [["127.0.0.1", 80], ["127.0.0.1", 8080]] 82 | 83 | proxy.get!(2) 84 | # => [["127.0.0.1", 80], ["127.0.0.1", 8080]] 85 | ``` 86 | 87 | ### Proxies list 88 | 89 | For display list of loaded proxies use `list` method: 90 | 91 | ```ruby 92 | proxy.list 93 | # => [["127.0.0.1", 80], ["127.0.0.1", 8080]] 94 | ``` 95 | 96 | ### Checking proxy manually 97 | 98 | You can also use class method for checking availability manually like this: 99 | 100 | ```ruby 101 | # by passing a string 102 | ProxyManager::Proxy.connectable?('127.0.0.1:80') 103 | # => false 104 | 105 | # or by passing an array 106 | ProxyManager::Proxy.connectable?('127.0.0.1', 80) 107 | # => false 108 | ``` 109 | 110 | Please, don't forget to star :star: the repository if you like (and use) the library. This will let me know how many users it has and then how to proceed with further development :). 111 | 112 | ## Documentation 113 | 114 | http://rubydoc.info/gems/proxy_manager 115 | 116 | ## Contributing 117 | 118 | 1. Fork it 119 | 2. Create your feature branch (`git checkout -b my-new-feature`) 120 | 3. Commit your changes (`git commit -am 'Add some feature'`) 121 | 4. Push to the branch (`git push origin my-new-feature`) 122 | 5. Create new Pull Request 123 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | begin 2 | require 'bundler/setup' 3 | rescue LoadError 4 | puts 'You must `gem install bundler` and `bundle install` to run rake tasks' 5 | end 6 | 7 | require 'rdoc/task' 8 | 9 | RDoc::Task.new(:rdoc) do |rdoc| 10 | rdoc.rdoc_dir = 'rdoc' 11 | rdoc.title = 'ProxyManager' 12 | rdoc.options << '--line-numbers' 13 | rdoc.rdoc_files.include('README.rdoc') 14 | rdoc.rdoc_files.include('lib/**/*.rb') 15 | end 16 | 17 | require 'rspec/core/rake_task' 18 | 19 | RSpec::Core::RakeTask.new 20 | task default: :spec -------------------------------------------------------------------------------- /lib/proxy_manager.rb: -------------------------------------------------------------------------------- 1 | require 'proxy_manager/proxy' 2 | 3 | module ProxyManager 4 | def self.root 5 | File.expand_path '../..', __FILE__ 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /lib/proxy_manager/proxy.rb: -------------------------------------------------------------------------------- 1 | module ProxyManager 2 | class Proxy 3 | attr_reader :list 4 | 5 | def initialize(proxies) 6 | @list = [] 7 | 8 | if proxies.is_a? Array 9 | load_list_from_array(proxies) 10 | else 11 | @list_file = proxies 12 | @list = load_from_file(@list_file) 13 | end 14 | end 15 | 16 | def get(count = 1) 17 | get_proxy(count) 18 | end 19 | 20 | def get!(count = 1) 21 | get_proxy(count, true) 22 | end 23 | 24 | def self.connectable?(proxy) 25 | proxy = proxy.chomp.split(':') if proxy.is_a? String 26 | ip, port = proxy 27 | connection = Net::HTTP.new("http://google.com", nil, ip, port) 28 | connection.open_timeout = 3 29 | connection.read_timeout = 3 30 | 31 | connection.start do |http| 32 | return true if http.get('/') 33 | end 34 | 35 | false 36 | rescue Exception => e 37 | false 38 | end 39 | 40 | private 41 | 42 | def load_list_from_array(proxies) 43 | @list = proxies.map { |arg| [arg.split(':')[0], arg.split(':')[1].to_i] } 44 | end 45 | 46 | def load_from_file(file) 47 | result = [] 48 | IO.readlines(file).each do |line| 49 | ip, port = line.chomp.split(':') 50 | result << [ip, port.to_i] if ip.is_a?(String) && port.is_a?(String) 51 | end 52 | result 53 | end 54 | 55 | def get_proxy(count, check_connection = false) 56 | raise 'List is empty' if @list.empty? 57 | 58 | items = [] 59 | new_list = @list.clone 60 | 61 | @list.each_with_index do |proxy, key| 62 | new_list.shift 63 | 64 | if !check_connection || self.class.connectable?(proxy) 65 | new_list << proxy 66 | 67 | if count == 1 68 | items = proxy 69 | break 70 | else 71 | items << proxy 72 | break if items.size == count 73 | end 74 | end 75 | end 76 | 77 | @list = new_list 78 | 79 | raise 'There are no available proxy' if items.empty? 80 | 81 | save_to_file(@list_file, @list) if @list_file 82 | 83 | items 84 | end 85 | 86 | def save_to_file(file, list) 87 | source = '' 88 | list.each { |p| source << "#{p[0]}:#{p[1]}\n" } 89 | 90 | IO.write(file, source.sub(/\n$/, '')) 91 | end 92 | end 93 | end 94 | -------------------------------------------------------------------------------- /proxy_manager.gemspec: -------------------------------------------------------------------------------- 1 | $:.push File.expand_path("../lib", __FILE__) 2 | 3 | # Describe your gem and declare its dependencies: 4 | Gem::Specification.new do |s| 5 | s.name = "proxy_manager" 6 | s.version = '1.0.1' 7 | s.authors = ["Kirill Platonov"] 8 | s.licenses = ['MIT'] 9 | s.email = ["platonov.kd@gmail.com"] 10 | s.homepage = "https://github.com/bloodyhistory/proxy_manager" 11 | s.summary = "This is gem for easy usage proxy in your parsers/web-bots." 12 | s.description = <<-DESCRIPTION 13 | Proxy manager for easy usage proxy in your parsers/web-bots 14 | DESCRIPTION 15 | 16 | s.files = `git ls-files`.split("\n") - %w[.gitignore .travis.yml] 17 | s.test_files = Dir["spec/**/*"] 18 | 19 | s.add_development_dependency 'turn', '~> 0.9', '>= 0.9.7' 20 | s.add_development_dependency 'rspec', '~> 2.14', '>= 2.14.1' 21 | s.add_development_dependency 'guard-rspec', '~> 4.2', '>= 4.2.8' 22 | s.add_development_dependency 'growl', '~> 1.0', '>= 1.0.3' 23 | s.add_development_dependency 'fuubar', '~> 1.3', '>= 1.3.2' 24 | s.add_development_dependency 'yard', '~> 0.8', '>= 0.8.7.4' 25 | end 26 | -------------------------------------------------------------------------------- /spec/proxy_manager/proxy_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'proxy_manager' 3 | 4 | describe ProxyManager::Proxy do 5 | let(:list) { [['127.0.0.1', 80], ['127.0.0.1', 8080]] } 6 | let(:proxies_file) { File.join(ProxyManager.root, 7 | 'spec', 8 | 'support', 9 | 'proxies.txt' 10 | ) } 11 | subject { ProxyManager::Proxy.new(['127.0.0.1:80', '127.0.0.1:8080']) } 12 | 13 | context '#get' do 14 | its(:get) { should be_a Array } 15 | 16 | it 'should return many proxies' do 17 | expect { subject.get(3) }.not_to raise_error 18 | end 19 | end 20 | 21 | context '#get!' do 22 | it { should respond_to :get! } 23 | end 24 | 25 | context '#list' do 26 | its(:list) { should match_array list } 27 | end 28 | 29 | context '::connectable' do 30 | it 'should receive array' do 31 | expect(ProxyManager::Proxy.connectable?(['127.0.0.1', 8080])).to be_false 32 | end 33 | 34 | it 'should receive string' do 35 | expect(ProxyManager::Proxy.connectable?("127.0.0.1:8080")).to be_false 36 | end 37 | end 38 | 39 | context 'when load from file' do 40 | subject { ProxyManager::Proxy.new(proxies_file) } 41 | 42 | its(:list) { should match_array list } 43 | 44 | context 'and save file' do 45 | let(:source) { IO.read(proxies_file) } 46 | 47 | it 'should update proxies source' do 48 | subject.get(2) 49 | 50 | source = '' 51 | subject.list.each { |p| source << "#{p[0]}:#{p[1]}\n" } 52 | source.sub!(/\n$/, '') 53 | 54 | expect(IO.read(proxies_file)).to eq(source) 55 | end 56 | 57 | after { IO.write(proxies_file, source) } 58 | end 59 | end 60 | end 61 | -------------------------------------------------------------------------------- /spec/proxy_manager_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'proxy_manager' 3 | 4 | describe ProxyManager do 5 | it { expect(subject.root).not_to be_empty } 6 | end 7 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require "codeclimate-test-reporter" 2 | CodeClimate::TestReporter.start 3 | 4 | RSpec.configure do |config| 5 | config.treat_symbols_as_metadata_keys_with_true_values = true 6 | config.run_all_when_everything_filtered = true 7 | config.filter_run :focus 8 | 9 | # Run specs in random order to surface order dependencies. If you find an 10 | # order dependency and want to debug it, you can fix the order by providing 11 | # the seed, which is printed after each run. 12 | # --seed 1234 13 | config.order = 'random' 14 | end 15 | -------------------------------------------------------------------------------- /spec/support/proxies.txt: -------------------------------------------------------------------------------- 1 | 127.0.0.1:80 2 | 127.0.0.1:8080 --------------------------------------------------------------------------------