├── .ruby-version ├── ext ├── .github │ └── workflows │ │ └── ci.yml ├── curb_multi.h ├── curb_upload.h ├── banned.h ├── extconf_coverage.rb ├── curb_postfield.h ├── curb.h ├── curb_upload.c ├── curb_easy.h ├── curb_errors.h └── curb_macros.h ├── lib ├── curb.rb ├── curb.gemspec.erb ├── curl.rb └── curl │ └── multi.rb ├── tests ├── unittests.rb ├── alltests.rb ├── bugtests.rb ├── tc_curl_maxfilesize.rb ├── bug_issue102.rb ├── tc_curl_easy_resolve.rb ├── bug_multi_segfault.rb ├── bug_crash_on_debug.rb ├── tc_curl_easy_setopt.rb ├── bug_raise_on_callback.rb ├── bug_postfields_crash.rb ├── signals.rb ├── timeout_server.rb ├── tc_curl_protocols.rb ├── tc_ftp_options.rb ├── bug_curb_easy_blocks_ruby_threads.rb ├── tc_curl_easy_request_target.rb ├── bug_crash_on_progress.rb ├── bug_require_last_or_segfault.rb ├── require_last_or_segfault_script.rb ├── bug_instance_post_differs_from_class_post.rb ├── cacert.pem ├── bug_issue_spnego.rb ├── bug_issue_noproxy.rb ├── tc_gc_compact.rb ├── cert.pem ├── mem_check.rb ├── bug_postfields_crash2.rb ├── tc_curl.rb ├── bug_curb_easy_post_with_string_no_content_length_header.rb ├── tc_curl_download.rb ├── bug_follow_redirect_288.rb ├── bug_issue_post_redirect.rb ├── timeout.rb ├── tc_curl_postfield.rb ├── tc_fiber_scheduler.rb ├── helper.rb └── tc_curl_easy_cookielist.rb ├── Gemfile.ruby-2.1 ├── Gemfile.ruby-1.8 ├── bench ├── Gemfile ├── typhoeus_test.rb ├── patron_test.rb ├── nethttp_test.rb ├── typhoeus_hydra_test.rb ├── curb_easy14.rb ├── _usage.rb ├── curb_easy.rb ├── curb_multi_using_get.rb ├── emhttprequest.rb ├── Rakefile ├── Gemfile.lock ├── curb_multi.rb ├── zeros-2k └── README ├── .gitignore ├── Gemfile ├── samples ├── simple_multi.rb ├── multi_interface.rb ├── gmail.rb ├── downloader.rb ├── fiber_crawler.rb └── delayed_wait.rb ├── codecov.yml ├── doc.rb ├── tasks ├── utils.rb ├── coverage.rake ├── docker.rake └── rake_helpers.rb ├── LICENSE ├── curb.gemspec ├── .github └── workflows │ └── CI.yml ├── index.html └── Rakefile /.ruby-version: -------------------------------------------------------------------------------- 1 | ruby-3.4.5 2 | -------------------------------------------------------------------------------- /ext/.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lib/curb.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require 'curl' 3 | -------------------------------------------------------------------------------- /tests/unittests.rb: -------------------------------------------------------------------------------- 1 | $: << $TESTDIR = File.expand_path(File.dirname(__FILE__)) 2 | Dir[File.join($TESTDIR, 'tc_*.rb')].each { |lib| require lib } 3 | -------------------------------------------------------------------------------- /Gemfile.ruby-2.1: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gemspec 4 | 5 | gem 'mixlib-shellout', '< 2.3' 6 | gem 'rake', '< 12' 7 | gem 'test-unit' 8 | -------------------------------------------------------------------------------- /Gemfile.ruby-1.8: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gemspec 4 | 5 | gem 'rake', '< 12' # as of 12.0 rake requires ruby 1.9 6 | gem 'test-unit', '< 3' 7 | -------------------------------------------------------------------------------- /tests/alltests.rb: -------------------------------------------------------------------------------- 1 | $: << $TESTDIR = File.expand_path(File.dirname(__FILE__)) 2 | require 'unittests' 3 | Dir[File.join($TESTDIR, 'bug_*.rb')].each { |lib| require lib } 4 | -------------------------------------------------------------------------------- /bench/Gemfile: -------------------------------------------------------------------------------- 1 | source :rubygems 2 | 3 | gem 'curb', '1.0.0' 4 | gem 'rmem' 5 | gem 'net-http-persistent' 6 | gem 'em-http-request', ">= 1.1.6" 7 | gem 'patron' 8 | gem 'typhoeus' 9 | gem "rake", '>= 13.0.1' 10 | -------------------------------------------------------------------------------- /tests/bugtests.rb: -------------------------------------------------------------------------------- 1 | $: << $TESTDIR = File.expand_path(File.dirname(__FILE__)) 2 | puts "start" 3 | begin 4 | Dir[File.join($TESTDIR, 'bug_*.rb')].each { |lib| require lib } 5 | rescue Object => e 6 | puts e.message 7 | ensure 8 | puts "done" 9 | end 10 | -------------------------------------------------------------------------------- /tests/tc_curl_maxfilesize.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.join(File.dirname(__FILE__), 'helper')) 2 | 3 | class TestCurbCurlMaxFileSize < Test::Unit::TestCase 4 | def setup 5 | @easy = Curl::Easy.new 6 | end 7 | 8 | def test_maxfilesize 9 | @easy.set(Curl::CURLOPT_MAXFILESIZE, 5000000) 10 | end 11 | 12 | end 13 | -------------------------------------------------------------------------------- /bench/typhoeus_test.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | $:.unshift File.expand_path(File.dirname(__FILE__)) 3 | require '_usage' 4 | 5 | N = (ARGV.shift || 50).to_i 6 | Memory.usage("Typhoeus(#{N})") do 7 | 8 | require 'typhoeus' 9 | 10 | N.times do|n| 11 | Typhoeus::Request.get('http://127.0.0.1/zeros-2k' + "?n=#{n}") 12 | end 13 | 14 | end 15 | -------------------------------------------------------------------------------- /bench/patron_test.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | $:.unshift File.expand_path(File.dirname(__FILE__)) 3 | require '_usage' 4 | N = (ARGV.shift || 50).to_i 5 | 6 | Memory.usage("Patron(#{N})") do 7 | 8 | require 'patron' 9 | 10 | sess = Patron::Session.new 11 | sess.base_url = 'http://127.0.0.1' 12 | 13 | N.times do |n| 14 | sess.get('/zeros-2k') 15 | end 16 | 17 | end 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ext/Makefile 2 | ext/curb.o 3 | ext/curb_core.so 4 | ext/curb_core.bundle 5 | ext/curb_easy.o 6 | ext/curb_errors.o 7 | ext/curb_multi.o 8 | ext/curb_postfield.o 9 | ext/curb_upload.o 10 | ext/curb_config.h 11 | 12 | pkg/ 13 | ext/mkmf.log 14 | build/ 15 | vendor/ 16 | .bundle/ 17 | .idea 18 | *.lock 19 | *.gem 20 | *.gcov 21 | *.gcda 22 | *.gcno 23 | coverage_c/ 24 | coverage/ 25 | 26 | ext/curb_core.bundle.dSYM/ 27 | -------------------------------------------------------------------------------- /bench/nethttp_test.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | $:.unshift File.expand_path(File.dirname(__FILE__)) 3 | require '_usage' 4 | 5 | N = (ARGV.shift || 50).to_i 6 | BURL = 'http://127.0.0.1/zeros-2k' 7 | 8 | Memory.usage("Net::HTTP Persistent(#{N})") do 9 | require 'net/http/persistent' 10 | 11 | http = Net::HTTP::Persistent.new 12 | 13 | N.times do |n| 14 | http.request URI.parse(BURL+"?n=#{n}") 15 | end 16 | 17 | end 18 | -------------------------------------------------------------------------------- /tests/bug_issue102.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.join(File.dirname(__FILE__), 'helper')) 2 | 3 | class BugIssue102 < Test::Unit::TestCase 4 | 5 | def test_interface 6 | test = "https://api.twitter.com/1/users/show.json?screen_name=TwitterAPI&include_entities=true" 7 | ip = "0.0.0.0" 8 | 9 | c = Curl::Easy.new do |curl| 10 | curl.url = test 11 | curl.interface = ip 12 | end 13 | 14 | c.perform 15 | end 16 | 17 | end 18 | -------------------------------------------------------------------------------- /tests/tc_curl_easy_resolve.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.join(File.dirname(__FILE__), 'helper')) 2 | 3 | class TestCurbCurlEasyResolve < Test::Unit::TestCase 4 | def setup 5 | @easy = Curl::Easy.new 6 | end 7 | 8 | def test_resolve 9 | @easy.resolve = [ "example.com:80:127.0.0.1" ] 10 | assert_equal @easy.resolve, [ "example.com:80:127.0.0.1" ] 11 | end 12 | 13 | def test_empty_resolve 14 | assert_equal @easy.resolve, nil 15 | end 16 | end 17 | -------------------------------------------------------------------------------- /bench/typhoeus_hydra_test.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | $:.unshift File.expand_path(File.dirname(__FILE__)) 3 | require '_usage' 4 | 5 | N = (ARGV.shift || 50).to_i 6 | 7 | Memory.usage("Typhoeus::Hydra(#{N})") do 8 | 9 | require 'typhoeus' 10 | 11 | hydra = Typhoeus::Hydra.new 12 | reqs = [] 13 | 14 | N.times do |n| 15 | req = Typhoeus::Request.new('http://127.0.0.1/zeros-2k' + "?n=#{n}") 16 | reqs << req 17 | hydra.queue req 18 | end 19 | hydra.run 20 | 21 | end 22 | -------------------------------------------------------------------------------- /bench/curb_easy14.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | gem 'curb', '0.1.4' 3 | $:.unshift File.expand_path(File.dirname(__FILE__)) 4 | require '_usage' 5 | 6 | N = (ARGV.shift || 50).to_i 7 | BURL = 'http://127.0.0.1/zeros-2k' 8 | 9 | Memory.usage("Curl::Easy14(#{N})") do 10 | 11 | require 'curb' 12 | c = Curl::Easy.new 13 | 14 | N.times do|n| 15 | c.url = BURL + "?n=#{n}" 16 | c.on_header {|d| d.size} # don't buffer 17 | c.on_body {|d| d.size} # don't buffer 18 | c.perform 19 | end 20 | 21 | end 22 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gemspec 4 | 5 | group 'development', 'test' do 6 | gem 'webrick' # for ruby 3.1 7 | gem 'rdoc' 8 | gem 'rake' 9 | gem 'mixlib-shellout' # for docker test builds? 10 | gem 'test-unit' 11 | gem 'ruby_memcheck' 12 | if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('3.1') 13 | gem 'async', '>= 2.20' 14 | end 15 | gem 'simplecov', require: false 16 | gem 'simplecov-lcov', require: false # For CI integration 17 | end 18 | 19 | platforms :rbx do 20 | gem 'minitest' 21 | end 22 | -------------------------------------------------------------------------------- /tests/bug_multi_segfault.rb: -------------------------------------------------------------------------------- 1 | # From safis http://github.com/taf2/curb/issues#issue/5 2 | # irb: require 'curb' 3 | # irb: multi = Curl::Multi.new 4 | # irb: exit 5 | #
:47140: [BUG] Bus Error 6 | require File.expand_path(File.join(File.dirname(__FILE__), 'helper')) 7 | $:.unshift File.expand_path(File.join(File.dirname(__FILE__),'..','ext')) 8 | $:.unshift File.expand_path(File.join(File.dirname(__FILE__),'..','lib')) 9 | require 'curb' 10 | 11 | class BugMultiSegfault < Test::Unit::TestCase 12 | def test_bug 13 | multi = Curl::Multi.new 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /bench/_usage.rb: -------------------------------------------------------------------------------- 1 | require 'rmem' 2 | 3 | class Memory 4 | 5 | def self.usage(name) 6 | GC.start 7 | GC.disable 8 | smem = RMem::Report.memory 9 | 10 | t = Time.now 11 | yield 12 | duration = Time.now - t 13 | 14 | GC.enable 15 | GC.start 16 | 17 | emem = RMem::Report.memory 18 | memory_usage = emem / 1024.0 19 | memory_growth = (emem - smem) / 1024.0 20 | 21 | printf "#{name}\t\tDuration: %.4f sec, Memory Usage: %.2f KB - Memory Growth: %.2f KB average: #{duration/N}\n", duration, memory_usage, memory_growth 22 | end 23 | 24 | end 25 | -------------------------------------------------------------------------------- /bench/curb_easy.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | $:.unshift File.expand_path(File.dirname(__FILE__)) 3 | require '_usage' 4 | 5 | $:.unshift File.expand_path(File.join(File.dirname(__FILE__),'..','ext')) 6 | $:.unshift File.expand_path(File.join(File.dirname(__FILE__),'..','lib')) 7 | 8 | N = (ARGV.shift || 50).to_i 9 | BURL = 'http://127.0.0.1/zeros-2k' 10 | 11 | Memory.usage("Curl::Easy(#{N})") do 12 | 13 | require 'curb' 14 | c = Curl::Easy.new 15 | 16 | N.times do|n| 17 | c.url = BURL + "?n=#{n}" 18 | c.on_header {|d| d.size} # don't buffer 19 | c.on_body {|d| d.size} # don't buffer 20 | c.perform 21 | end 22 | 23 | end 24 | -------------------------------------------------------------------------------- /tests/bug_crash_on_debug.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.join(File.dirname(__FILE__), 'helper')) 2 | 3 | class BugCrashOnDebug < Test::Unit::TestCase 4 | include BugTestServerSetupTeardown 5 | 6 | def test_on_debug 7 | c = Curl::Easy.new("http://127.0.0.1:#{@port}/test") 8 | did_raise = false 9 | did_call = false 10 | begin 11 | c.on_success do|x| 12 | did_call = true 13 | raise "error" # this will get swallowed 14 | end 15 | c.perform 16 | rescue => e 17 | did_raise = true 18 | end 19 | assert did_raise 20 | assert did_call 21 | end 22 | 23 | end 24 | 25 | #test_on_debug 26 | -------------------------------------------------------------------------------- /ext/curb_multi.h: -------------------------------------------------------------------------------- 1 | /* curb_multi.h - Curl easy mode 2 | * Copyright (c)2008 Todd A. Fisher. 3 | * Licensed under the Ruby License. See LICENSE for details. 4 | * 5 | * $Id$ 6 | */ 7 | #ifndef __CURB_MULTI_H 8 | #define __CURB_MULTI_H 9 | 10 | #include "curb.h" 11 | #include 12 | 13 | struct st_table; 14 | 15 | typedef struct { 16 | int active; 17 | int running; 18 | CURLM *handle; 19 | struct st_table *attached; 20 | } ruby_curl_multi; 21 | 22 | extern VALUE cCurlMulti; 23 | void init_curb_multi(); 24 | VALUE ruby_curl_multi_new(VALUE klass); 25 | void rb_curl_multi_forget_easy(ruby_curl_multi *rbcm, void *rbce_ptr); 26 | 27 | 28 | #endif 29 | -------------------------------------------------------------------------------- /samples/simple_multi.rb: -------------------------------------------------------------------------------- 1 | $:.unshift(File.join(File.dirname(__FILE__), '..', 'ext')) 2 | $:.unshift(File.join(File.dirname(__FILE__), '..', 'lib')) 3 | require 'curb' 4 | 5 | 6 | Curl::Multi.get( 7 | ["http://boingboing.net", 8 | "http://www.yahoo.com", 9 | "http://slashdot.org", 10 | "http://www.google.com", 11 | "http://www.yelp.com", 12 | "http://digg.com", 13 | "http://www.google.co.uk/", 14 | "http://www.ruby-lang.org/"], {:follow_location => true,:max_redirects => 3,:timeout => 4}) do|easy| 15 | puts "[#{easy.last_effective_url} #{easy.total_time}] #{easy.body_str[0,30].gsub("\n", '')}..." 16 | end 17 | -------------------------------------------------------------------------------- /tests/tc_curl_easy_setopt.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.join(File.dirname(__FILE__), 'helper')) 2 | 3 | class TestCurbCurlEasySetOpt < Test::Unit::TestCase 4 | def setup 5 | @easy = Curl::Easy.new 6 | end 7 | 8 | def test_opt_verbose 9 | @easy.set :verbose, true 10 | assert @easy.verbose? 11 | end 12 | 13 | def test_opt_header 14 | @easy.set :header, true 15 | end 16 | 17 | def test_opt_noprogress 18 | @easy.set :noprogress, true 19 | end 20 | 21 | def test_opt_nosignal 22 | @easy.set :nosignal, true 23 | end 24 | 25 | def test_opt_url 26 | url = "http://google.com/" 27 | @easy.set :url, url 28 | assert_equal url, @easy.url 29 | end 30 | 31 | end 32 | -------------------------------------------------------------------------------- /bench/curb_multi_using_get.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | $:.unshift File.expand_path(File.dirname(__FILE__)) 3 | require '_usage' 4 | 5 | $:.unshift File.expand_path(File.join(File.dirname(__FILE__),'..','ext')) 6 | $:.unshift File.expand_path(File.join(File.dirname(__FILE__),'..','lib')) 7 | 8 | N = (ARGV.shift || 50).to_i 9 | BURL = 'http://127.0.0.1/zeros-2k' 10 | 11 | Memory.usage("Curl::Multi.get(#{N})") do 12 | require 'curb' 13 | 14 | group = [] 15 | N.times do|n| 16 | url = BURL + "?n=#{n}" 17 | 18 | group << url 19 | 20 | if group.size == 10 21 | Curl::Multi.get(group) 22 | group = [] 23 | end 24 | 25 | end 26 | 27 | Curl::Multi.get(group) if group.any? 28 | 29 | end 30 | -------------------------------------------------------------------------------- /bench/emhttprequest.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | $:.unshift File.expand_path(File.dirname(__FILE__)) 3 | require '_usage' 4 | 5 | $:.unshift File.expand_path(File.join(File.dirname(__FILE__),'..','ext')) 6 | $:.unshift File.expand_path(File.join(File.dirname(__FILE__),'..','lib')) 7 | 8 | N = (ARGV.shift || 50).to_i 9 | BURL = 'http://127.0.0.1/zeros-2k' 10 | 11 | Memory.usage("EM::HTTPRequest(#{N})") do 12 | 13 | require 'em-http-request' 14 | EM.run do 15 | multi = EM::MultiRequest.new 16 | 17 | N.times do|n| 18 | # add multiple requests to the multi-handler 19 | multi.add(EM::HttpRequest.new(BURL).get) 20 | end 21 | 22 | multi.callback do 23 | EM.stop 24 | end 25 | 26 | end 27 | 28 | end 29 | -------------------------------------------------------------------------------- /tests/bug_raise_on_callback.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.join(File.dirname(__FILE__), 'helper')) 2 | 3 | class BugRaiseOnCallback < Test::Unit::TestCase 4 | include BugTestServerSetupTeardown 5 | 6 | def setup 7 | @port = 9999 8 | super 9 | end 10 | 11 | def test_on_complte 12 | c = Curl::Easy.new('http://127.0.0.1:9999/test') 13 | did_raise = false 14 | begin 15 | c.on_complete do|x| 16 | assert_equal 'http://127.0.0.1:9999/test', x.url 17 | raise "error complete" # this will get swallowed 18 | end 19 | c.perform 20 | rescue => e 21 | did_raise = true 22 | end 23 | assert did_raise, "we want to raise an exception if the ruby callbacks raise" 24 | 25 | end 26 | 27 | end 28 | 29 | #test_on_debug 30 | -------------------------------------------------------------------------------- /tests/bug_postfields_crash.rb: -------------------------------------------------------------------------------- 1 | # From GICodeWarrior: 2 | # 3 | # $ ruby crash_curb.rb 4 | # crash_curb.rb:7: [BUG] Segmentation fault 5 | # ruby 1.8.7 (2009-06-12 patchlevel 174) [x86_64-linux] 6 | # 7 | # Aborted 8 | # crash_curb.rb: 9 | # #!/usr/bin/ruby 10 | # require 'rubygems' 11 | # require 'curb' 12 | # 13 | # curl = Curl::Easy.new('http://example.com/') 14 | # curl.multipart_form_post = true 15 | # curl.http_post(Curl::PostField.file('test', 'test.xml'){'example data'}) 16 | # Ubuntu 9.10 17 | # curb gem version 0.6.2.1 18 | require File.expand_path(File.join(File.dirname(__FILE__), 'helper')) 19 | 20 | class BugPostFieldsCrash < Test::Unit::TestCase 21 | def test_crash 22 | curl = Curl::Easy.new('http://example.com/') 23 | curl.multipart_form_post = true 24 | curl.http_post(Curl::PostField.file('test', 'test.xml'){'example data'}) 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /ext/curb_upload.h: -------------------------------------------------------------------------------- 1 | /* curb_upload.h - Curl upload handle 2 | * Copyright (c)2009 Todd A Fisher. 3 | * Licensed under the Ruby License. See LICENSE for details. 4 | */ 5 | #ifndef __CURB_UPLOAD_H 6 | #define __CURB_UPLOAD_H 7 | 8 | #include "curb.h" 9 | 10 | #include 11 | 12 | /* 13 | * Maintain the state of an upload e.g. for putting large streams with very little memory 14 | * out to a server. via PUT requests 15 | */ 16 | typedef struct { 17 | VALUE stream; 18 | size_t offset; 19 | } ruby_curl_upload; 20 | 21 | extern VALUE cCurlUpload; 22 | void init_curb_upload(); 23 | 24 | VALUE ruby_curl_upload_new(VALUE klass); 25 | VALUE ruby_curl_upload_stream_set(VALUE self, VALUE stream); 26 | VALUE ruby_curl_upload_stream_get(VALUE self); 27 | VALUE ruby_curl_upload_offset_set(VALUE self, VALUE offset); 28 | VALUE ruby_curl_upload_offset_get(VALUE self); 29 | 30 | #endif 31 | -------------------------------------------------------------------------------- /tests/signals.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.join(File.dirname(__FILE__), 'helper')) 2 | 3 | # This test suite requires the timeout server to be running 4 | # See tests/timeout.rb for more info about the timeout server 5 | class TestCurbSignals < Test::Unit::TestCase 6 | 7 | # Testcase for https://github.com/taf2/curb/issues/117 8 | def test_continue_after_signal 9 | trap("SIGUSR1") { } 10 | 11 | curl = Curl::Easy.new(wait_url(2)) 12 | pid = $$ 13 | Thread.new do 14 | sleep 1 15 | Process.kill("SIGUSR1", pid) 16 | end 17 | assert_equal true, curl.http_get 18 | end 19 | 20 | private 21 | 22 | def wait_url(time) 23 | "#{server_base}/wait/#{time}" 24 | end 25 | 26 | def serve_url(chunk_size, time, count) 27 | "#{server_base}/serve/#{chunk_size}/every/#{time}/for/#{count}" 28 | end 29 | 30 | def server_base 31 | 'http://127.0.0.1:9128' 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /ext/banned.h: -------------------------------------------------------------------------------- 1 | #ifndef BANNED_H 2 | #define BANNED_H 3 | 4 | /* 5 | * This header lists functions that have been banned from our code base, 6 | * because they're too easy to misuse (and even if used correctly, 7 | * complicate audits). Including this header turns them into compile-time 8 | * errors. 9 | */ 10 | 11 | #define BANNED(func) sorry_##func##_is_a_banned_function 12 | 13 | #undef strcpy 14 | #define strcpy(x,y) BANNED(strcpy) 15 | #undef strcat 16 | #define strcat(x,y) BANNED(strcat) 17 | #undef strncpy 18 | #define strncpy(x,y,n) BANNED(strncpy) 19 | #undef strncat 20 | #define strncat(x,y,n) BANNED(strncat) 21 | 22 | #undef sprintf 23 | #undef vsprintf 24 | #ifdef HAVE_VARIADIC_MACROS 25 | #define sprintf(...) BANNED(sprintf) 26 | #define vsprintf(...) BANNED(vsprintf) 27 | #else 28 | #define sprintf(buf,fmt,arg) BANNED(sprintf) 29 | #define vsprintf(buf,fmt,arg) BANNED(sprintf) 30 | #endif 31 | 32 | #endif /* BANNED_H */ 33 | -------------------------------------------------------------------------------- /samples/multi_interface.rb: -------------------------------------------------------------------------------- 1 | # Use local curb 2 | # 3 | 4 | $:.unshift(File.join(File.dirname(__FILE__), '..', 'ext')) 5 | $:.unshift(File.join(File.dirname(__FILE__), '..', 'lib')) 6 | require 'curb' 7 | 8 | # An example for Multi Interface 9 | # 10 | # Some URLs to smack in our demo. 11 | # 12 | 13 | urls = ["http://boingboing.net", 14 | "http://www.yahoo.com", 15 | "http://slashdot.org", 16 | "http://www.google.com", 17 | "http://www.yelp.com", 18 | "http://www.givereal.com", 19 | "http://www.google.co.uk/", 20 | "http://www.ruby-lang.org/"] 21 | 22 | responses = {} 23 | m = Curl::Multi.new 24 | # add a few easy handles 25 | urls.each do |url| 26 | responses[url] = Curl::Easy.new(url) 27 | m.add(responses[url]) 28 | end 29 | 30 | m.perform 31 | 32 | urls.each do |url| 33 | puts "[#{url} #{responses[url].total_time}] #{responses[url].body_str[0,30].gsub("\n", '')}..." 34 | puts 35 | end 36 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | codecov: 2 | require_ci_to_pass: yes 3 | 4 | coverage: 5 | precision: 2 6 | round: down 7 | range: "60...100" 8 | 9 | status: 10 | project: 11 | default: 12 | target: auto 13 | threshold: 1% 14 | paths: 15 | - "lib/" 16 | - "ext/" 17 | patch: 18 | default: 19 | target: auto 20 | threshold: 1% 21 | 22 | parsers: 23 | gcov: 24 | branch_detection: 25 | conditional: yes 26 | loop: yes 27 | method: no 28 | macro: no 29 | 30 | comment: 31 | layout: "reach,diff,flags,files,footer" 32 | behavior: default 33 | require_changes: no 34 | 35 | flags: 36 | ruby: 37 | paths: 38 | - lib/ 39 | carryforward: false 40 | c: 41 | paths: 42 | - ext/ 43 | carryforward: false 44 | 45 | ignore: 46 | - "test/**/*" 47 | - "tests/**/*" 48 | - "spec/**/*" 49 | - "*.gemspec" 50 | - "Gemfile" 51 | - "Rakefile" 52 | -------------------------------------------------------------------------------- /tests/timeout_server.rb: -------------------------------------------------------------------------------- 1 | # This Sinatra application must be run with mongrel 2 | # or possibly with unicorn for the serve action to work properly. 3 | # See http://efreedom.com/Question/1-3669674/Streaming-Data-Sinatra-Rack-Application 4 | 5 | require 'sinatra' 6 | 7 | get '/wait/:time' do |time| 8 | time = time.to_i 9 | sleep(time) 10 | "Slept #{time} at #{Time.now}" 11 | end 12 | 13 | # http://efreedom.com/Question/1-3027435/Way-Flush-Html-Wire-Sinatra 14 | class Streamer 15 | def initialize(time, chunks) 16 | @time = time 17 | @chunks = chunks 18 | end 19 | 20 | def each 21 | @chunks.each do |chunk| 22 | sleep(@time) 23 | yield chunk 24 | end 25 | end 26 | end 27 | 28 | get '/serve/:chunk_size/every/:time/for/:count' do |chunk_size, time, count| 29 | chunk_size, time, count = chunk_size.to_i, time.to_i, count.to_i 30 | chunk = 'x' * chunk_size 31 | chunks = [chunk] * count 32 | Streamer.new(time, chunks) 33 | end 34 | -------------------------------------------------------------------------------- /ext/extconf_coverage.rb: -------------------------------------------------------------------------------- 1 | # Special extconf for coverage builds 2 | # This ensures coverage flags are properly applied 3 | 4 | # First, load the original extconf 5 | load File.join(File.dirname(__FILE__), 'extconf.rb') 6 | 7 | # After the Makefile is created, modify it to add coverage flags 8 | if File.exist?('Makefile') 9 | makefile = File.read('Makefile') 10 | 11 | # Add coverage flags to CFLAGS 12 | makefile.gsub!(/^CFLAGS\s*=(.*)$/) do |match| 13 | "CFLAGS = -fprofile-arcs -ftest-coverage -g -O0 #{$1}" 14 | end 15 | 16 | # Add coverage flags to LDFLAGS 17 | makefile.gsub!(/^LDFLAGS\s*=(.*)$/) do |match| 18 | "LDFLAGS = -fprofile-arcs -ftest-coverage #{$1}" 19 | end 20 | 21 | # Also add to dldflags for shared library 22 | makefile.gsub!(/^dldflags\s*=(.*)$/) do |match| 23 | "dldflags = -fprofile-arcs -ftest-coverage #{$1}" 24 | end 25 | 26 | File.write('Makefile', makefile) 27 | puts "Modified Makefile for coverage support" 28 | end -------------------------------------------------------------------------------- /bench/Rakefile: -------------------------------------------------------------------------------- 1 | task :default do 2 | sh "bundle install" 3 | n = 20000 4 | results = {} 5 | system("cd ../ && rake compile") 6 | [:curb_easy14, :curb_easy, :curb_multi, :curb_multi_using_get, :emhttprequest, :nethttp_test, :patron_test, :typhoeus_hydra_test, :typhoeus_test].each do|bench| 7 | result = `ruby #{bench}.rb #{n}` 8 | results[bench] = result.scan(/Duration: (\d*\.\d*) sec, Memory Usage: (\d*\.\d*) KB - Memory Growth: (\d*\.\d*) KB/).flatten.map {|v| v.to_f} 9 | results.delete(bench) if results[bench].empty? 10 | puts result 11 | end 12 | # find shortest time 13 | best_time = nil 14 | best_key_time = nil 15 | results.each do|k,v| 16 | if best_time.nil? || best_time.first > v.first 17 | best_key_time = k 18 | best_time = v 19 | end 20 | end 21 | puts "Fastest: #{best_key_time} in #{best_time.inspect}" 22 | 23 | # find smallest memory 24 | best_mem = nil 25 | best_key_mem = nil 26 | results.each do|k,v| 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /bench/Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: http://rubygems.org/ 3 | specs: 4 | addressable (2.8.0) 5 | public_suffix (>= 2.0.2, < 5.0) 6 | connection_pool (2.2.5) 7 | cookiejar (0.3.3) 8 | curb (1.0.0) 9 | em-http-request (1.1.7) 10 | addressable (>= 2.3.4) 11 | cookiejar (!= 0.3.1) 12 | em-socksify (>= 0.3) 13 | eventmachine (>= 1.0.3) 14 | http_parser.rb (>= 0.6.0) 15 | em-socksify (0.3.2) 16 | eventmachine (>= 1.0.0.beta.4) 17 | ethon (0.15.0) 18 | ffi (>= 1.15.0) 19 | eventmachine (1.2.7) 20 | ffi (1.15.5) 21 | http_parser.rb (0.8.0) 22 | net-http-persistent (4.0.1) 23 | connection_pool (~> 2.2) 24 | patron (0.13.3) 25 | public_suffix (4.0.6) 26 | rake (13.0.6) 27 | rmem (1.0.0) 28 | typhoeus (1.4.0) 29 | ethon (>= 0.9.0) 30 | 31 | PLATFORMS 32 | ruby 33 | 34 | DEPENDENCIES 35 | curb (= 1.0.0) 36 | em-http-request (>= 1.1.6) 37 | net-http-persistent 38 | patron 39 | rake (>= 13.0.1) 40 | rmem 41 | typhoeus 42 | 43 | BUNDLED WITH 44 | 2.3.3 45 | -------------------------------------------------------------------------------- /doc.rb: -------------------------------------------------------------------------------- 1 | require 'fileutils' 2 | include FileUtils 3 | 4 | begin 5 | incflags = File.read('ext/Makefile')[/INCFLAGS\s*=\s*(.*)$/,1] 6 | rescue Errno::ENOENT 7 | $stderr.puts("No makefile found; run `rake ext/Makefile' first.") 8 | end 9 | 10 | pp_srcdir = 'ext' 11 | 12 | rm_rf(tmpdir = '.doc-tmp') 13 | mkdir(tmpdir) 14 | 15 | begin 16 | if ARGV.include?('--cpp') 17 | begin 18 | if `cpp --version` =~ /\(GCC\)/ 19 | # gnu cpp 20 | $stderr.puts "Running GNU cpp over source" 21 | 22 | Dir['ext/*.c'].each do |fn| 23 | system("cpp -DRDOC_NEVER_DEFINED -C #{incflags} -o " + 24 | "#{File.join(tmpdir, File.basename(fn))} #{fn}") 25 | end 26 | 27 | pp_srcdir = tmpdir 28 | else 29 | $stderr.puts "Not running cpp (non-GNU)" 30 | end 31 | rescue 32 | # no cpp 33 | $stderr.puts "No cpp found" 34 | end 35 | end 36 | 37 | system("rdoc --title='Curb - libcurl bindings for ruby' --main=README #{pp_srcdir}/*.c README LICENSE lib/curb.rb") 38 | ensure 39 | rm_rf(tmpdir) 40 | end 41 | 42 | 43 | -------------------------------------------------------------------------------- /tests/tc_curl_protocols.rb: -------------------------------------------------------------------------------- 1 | class TestCurbCurlProtocols < Test::Unit::TestCase 2 | include TestServerMethods 3 | 4 | def setup 5 | @easy = Curl::Easy.new 6 | @easy.set :protocols, Curl::CURLPROTO_HTTP | Curl::CURLPROTO_HTTPS 7 | @easy.follow_location = true 8 | server_setup 9 | end 10 | 11 | def test_protocol_allowed 12 | @easy.set :url, "http://127.0.0.1:9129/this_file_does_not_exist.html" 13 | @easy.perform 14 | assert_equal 404, @easy.response_code 15 | end 16 | 17 | def test_protocol_denied 18 | @easy.set :url, "gopher://google.com/" 19 | assert_raises Curl::Err::UnsupportedProtocolError do 20 | @easy.perform 21 | end 22 | end 23 | 24 | def test_redir_protocol_allowed 25 | @easy.set :url, TestServlet.url + "/redirect" 26 | @easy.set :redir_protocols, Curl::CURLPROTO_HTTP 27 | @easy.perform 28 | end 29 | 30 | def test_redir_protocol_denied 31 | @easy.set :url, TestServlet.url + "/redirect" 32 | @easy.set :redir_protocols, Curl::CURLPROTO_HTTPS 33 | assert_raises Curl::Err::UnsupportedProtocolError do 34 | @easy.perform 35 | end 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /tests/tc_ftp_options.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.join(File.dirname(__FILE__), 'helper')) 2 | 3 | class TestCurbFtpOptions < Test::Unit::TestCase 4 | # Ensure FTP-related set(:option, ...) mappings are accepted and do not raise 5 | # a TypeError (they used to be unsupported in setopt dispatch). 6 | def test_can_set_ftp_listing_related_flags 7 | c = Curl::Easy.new('ftp://example.com/') 8 | 9 | assert_nothing_raised do 10 | c.set(:dirlistonly, true) if Curl.const_defined?(:CURLOPT_DIRLISTONLY) 11 | c.set(:ftp_use_epsv, 0) if Curl.const_defined?(:CURLOPT_FTP_USE_EPSV) 12 | # These may not be present on all libcurl builds; guard by constant 13 | c.set(:ftp_use_eprt, 0) if Curl.const_defined?(:CURLOPT_FTP_USE_EPRT) 14 | c.set(:ftp_skip_pasv_ip, 1) if Curl.const_defined?(:CURLOPT_FTP_SKIP_PASV_IP) 15 | end 16 | end 17 | 18 | # Setting ftp_commands remains supported for control-connection commands. 19 | def test_can_assign_ftp_commands 20 | c = Curl::Easy.new('ftp://example.com/') 21 | c.ftp_commands = ["PWD", "CWD /"] 22 | assert_kind_of(Array, c.ftp_commands) 23 | assert_equal ["PWD", "CWD /"], c.ftp_commands 24 | end 25 | end 26 | 27 | -------------------------------------------------------------------------------- /ext/curb_postfield.h: -------------------------------------------------------------------------------- 1 | /* curb_postfield.h - Field class for POST method 2 | * Copyright (c)2006 Ross Bamford. 3 | * Licensed under the Ruby License. See LICENSE for details. 4 | * 5 | * $Id: curb_postfield.h 4 2006-11-17 18:35:31Z roscopeco $ 6 | */ 7 | #ifndef __CURB_POSTFIELD_H 8 | #define __CURB_POSTFIELD_H 9 | 10 | #include "curb.h" 11 | 12 | /* 13 | * postfield doesn't actually wrap a curl_httppost - instead, 14 | * it just holds together some ruby objects and has a C-side 15 | * method to add it to a given form list during the perform. 16 | */ 17 | typedef struct { 18 | /* Objects we associate */ 19 | VALUE name; 20 | VALUE content; 21 | VALUE content_type; 22 | VALUE content_proc; 23 | VALUE local_file; 24 | VALUE remote_file; 25 | 26 | /* this will sometimes hold a string, which is the result 27 | * of the content_proc invocation. We need it to hang around. 28 | */ 29 | VALUE buffer_str; 30 | } ruby_curl_postfield; 31 | 32 | extern VALUE cCurlPostField; 33 | 34 | void append_to_form(VALUE self, 35 | struct curl_httppost **first, 36 | struct curl_httppost **last); 37 | 38 | void init_curb_postfield(); 39 | 40 | #endif 41 | -------------------------------------------------------------------------------- /tests/bug_curb_easy_blocks_ruby_threads.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.join(File.dirname(__FILE__), 'helper')) 2 | 3 | class BugTestInstancePostDiffersFromClassPost < Test::Unit::TestCase 4 | include BugTestServerSetupTeardown 5 | 6 | def setup 7 | @port = 9999 8 | @response_proc = lambda do|res| 9 | sleep 0.5 10 | res.body = "hi" 11 | res['Content-Type'] = "text/html" 12 | end 13 | super 14 | end 15 | 16 | def test_bug 17 | threads = [] 18 | timer = Time.now 19 | 20 | 5.times do |i| 21 | t = Thread.new do 22 | c = Curl::Easy.perform('http://127.0.0.1:9999/test') 23 | c.header_str 24 | end 25 | threads << t 26 | end 27 | 28 | multi_responses = threads.collect do|t| 29 | t.value 30 | end 31 | 32 | multi_time = (Time.now - timer) 33 | puts "requested in #{multi_time}" 34 | 35 | timer = Time.now 36 | single_responses = [] 37 | 5.times do |i| 38 | c = Curl::Easy.perform('http://127.0.0.1:9999/test') 39 | single_responses << c.header_str 40 | end 41 | 42 | single_time = (Time.now - timer) 43 | puts "requested in #{single_time}" 44 | 45 | assert single_time > multi_time 46 | end 47 | end 48 | -------------------------------------------------------------------------------- /tests/tc_curl_easy_request_target.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.join(File.dirname(__FILE__), 'helper')) 2 | 3 | class TestCurbCurlEasyRequestTarget < Test::Unit::TestCase 4 | include TestServerMethods 5 | 6 | def setup 7 | server_setup 8 | end 9 | 10 | def test_request_target_absolute_form 11 | unless Curl.const_defined?(:CURLOPT_REQUEST_TARGET) 12 | omit('libcurl lacks CURLOPT_REQUEST_TARGET support') 13 | end 14 | 15 | tmp = Tempfile.new('curb_test_request_target') 16 | path = tmp.path 17 | fd = IO.sysopen(path, 'w') 18 | io = IO.new(fd, 'w') 19 | io.sync = true 20 | 21 | easy = Curl::Easy.new(TestServlet.url) 22 | easy.verbose = true 23 | easy.setopt(Curl::CURLOPT_STDERR, io) 24 | 25 | # Force absolute-form request target, different from the URL host 26 | easy.request_target = "http://localhost:#{TestServlet.port}#{TestServlet.path}" 27 | easy.headers = { 'Host' => "example.com" } 28 | 29 | easy.perform 30 | 31 | io.flush 32 | io.close 33 | output = File.read(path) 34 | 35 | assert_match(/GET\s+http:\/\/localhost:#{TestServlet.port}#{Regexp.escape(TestServlet.path)}\s+HTTP\/1\.1/, output) 36 | assert_match(/Host:\s+example\.com/, output) 37 | ensure 38 | tmp.close! if defined?(tmp) && tmp 39 | end 40 | end 41 | 42 | -------------------------------------------------------------------------------- /tests/bug_crash_on_progress.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.join(File.dirname(__FILE__), 'helper')) 2 | 3 | class BugCrashOnDebug < Test::Unit::TestCase 4 | include BugTestServerSetupTeardown 5 | 6 | def test_on_progress_raise 7 | c = Curl::Easy.new("http://127.0.0.1:#{@port}/test") 8 | c.on_progress do|x| 9 | raise "error" 10 | end 11 | c.perform 12 | 13 | assert false, "should not reach this point" 14 | 15 | rescue => e 16 | assert_equal 'Curl::Err::AbortedByCallbackError', e.class.to_s 17 | c.close 18 | end 19 | 20 | def test_on_progress_abort 21 | # see: https://github.com/taf2/curb/issues/192, 22 | # to pass: 23 | # 24 | # c = Curl::Easy.new('http://127.0.0.1:9999/test') 25 | # c.on_progress do|x| 26 | # puts "we're in the progress callback" 27 | # false 28 | # end 29 | # c.perform 30 | # 31 | # notice no return keyword 32 | # 33 | c = Curl::Easy.new("http://127.0.0.1:#{@port}/test") 34 | did_progress = false 35 | c.on_progress do|x| 36 | did_progress = true 37 | return false 38 | end 39 | c.perform 40 | assert did_progress 41 | 42 | assert false, "should not reach this point" 43 | 44 | rescue => e 45 | assert_equal 'Curl::Err::AbortedByCallbackError', e.class.to_s 46 | c.close 47 | end 48 | 49 | end 50 | -------------------------------------------------------------------------------- /bench/curb_multi.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | $:.unshift File.expand_path(File.dirname(__FILE__)) 3 | require '_usage' 4 | 5 | $:.unshift File.expand_path(File.join(File.dirname(__FILE__),'..','ext')) 6 | $:.unshift File.expand_path(File.join(File.dirname(__FILE__),'..','lib')) 7 | 8 | N = (ARGV.shift || 50).to_i 9 | BURL = 'http://127.0.0.1/zeros-2k' 10 | count = 0 11 | 12 | Memory.usage("Curl::Multi.pipelined(#{N})") do 13 | require 'curb' 14 | 15 | count = 0 16 | multi = Curl::Multi.new 17 | 18 | multi.pipeline = Curl::CURLPIPE_HTTP1 19 | multi.max_connects = 10 20 | 21 | # maintain a free list of easy handles, better to reuse an open connection than create a new one... 22 | free = [] 23 | 24 | pending = N 25 | bytes_received = 0 26 | 27 | # initialize first 10 28 | 10.times do 29 | easy = Curl::Easy.new(BURL + "?n=#{count}") 30 | count+=1 31 | easy.on_body {|d| bytes_received += d.size; d.size } # don't buffer 32 | easy.on_complete do|c| 33 | free << c 34 | end 35 | multi.add easy 36 | pending-=1 37 | break if pending <= 0 38 | end 39 | 40 | until pending == 0 41 | multi.perform do 42 | # idle 43 | if pending > 0 && free.size > 0 44 | easy = free.pop 45 | easy.url = BURL 46 | multi.add easy 47 | pending -= 1 48 | end 49 | end 50 | end 51 | 52 | end 53 | -------------------------------------------------------------------------------- /ext/curb.h: -------------------------------------------------------------------------------- 1 | /* Curb - Libcurl(3) bindings for Ruby. 2 | * Copyright (c)2006 Ross Bamford. 3 | * Licensed under the Ruby License. See LICENSE for details. 4 | * 5 | * $Id: curb.h 39 2006-12-23 15:28:45Z roscopeco $ 6 | */ 7 | 8 | #ifndef __CURB_H 9 | #define __CURB_H 10 | 11 | #include 12 | 13 | #ifdef HAVE_RUBY_IO_H 14 | #include "ruby/io.h" 15 | #else 16 | #include "rubyio.h" // ruby 1.8 17 | #endif 18 | 19 | #include 20 | 21 | #include "banned.h" 22 | #include "curb_config.h" 23 | #include "curb_easy.h" 24 | #include "curb_errors.h" 25 | #include "curb_postfield.h" 26 | #include "curb_multi.h" 27 | 28 | #include "curb_macros.h" 29 | 30 | // These should be managed from the Rake 'release' task. 31 | #define CURB_VERSION "1.2.2" 32 | #define CURB_VER_NUM 1022 33 | #define CURB_VER_MAJ 1 34 | #define CURB_VER_MIN 2 35 | #define CURB_VER_MIC 2 36 | #define CURB_VER_PATCH 0 37 | 38 | 39 | // Maybe not yet defined in Ruby 40 | #ifndef RSTRING_LEN 41 | #define RSTRING_LEN(x) RSTRING(x)->len 42 | #endif 43 | 44 | #ifndef RSTRING_PTR 45 | #define RSTRING_PTR(x) RSTRING(x)->ptr 46 | #endif 47 | 48 | #ifndef RHASH_SIZE 49 | #define RHASH_SIZE(hash) RHASH(hash)->tbl->num_entries 50 | #endif 51 | 52 | // ruby 1.8 does not provide the macro 53 | #ifndef DBL2NUM 54 | #define DBL2NUM(dbl) rb_float_new(dbl) 55 | #endif 56 | 57 | extern VALUE mCurl; 58 | 59 | extern void Init_curb_core(); 60 | 61 | #endif 62 | -------------------------------------------------------------------------------- /tests/bug_require_last_or_segfault.rb: -------------------------------------------------------------------------------- 1 | # From Vlad Jebelev: 2 | # 3 | # - if I have a require statement after "require 'curb'" and there is a 4 | # POST with at least 1 field, the script will fail with a segmentation 5 | # fault, e.g. the following sequence fails every time for me (Ruby 1.8.5): 6 | # ----------------------------------------------------------------- 7 | # require 'curb' 8 | # require 'uri' 9 | # 10 | # url = 'https://www.google.com/accounts/ServiceLoginAuth' 11 | # 12 | # c = Curl::Easy.http_post( 13 | # 'https://www.google.com/accounts/ServiceLoginAuth', 14 | # [Curl:: PostField.content('ltmpl','m_blanco')] ) do |curl| 15 | # end 16 | # ------------------------------------------------------------------ 17 | # :..dev/util$ ruby seg.rb 18 | # seg.rb:6: [BUG] Segmentation fault 19 | # ruby 1.8.5 (2006-08-25) [i686-linux] 20 | # 21 | # Aborted 22 | # ------------------------------------------------------------------ 23 | # 24 | require 'test/unit' 25 | require 'rbconfig' 26 | 27 | $rubycmd = RbConfig::CONFIG['RUBY_INSTALL_NAME'] || 'ruby' 28 | 29 | class BugTestRequireLastOrSegfault < Test::Unit::TestCase 30 | def test_bug 31 | 5.times do |i| 32 | puts "Test ##{i}" 33 | 34 | # will be empty string if it segfaults... 35 | assert_equal 'success', `#$rubycmd #{File.dirname(__FILE__)}/require_last_or_segfault_script.rb`.chomp 36 | sleep 5 37 | end 38 | end 39 | end 40 | 41 | -------------------------------------------------------------------------------- /tests/require_last_or_segfault_script.rb: -------------------------------------------------------------------------------- 1 | # From Vlad Jebelev: 2 | # 3 | # - if I have a require statement after "require 'curb'" and there is a 4 | # POST with at least 1 field, the script will fail with a segmentation 5 | # fault, e.g. the following sequence fails every time for me (Ruby 1.8.5): 6 | # ----------------------------------------------------------------- 7 | # require 'curb' 8 | # require 'uri' 9 | # 10 | # url = 'https://www.google.com/accounts/ServiceLoginAuth' 11 | # 12 | # c = Curl::Easy.http_post( 13 | # 'https://www.google.com/accounts/ServiceLoginAuth', 14 | # [Curl:: PostField.content('ltmpl','m_blanco')] ) do |curl| 15 | # end 16 | # ------------------------------------------------------------------ 17 | # :..dev/util$ ruby seg.rb 18 | # seg.rb:6: [BUG] Segmentation fault 19 | # ruby 1.8.5 (2006-08-25) [i686-linux] 20 | # 21 | # Aborted 22 | # ------------------------------------------------------------------ 23 | # 24 | $:.unshift(File.expand_path(File.join(File.dirname(__FILE__), '..', 'ext'))) 25 | $:.unshift(File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib'))) 26 | require 'curb' 27 | require 'uri' 28 | 29 | url = 'https://www.google.com/accounts/ServiceLoginAuth' 30 | 31 | c = Curl::Easy.http_post('https://www.google.com/accounts/ServiceLoginAuth', 32 | Curl:: PostField.content('ltmpl','m_blanco')) #do 33 | # end 34 | 35 | puts "success" 36 | 37 | -------------------------------------------------------------------------------- /lib/curb.gemspec.erb: -------------------------------------------------------------------------------- 1 | Gem::Specification.new do |s| 2 | s.name = "curb" 3 | s.authors = ["Ross Bamford", "Todd A. Fisher"] 4 | s.version = '<%= CURRENT_VERSION %>' 5 | s.date = '<%= Time.now.strftime("%Y-%m-%d") %>' 6 | s.description = %q{Curb (probably CUrl-RuBy or something) provides Ruby-language bindings for the libcurl(3), a fully-featured client-side URL transfer library. cURL and libcurl live at http://curl.haxx.se/} 7 | s.email = 'todd.fisher@gmail.com' 8 | s.extra_rdoc_files = ['LICENSE', 'README.md'] 9 | <% 10 | files = %w(LICENSE README.md Rakefile doc.rb ext/extconf.rb) + 11 | Dir["lib/**/**.rb"] + 12 | Dir["ext/**/**.c"] + 13 | Dir["ext/**/**.h"]. 14 | reject{|h| h == 'ext/curb_config.h' } 15 | 16 | if ENV['BINARY_PACKAGE'] 17 | files += Dir['ext/**/*.{o,so,bundle}'] 18 | end 19 | %> 20 | s.files = <%= files.inspect %> 21 | 22 | #### Load-time details 23 | s.require_paths = ['lib','ext'] 24 | s.summary = %q{Ruby libcurl bindings} 25 | s.test_files = <%= Dir['tests/**/**.rb'].inspect %> 26 | <% unless ENV['BINARY_PACKAGE'] %> 27 | s.extensions << 'ext/extconf.rb' 28 | <% end %> 29 | 30 | #### Documentation and testing. 31 | s.homepage = 'https://github.com/taf2/curb' 32 | s.rdoc_options = ['--main', 'README.md'] 33 | 34 | s.metadata = { 35 | 'changelog_uri' => 'https://github.com/taf2/curb/blob/master/ChangeLog.md' 36 | } 37 | 38 | <% if ENV['BINARY_PACKAGE'] %> 39 | s.platform = Gem::Platform::CURRENT 40 | <% else %> 41 | s.platform = Gem::Platform::RUBY 42 | <% end %> 43 | s.licenses = ['Ruby'] 44 | end 45 | -------------------------------------------------------------------------------- /tests/bug_instance_post_differs_from_class_post.rb: -------------------------------------------------------------------------------- 1 | # From Vlad Jebelev: 2 | # 3 | # - Second thing - I think you just probably didn't have the time to update 4 | # instance methods yet but when I POST with a reusal of a previous curl 5 | # instance, it doesnt' work for me, e.g. when I create a curl previously and 6 | # then issue: 7 | # 8 | # c.http_post(login_url, *fields) 9 | # 10 | # instead of: 11 | # 12 | # c = Curl::Easy.http_post(login_url, *fields) do |curl| 13 | # ... 14 | # end 15 | # 16 | # then the result I am getting is quite different. 17 | # 18 | # ================ 19 | # 20 | # Update: 21 | # 22 | # It seems that class httpost is incorrectly passing arguments down to 23 | # instance httppost. This bug is intermittent, but results in an 24 | # exception from the first post when it occurs. 25 | # 26 | require File.expand_path(File.join(File.dirname(__FILE__), 'helper')) 27 | 28 | class BugTestInstancePostDiffersFromClassPost < Test::Unit::TestCase 29 | def test_bug 30 | 5.times do |i| 31 | puts "Test ##{i}" 32 | do_test 33 | sleep 2 34 | end 35 | end 36 | 37 | def do_test 38 | c = Curl::Easy.http_post('https://www.google.com/accounts/ServiceLoginAuth', Curl::PostField.content('ltmpl','m_blanco')) 39 | body_c, header_c = c.body_str, c.header_str 40 | 41 | sleep 2 42 | 43 | c.http_post('https://www.google.com/accounts/ServiceLoginAuth', Curl::PostField.content('ltmpl','m_blanco')) 44 | body_i, header_i = c.body, c.head 45 | 46 | # timestamps will differ, just check first bit. We wont get here if 47 | # the bug bites anyway... 48 | assert_equal header_c[0..50], header_i[0..50] 49 | end 50 | end 51 | 52 | -------------------------------------------------------------------------------- /samples/gmail.rb: -------------------------------------------------------------------------------- 1 | # This logs into gmail, up to the point where it hits the 2 | # security redirect implemented as a refresh. It will probably 3 | # stop working altogether when they next change gmail but 4 | # it's still an example of posting with curb... 5 | 6 | $:.unshift(File.join(File.dirname(__FILE__), '..', 'ext')) 7 | $:.unshift(File.join(File.dirname(__FILE__), '..', 'lib')) 8 | require 'curb' 9 | 10 | $EMAIL = '' 11 | $PASSWD = '' 12 | 13 | url = 'https://www.google.com/accounts/ServiceLoginAuth' 14 | 15 | fields = [ 16 | Curl::PostField.content('ltmpl','m_blanco'), 17 | Curl::PostField.content('ltmplcache', '2'), 18 | Curl::PostField.content('continue', 19 | 'http://mail.google.com/mail/?ui.html&zy=l'), 20 | Curl::PostField.content('service', 'mail'), 21 | Curl::PostField.content('rm', 'false'), 22 | Curl::PostField.content('rmShown', '1'), 23 | Curl::PostField.content('PersistentCookie', ''), 24 | Curl::PostField.content('Email', $EMAIL), 25 | Curl::PostField.content('Passwd', $PASSWD) 26 | ] 27 | 28 | c = Curl::Easy.http_post(url, *fields) do |curl| 29 | # Gotta put yourself out there... 30 | curl.headers["User-Agent"] = "Curl/Ruby" 31 | 32 | # Let's see what happens under the hood 33 | curl.verbose = true 34 | 35 | # Google will redirect us a bit 36 | curl.follow_location = true 37 | 38 | # Google will make sure we retain cookies 39 | curl.enable_cookies = true 40 | end 41 | 42 | puts "FINISHED: HTTP #{c.response_code}" 43 | puts c.body_str 44 | 45 | # As an alternative to passing the PostFields, we could have supplied 46 | # individual pre-encoded option strings, or a single string with the 47 | # entire form data. 48 | 49 | -------------------------------------------------------------------------------- /tests/cacert.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE REQUEST----- 2 | MIIEljCCAn4CAQAwUTELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAk1EMRIwEAYDVQQH 3 | EwlCYWx0aW1vcmUxITAfBgNVBAoTGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDCC 4 | AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK07Mq4E65J2YY8X5kl28sXr 5 | QXRLK3MZF2uoB+NrXPm3mshGZ9pGmY6iy+5rtta/aSfJUYuevq3/MBrqSI1SCibG 6 | JsPDajnn0Mofg5Sa0gLSpLA5qvEhNT3PfGf8RbsY1yGKeK7P4vWGBe5CIxYH4yIl 7 | XwYIaSOqqxtmqzKIQeyORB/9s3gbosnUhB2Z1UVwPbdy9AnuPoYNgngdelU0Hq8T 8 | T/CAQrKTdVn0cAsaQ1ZOiFUgDyReY5xuW4gN+SdXYz1ZtT4zgH4+1gRkr1pjv6hb 9 | N7h5UcGZE0sAKyTrQlgAZ0+1rHADwdp5577LJpd0Cf+dw5vxBTo8Dl9mOGnxBQxk 10 | KJOotVdEv2ZcmwLN/8D4/XP9x93zDOv3MGG8+bfC/4o2xDRx14FCYczJWjSIZfEM 11 | Y0cDOVwGSB+IMsPUwYIsI2tTf5FD2cWm0hrsSPXqM90csyuSBlnzpULMXzOC1oFM 12 | es8zqg0/lYEGQ6O8XZIh0MGLLWToozaZgC+XNA5XmvEO+3zhjJWnlqexyy2Ux3q2 13 | JyZF9CcJvdkbZeDft7b0CXioz2ukhIGpfVhqZkF6lVRQJiQVervqLFLrp0Fppu7N 14 | pcGYOXmEf/0OqN6DKSjBiyRy15WAfYRmtnHsnz+cG9YUg7URLVH6+zjvwvWy9Cbt 15 | lWBzyrnSTgNMQJlmF8bzAgMBAAGgADANBgkqhkiG9w0BAQQFAAOCAgEAq1E8hRBz 16 | gPv/B58ru2Kor0QRMuaZELXDZVGXCCqlTQqOmDXkJuc5ll4+VSofpEHRYHRBdrgB 17 | sV4oq0iv9AkRWk991OlgwZijIwka9dGYaMhCOravefJGvkk0gg5E+H8k2qZUEpTK 18 | r+oU9y3da3nLKOiD4jbFS6hhFspv1Cfibv2xSynwWZBNP8IJX0CRIyEoLHvxgZwV 19 | tHZh26R5ysgavqpiAxs5qdWRcG0/cdiAMRbv73LB+pf1eIN+mewbrYNnuKxy++YY 20 | RyyukD7bTN9y20qjN+kZp1dc73cuCELE2/R8DVXQYgQIzkQWbBFhG2+E9tFICXIx 21 | y3GPC7KcUjyaH2vpdCSr4ktNJx9ra37KKJTYlcRVtPcKOAhF76QMvk6R9fTvF6Qe 22 | QNPKrC/sYtGkh9WubGbZFP5FVnONT5Eot3U4JUPCNuYZyLCxhIbjayd7jR2B8hFI 23 | GOFW8oA0sk7lr5obzzExHce1oRUUsiRVVW0xZBo3q0PB5pAPtLIX8KypNHlKu8N1 24 | cAEp+9DzSlWRavDCD3k3q3cnFp3h7IQDFSuU6vioWGhpIiIU2K71Eojfa9VYw6Fb 25 | pZ8WLM6VPqnsjqY/HMgRb8l/Dee9UEJdgTtAuHBE26EmTaQySLam6XtrZPqEjLfG 26 | sch1hu9i2KjTgV4O2MnqgIvhtYHtwxnTanw= 27 | -----END CERTIFICATE REQUEST----- 28 | -------------------------------------------------------------------------------- /tests/bug_issue_spnego.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.join(File.dirname(__FILE__), 'helper')) 2 | 3 | class BugIssueSpnego < Test::Unit::TestCase 4 | def test_spnego_detection_works 5 | # The fix for issue #227 ensures that Curl.spnego? checks for both 6 | # CURL_VERSION_SPNEGO (newer libcurl) and CURL_VERSION_GSSNEGOTIATE (older libcurl) 7 | # 8 | # We can't test that it returns true because that depends on how libcurl 9 | # was compiled on the system. We can only test that: 10 | # 1. The method exists and works 11 | # 2. If CURLAUTH_GSSNEGOTIATE is available, we can use it 12 | 13 | # Check if CURLAUTH_GSSNEGOTIATE is available 14 | if Curl.const_defined?(:CURLAUTH_GSSNEGOTIATE) && Curl.const_get(:CURLAUTH_GSSNEGOTIATE) != 0 15 | # Test that we can use GSSNEGOTIATE auth type 16 | c = Curl::Easy.new('http://example.com') 17 | assert_nothing_raised do 18 | c.http_auth_types = Curl::CURLAUTH_GSSNEGOTIATE 19 | end 20 | 21 | # The fix ensures spnego? won't incorrectly return false when 22 | # older libcurl versions have GSSNEGOTIATE support 23 | # (The actual return value depends on system libcurl compilation) 24 | end 25 | 26 | # The important fix is that the method now checks both constants 27 | # This test passes if no exceptions are raised 28 | assert true, "SPNEGO detection is working correctly" 29 | end 30 | 31 | def test_spnego_method_exists 32 | # The method should always exist 33 | assert Curl.respond_to?(:spnego?), "Curl should respond to spnego?" 34 | end 35 | 36 | def test_spnego_returns_boolean 37 | # The method should return a boolean 38 | result = Curl.spnego? 39 | assert [true, false].include?(result), "Curl.spnego? should return true or false, got #{result.inspect}" 40 | end 41 | end -------------------------------------------------------------------------------- /tests/bug_issue_noproxy.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.join(File.dirname(__FILE__), 'helper')) 2 | 3 | class BugIssueNoproxy < Test::Unit::TestCase 4 | def test_noproxy_option_support 5 | # Test that CURLOPT_NOPROXY constant is defined 6 | assert Curl.const_defined?(:CURLOPT_NOPROXY), "CURLOPT_NOPROXY constant should be defined" 7 | 8 | # Test basic noproxy setting using setopt 9 | c = Curl::Easy.new('https://google.com') 10 | 11 | # This should not raise an error 12 | assert_nothing_raised do 13 | c.setopt(Curl::CURLOPT_NOPROXY, "localhost,127.0.0.1") 14 | end 15 | 16 | # Test using the convenience method if it exists 17 | if c.respond_to?(:noproxy=) 18 | assert_nothing_raised do 19 | c.noproxy = "localhost,127.0.0.1" 20 | end 21 | 22 | # Test getter if it exists 23 | if c.respond_to?(:noproxy) 24 | assert_equal "localhost,127.0.0.1", c.noproxy 25 | end 26 | end 27 | end 28 | 29 | def test_noproxy_with_set_method 30 | c = Curl::Easy.new('https://google.com') 31 | 32 | # The issue specifically mentions using the set method 33 | # This currently raises TypeError as reported in the issue 34 | assert_nothing_raised(TypeError) do 35 | c.set(:noproxy, "localhost,127.0.0.1") 36 | end 37 | end 38 | 39 | def test_noproxy_empty_string 40 | c = Curl::Easy.new('https://google.com') 41 | 42 | # Test setting empty string to override environment variables 43 | assert_nothing_raised do 44 | c.setopt(Curl::CURLOPT_NOPROXY, "") 45 | end 46 | end 47 | 48 | def test_noproxy_nil_value 49 | c = Curl::Easy.new('https://google.com') 50 | 51 | # Test setting nil to reset 52 | assert_nothing_raised do 53 | c.setopt(Curl::CURLOPT_NOPROXY, nil) 54 | end 55 | end 56 | end -------------------------------------------------------------------------------- /samples/downloader.rb: -------------------------------------------------------------------------------- 1 | # Easy download examples 2 | # 3 | # First example, downloads 10 files from the same server sequencially using the same 4 | # easy handle with a persistent connection 5 | # 6 | # Second example, sends all 10 requests in parallel using 10 easy handles 7 | # 8 | $:.unshift(File.expand_path(File.join(File.dirname(__FILE__), '..', 'ext'))) 9 | $:.unshift(File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib'))) 10 | require 'curb' 11 | 12 | urls = ['http://www.cnn.com/', 13 | 'http://www.cnn.com/video/', 14 | 'http://newspulse.cnn.com/', 15 | 'http://www.cnn.com/US/', 16 | 'http://www.cnn.com/WORLD/', 17 | 'http://www.cnn.com/POLITICS/', 18 | 'http://www.cnn.com/JUSTICE/', 19 | 'http://www.cnn.com/SHOWBIZ/', 20 | 'http://www.cnn.com/TECH/', 21 | 'http://www.cnn.com/HEALTH/'] 22 | 23 | def cleanup(urls) 24 | urls.each do|url| 25 | filename = url.split(/\?/).first.split(/\//).last 26 | File.unlink(filename) if File.exist?(filename) 27 | end 28 | end 29 | 30 | 31 | # first sequential 32 | def fetch_sequential(urls) 33 | easy = Curl::Easy.new 34 | easy.follow_location = true 35 | 36 | urls.each do|url| 37 | easy.url = url 38 | filename = url.split(/\?/).first.split(/\//).last 39 | print "'#{url}' :" 40 | File.open(filename, 'wb') do|f| 41 | easy.on_progress {|dl_total, dl_now, ul_total, ul_now| print "="; true } 42 | easy.on_body {|data| f << data; data.size } 43 | easy.perform 44 | puts "=> '#{filename}'" 45 | end 46 | end 47 | end 48 | 49 | 50 | # using multi interface 51 | def fetch_parallel(urls) 52 | Curl::Multi.download(urls){|c,code,method| 53 | filename = c.url.split(/\?/).first.split(/\//).last 54 | puts filename 55 | } 56 | end 57 | 58 | 59 | fetch_sequential(urls) 60 | cleanup(urls) 61 | fetch_parallel(urls) 62 | -------------------------------------------------------------------------------- /tasks/utils.rb: -------------------------------------------------------------------------------- 1 | require 'open3' 2 | 3 | module Curb 4 | module RakeHelpers 5 | # A simple shell wrapper using stdlib open3. It implements very basic set of methods to 6 | # mimic MixLib. 7 | # 8 | # It's used for old rubies where MixLib is not available. 9 | class ShellWrapper 10 | def initialize(args, opts = {}) 11 | @cmd, @live_stream, @cwd = args, opts[:live_stdout], opts[:cwd] 12 | @stdout, @stderr = '', '' 13 | end 14 | 15 | 16 | def run_command 17 | # Ruby 1.8 Open3.popen3 does not support changing directory, we 18 | # need to do it before shelling out. 19 | if @cwd 20 | Dir.chdir(@cwd) { execute! } 21 | else 22 | execute! 23 | end 24 | self 25 | end 26 | 27 | def execute! 28 | wait_thr = nil 29 | 30 | Open3.popen3(*@cmd) do |stdin, stdout, stderr, thr| 31 | stdin.close 32 | wait_thr = thr # Ruby 1.8 will not yield thr, this will be nil 33 | 34 | while line = stdout.gets do 35 | @stdout << line 36 | @live_stream.puts(line) if @live_stream 37 | end 38 | 39 | while line = stderr.gets do 40 | @stderr << line 41 | puts line 42 | end 43 | end 44 | 45 | # prefer process handle directly from popen3, but if not available 46 | # fallback to global. 47 | p_status = wait_thr ? wait_thr.value : $? 48 | @exit_code = p_status.exitstatus 49 | @error = (@exit_code != 0) 50 | end 51 | 52 | def stderr 53 | @stderr 54 | end 55 | 56 | def stdout 57 | @stdout 58 | end 59 | 60 | def error! 61 | fail "Command failed with exit-code #{@exit_code}" if @error 62 | end 63 | 64 | def error? 65 | !!@error 66 | end 67 | end 68 | end 69 | end 70 | -------------------------------------------------------------------------------- /tests/tc_gc_compact.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require File.expand_path('helper', __dir__) 4 | 5 | class TestGcCompact < Test::Unit::TestCase 6 | ITERATIONS = (ENV['CURB_GC_COMPACT_ITERATIONS'] || 50).to_i 7 | EASY_PER_MULTI = 3 8 | 9 | def setup 10 | omit('GC.compact unavailable on this Ruby') unless defined?(GC.compact) 11 | end 12 | 13 | def test_multi_perform_with_gc_compact 14 | ITERATIONS.times do 15 | multi = Curl::Multi.new 16 | add_easy_handles(multi) 17 | 18 | compact 19 | assert_nothing_raised { multi.perform } 20 | compact 21 | end 22 | end 23 | 24 | def test_gc_compact_during_multi_cleanup 25 | ITERATIONS.times do 26 | multi = Curl::Multi.new 27 | add_easy_handles(multi) 28 | 29 | compact 30 | multi = nil 31 | compact 32 | end 33 | end 34 | 35 | def test_gc_compact_after_detach 36 | multi = Curl::Multi.new 37 | handles = add_easy_handles(multi) 38 | 39 | compact 40 | assert_nothing_raised { multi.perform } 41 | 42 | handles.each { |easy| multi.remove(easy) } 43 | compact 44 | end 45 | 46 | def test_gc_compact_easy 47 | iteration = 0 48 | responses = [] 49 | while iteration < ITERATIONS 50 | res = Curl.get($TEST_URL) do |easy| 51 | easy.timeout = 5 52 | easy.on_complete { |_e| } 53 | easy.on_failure { |_e, _code| } 54 | end 55 | iteration += 1 56 | responses << res.body 57 | compact 58 | end 59 | end 60 | 61 | private 62 | 63 | def add_easy_handles(multi) 64 | Array.new(EASY_PER_MULTI) do 65 | Curl::Easy.new($TEST_URL) do |easy| 66 | easy.timeout = 5 67 | easy.on_complete { |_e| } 68 | easy.on_failure { |_e, _code| } 69 | end.tap { |easy| multi.add(easy) } 70 | end 71 | end 72 | 73 | def compact 74 | GC.compact 75 | end 76 | end 77 | -------------------------------------------------------------------------------- /bench/zeros-2k: -------------------------------------------------------------------------------- 1 | 00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 2 | -------------------------------------------------------------------------------- /tests/cert.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIICXAIBAAKBgQC3TtLMdycGQclGWVDqlCw/DVPxkU9+TJwr/mjkdSea5vkbp8wT 3 | LD4gvCdKWsPqtSpPtYgaPZ2Ly82FOceq+zhA4WEk16ms5DqvFAHoDlE5oObg6SyK 4 | ByCnw2yoU4ga0dqzc+FPCyl9IUpCkEN8Rl6nPqg4Rr22h+ElSZde+94BTwIDAQAB 5 | AoGBAKm9Vbl7xCkpFcYMwr7VQjuIjeis091x91NNm7ehPHFV2+pd8Lz4RPdvAzr4 6 | 3V4jL/DregJCd/aRW9g37spHIohuglHA0GSZwOxTQ9zDMtmSIInM0n2UL6LZL9l5 7 | Q+WChB4GqFitNEl/33+6CgR77VDr9dUAmc8WQRieUU+PaG6hAkEA7y2mkFFZLK8G 8 | X5N2DfuiFWonV0P1gy5CYhF4zUsFJ+dCf4TIcAi24VVbK8USqYoqjdCOyZutoSb0 9 | i+D+RkV51wJBAMQzPTNIElbLIz/kYl0Brx0F1THpnb4UiSHdpGdyWg0pZIpmtBdO 10 | aA9W1RP7V/Ivy7atdO8G/MtKMhfrdGO5dUkCQE3L9oK4wx3CrHsIFv1DXRxEFBnR 11 | dBlAQb1uW3HDNiEdmsappRyz6PBweCBLkN9unprUPK2dIqPpbN/Wxj6LOK8CQDk/ 12 | xkTXa9p9jbyP9I+09Rbf49SbmVakgVsrZFR3DoW2pUqpKzV9wGlxad1ZwtC9V5Dn 13 | Ti6M+GiNLs7B+oU60VkCQHnsaE9C/r/qLUDNi/MqkFZJEhYGeVYPcWgh6MeQajkH 14 | nEu7pu+ySxgyq7RazlEDDsFlUESrfyx20wEO7YQxJio= 15 | -----END RSA PRIVATE KEY----- 16 | -----BEGIN CERTIFICATE----- 17 | MIIC1TCCAj6gAwIBAgIJALX/ECcf/REhMA0GCSqGSIb3DQEBBAUAMFExCzAJBgNV 18 | BAYTAlVTMQswCQYDVQQIEwJNRDESMBAGA1UEBxMJQmFsdGltb3JlMSEwHwYDVQQK 19 | ExhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwHhcNMDkwNzAxMTg0MjE1WhcNMTAw 20 | NzAxMTg0MjE1WjBRMQswCQYDVQQGEwJVUzELMAkGA1UECBMCTUQxEjAQBgNVBAcT 21 | CUJhbHRpbW9yZTEhMB8GA1UEChMYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIGf 22 | MA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC3TtLMdycGQclGWVDqlCw/DVPxkU9+ 23 | TJwr/mjkdSea5vkbp8wTLD4gvCdKWsPqtSpPtYgaPZ2Ly82FOceq+zhA4WEk16ms 24 | 5DqvFAHoDlE5oObg6SyKByCnw2yoU4ga0dqzc+FPCyl9IUpCkEN8Rl6nPqg4Rr22 25 | h+ElSZde+94BTwIDAQABo4G0MIGxMB0GA1UdDgQWBBT3FKoWGmvQCtpeTGOacrRr 26 | KJUlWTCBgQYDVR0jBHoweIAU9xSqFhpr0AraXkxjmnK0ayiVJVmhVaRTMFExCzAJ 27 | BgNVBAYTAlVTMQswCQYDVQQIEwJNRDESMBAGA1UEBxMJQmFsdGltb3JlMSEwHwYD 28 | VQQKExhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGSCCQC1/xAnH/0RITAMBgNVHRME 29 | BTADAQH/MA0GCSqGSIb3DQEBBAUAA4GBAJb9UIK/OkNwNTz40GowRpg/vOyL8ciI 30 | Eayor9ECqzykmG45DITCyekrpfL9sltPuQtgEhY8fS08B3MVkuWH2Fqx97sP6voD 31 | hWJf98QTchs6AIRwq9xaqQnt+o6DtGNgSFaqU2huURYVuXnxHAM7qNnNkZuaAEmn 32 | xp8bjBWgywre 33 | -----END CERTIFICATE----- 34 | -------------------------------------------------------------------------------- /tests/mem_check.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.join(File.dirname(__FILE__), 'helper')) 2 | #require 'rubygems' 3 | #require 'rmem' 4 | 5 | # 6 | # Run some tests to measure the memory usage of curb, these tests require fork and ps 7 | # 8 | class TestCurbMemory < Test::Unit::TestCase 9 | def setup 10 | omit('Memory fork/ps tests not supported on Windows') if WINDOWS || NO_FORK 11 | end 12 | 13 | def test_easy_memory 14 | easy_avg, easy_std = measure_object_memory(Curl::Easy) 15 | printf "Easy average: %.2f kilobytes +/- %.2f kilobytes\n", easy_avg.to_f, easy_std.to_f 16 | 17 | multi_avg, multi_std = measure_object_memory(Curl::Multi) 18 | printf "Multi average: %.2f kilobytes +/- %.2f kilobytes\n", multi_avg.to_f, multi_std.to_f 19 | 20 | # now that we have the average size of an easy handle lets see how much a multi request consumes with 10 requests 21 | end 22 | 23 | def c_avg(report) 24 | sum = 0 25 | report.each {|r| sum += r.last } 26 | (sum.to_f / report.size) 27 | end 28 | 29 | def c_std(report,avg) 30 | var = 0 31 | report.each {|r| var += (r.last-avg)*(r.last-avg) } 32 | Math.sqrt(var / (report.size-1)) 33 | end 34 | 35 | def measure_object_memory(klass) 36 | report = [] 37 | 200.times do 38 | res = mem_check do 39 | obj = klass.new 40 | end 41 | report << res 42 | end 43 | avg = c_avg(report) 44 | std = c_std(report,avg) 45 | [avg,std] 46 | end 47 | 48 | def mem_check 49 | # see: http://gist.github.com/264060 for inspiration of ps command line 50 | rd, wr = IO.pipe 51 | memory_usage = `ps -o rss= -p #{Process.pid}`.to_i # in kilobytes 52 | fork do 53 | before = `ps -o rss= -p #{Process.pid}`.to_i # in kilobytes 54 | rd.close 55 | yield 56 | after = `ps -o rss= -p #{Process.pid}`.to_i # in kilobytes 57 | wr.write((after - before)) 58 | wr.flush 59 | wr.close 60 | end 61 | wr.close 62 | total = rd.read.to_i 63 | rd.close 64 | Process.wait 65 | # return the delta and the total 66 | [memory_usage, total] 67 | end 68 | end 69 | -------------------------------------------------------------------------------- /tests/bug_postfields_crash2.rb: -------------------------------------------------------------------------------- 1 | # Not sure if this is an IRB bug, but thought you guys should know. 2 | # 3 | # ** My Ruby version: ruby 1.8.7 (2009-06-12 patchlevel 174) [x86_64-linux] 4 | # ** Version of Rubygems: 1.3.5 5 | # ** Version of the Curb gem: 0.6.6.0 6 | # 7 | # 8 | # Transcript of IRB session: 9 | # ------------------------------------------------------------------------------------------------------------------ 10 | # irb(main):001:0> a = { 11 | # irb(main):002:1* :type => :pie, 12 | # irb(main):003:1* :series => { 13 | # irb(main):004:2* :names => [:a,:b], 14 | # irb(main):005:2* :values => [70,30], 15 | # irb(main):006:2* :colors => [:red,:green] 16 | # irb(main):007:2> }, 17 | # irb(main):008:1* :output_format => :png 18 | # irb(main):009:1> } 19 | # => {:type=>:pie, :output_format=>:png, :series=>{:names=>[:a, :b], :values=>[70, 30], :colors=>[:red, :green]}} 20 | # irb(main):010:0> post = [] 21 | # => [] 22 | # irb(main):011:0> require 'rubygems' 23 | # => true 24 | # irb(main):012:0> require 'curb' 25 | # => true 26 | # irb(main):013:0> include Curl 27 | # => Object 28 | # irb(main):014:0> a.each_pair do |k,v| 29 | # irb(main):015:1* post << PostField.content(k,v) 30 | # irb(main):016:1> end 31 | # => {:type=>:pie, :output_format=>:png, :series=>{:names=>[:a, :b], :values=>[70, 30], :colors=>[:red, :green]}} 32 | # irb(main):017:0> post 33 | # /usr/lib/ruby/1.8/irb.rb:302: [BUG] Segmentation fault 34 | # ruby 1.8.7 (2009-06-12 patchlevel 174) [x86_64-linux] 35 | # 36 | # Aborted 37 | # ------------------------------------------------------------------------------------------------------------------ 38 | require File.expand_path(File.join(File.dirname(__FILE__), 'helper')) 39 | 40 | class BugPostFieldsCrash2 < Test::Unit::TestCase 41 | def test_crash 42 | a = { 43 | :type => :pie, 44 | :series => { 45 | :names => [:a,:b], 46 | :values => [70,30], 47 | :colors => [:red,:green] 48 | }, 49 | :output_format => :png 50 | } 51 | post = [] 52 | a.each_pair do |k,v| 53 | post << Curl::PostField.content(k,v) 54 | end 55 | post.inspect 56 | end 57 | end 58 | -------------------------------------------------------------------------------- /lib/curl.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | require 'curb_core' 3 | require 'curl/easy' 4 | require 'curl/multi' 5 | require 'uri' 6 | 7 | # expose shortcut methods 8 | module Curl 9 | 10 | def self.http(verb, url, post_body=nil, put_data=nil, &block) 11 | if Thread.current[:curb_curl_yielding] 12 | handle = Curl::Easy.new # we can't reuse this 13 | else 14 | handle = Thread.current[:curb_curl] ||= Curl::Easy.new 15 | handle.reset 16 | end 17 | handle.url = url 18 | handle.post_body = post_body if post_body 19 | handle.put_data = put_data if put_data 20 | if block_given? 21 | Thread.current[:curb_curl_yielding] = true 22 | yield handle 23 | Thread.current[:curb_curl_yielding] = false 24 | end 25 | handle.http(verb) 26 | handle 27 | end 28 | 29 | def self.get(url, params={}, &block) 30 | http :GET, urlalize(url, params), nil, nil, &block 31 | end 32 | 33 | def self.post(url, params={}, &block) 34 | http :POST, url, postalize(params), nil, &block 35 | end 36 | 37 | def self.put(url, params={}, &block) 38 | http :PUT, url, nil, postalize(params), &block 39 | end 40 | 41 | def self.delete(url, params={}, &block) 42 | http :DELETE, url, postalize(params), nil, &block 43 | end 44 | 45 | def self.patch(url, params={}, &block) 46 | http :PATCH, url, postalize(params), nil, &block 47 | end 48 | 49 | def self.head(url, params={}, &block) 50 | http :HEAD, urlalize(url, params), nil, nil, &block 51 | end 52 | 53 | def self.options(url, params={}, &block) 54 | http :OPTIONS, urlalize(url, params), nil, nil, &block 55 | end 56 | 57 | def self.urlalize(url, params={}) 58 | uri = URI(url) 59 | # early return if we didn't specify any extra params 60 | return uri.to_s if (params || {}).empty? 61 | 62 | params_query = URI.encode_www_form(params || {}) 63 | uri.query = [uri.query.to_s, params_query].reject(&:empty?).join('&') 64 | uri.to_s 65 | end 66 | 67 | def self.postalize(params={}) 68 | params.respond_to?(:map) ? URI.encode_www_form(params) : (params.respond_to?(:to_s) ? params.to_s : params) 69 | end 70 | 71 | def self.reset 72 | Thread.current[:curb_curl] = Curl::Easy.new 73 | end 74 | 75 | end 76 | -------------------------------------------------------------------------------- /tests/tc_curl.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.join(File.dirname(__FILE__), 'helper')) 2 | 3 | class TestCurl < Test::Unit::TestCase 4 | def test_get 5 | curl = Curl.get(TestServlet.url, {:foo => "bar"}) 6 | assert_equal "GETfoo=bar", curl.body_str 7 | 8 | curl = Curl.options(TestServlet.url, {:foo => "bar"}) do|http| 9 | http.headers['Cookie'] = 'foo=1;bar=2' 10 | end 11 | assert_equal "OPTIONSfoo=bar", curl.body_str 12 | end 13 | 14 | def test_post 15 | curl = Curl.post(TestServlet.url, {:foo => "bar"}) 16 | assert_equal "POST\nfoo=bar", curl.body_str 17 | end 18 | 19 | def test_put 20 | curl = Curl.put(TestServlet.url, {:foo => "bar"}) 21 | assert_equal "PUT\nfoo=bar", curl.body_str 22 | end 23 | 24 | def test_patch 25 | curl = Curl.patch(TestServlet.url, {:foo => "bar"}) 26 | assert_equal "PATCH\nfoo=bar", curl.body_str 27 | end 28 | 29 | def test_options 30 | curl = Curl.options(TestServlet.url, {:foo => "bar"}) 31 | assert_equal "OPTIONSfoo=bar", curl.body_str 32 | end 33 | 34 | def test_urlalize_without_extra_params 35 | url_no_params = 'http://localhost/test' 36 | url_with_params = 'http://localhost/test?a=1' 37 | 38 | assert_equal(url_no_params, Curl.urlalize(url_no_params)) 39 | assert_equal(url_with_params, Curl.urlalize(url_with_params)) 40 | end 41 | 42 | def test_urlalize_with_nil_as_params 43 | url = 'http://localhost/test' 44 | assert_equal(url, Curl.urlalize(url, nil)) 45 | end 46 | 47 | def test_urlalize_with_extra_params 48 | url_no_params = 'http://localhost/test' 49 | url_with_params = 'http://localhost/test?a=1' 50 | extra_params = { :b => 2 } 51 | 52 | expected_url_no_params = 'http://localhost/test?b=2' 53 | expected_url_with_params = 'http://localhost/test?a=1&b=2' 54 | 55 | assert_equal(expected_url_no_params, Curl.urlalize(url_no_params, extra_params)) 56 | assert_equal(expected_url_with_params, Curl.urlalize(url_with_params, extra_params)) 57 | end 58 | 59 | def test_urlalize_does_not_strip_trailing_? 60 | url_empty_params = 'http://localhost/test?' 61 | assert_equal(url_empty_params, Curl.urlalize(url_empty_params)) 62 | end 63 | 64 | include TestServerMethods 65 | 66 | def setup 67 | server_setup 68 | end 69 | end 70 | -------------------------------------------------------------------------------- /tests/bug_curb_easy_post_with_string_no_content_length_header.rb: -------------------------------------------------------------------------------- 1 | =begin 2 | From jwhitmire 3 | Todd, I'm trying to use curb to post data to a REST url. We're using it to post support questions from our iphone app directly to tender. The post looks good to me, but curl is not adding the content-length header so I get a 411 length required response from the server. 4 | 5 | Here's my post block, do you see anything obvious? Do I need to manually add the Content-Length header? 6 | 7 | c = Curl::Easy.http_post(url) do |curl| 8 | curl.headers["User-Agent"] = "Curl/Ruby" 9 | if user 10 | curl.headers["X-Multipass"] = user.multipass 11 | else 12 | curl.headers["X-Tender-Auth"] = TOKEN 13 | end 14 | curl.headers["Accept"] = "application/vnd.tender-v1+json" 15 | 16 | curl.post_body = params.map{|f,k| "#{curl.escape(f)}=#{curl.escape(k)}"}.join('&') 17 | 18 | curl.verbose = true 19 | curl.follow_location = true 20 | curl.enable_cookies = true 21 | end 22 | Any insight you care to share would be helpful. Thanks. 23 | =end 24 | require File.expand_path(File.join(File.dirname(__FILE__), 'helper')) 25 | 26 | class BugCurbEasyPostWithStringNoContentLengthHeader < Test::Unit::TestCase 27 | include BugTestServerSetupTeardown 28 | 29 | def test_bug_workaround 30 | params = {:cat => "hat", :foo => "bar"} 31 | 32 | post_body = params.map{|f,k| "#{Curl::Easy.new.escape(f)}=#{Curl::Easy.new.escape(k)}"}.join('&') 33 | c = Curl::Easy.http_post("http://127.0.0.1:#{@port}/test",post_body) do |curl| 34 | curl.headers["User-Agent"] = "Curl/Ruby" 35 | curl.headers["X-Tender-Auth"] = "A Token" 36 | curl.headers["Accept"] = "application/vnd.tender-v1+json" 37 | 38 | curl.follow_location = true 39 | curl.enable_cookies = true 40 | end 41 | 42 | end 43 | 44 | def test_bug 45 | params = {:cat => "hat", :foo => "bar"} 46 | 47 | c = Curl::Easy.http_post("http://127.0.0.1:#{@port}/test") do |curl| 48 | curl.headers["User-Agent"] = "Curl/Ruby" 49 | curl.headers["X-Tender-Auth"] = "A Token" 50 | curl.headers["Accept"] = "application/vnd.tender-v1+json" 51 | 52 | curl.post_body = params.map{|f,k| "#{curl.escape(f)}=#{curl.escape(k)}"}.join('&') 53 | 54 | curl.follow_location = true 55 | curl.enable_cookies = true 56 | end 57 | end 58 | 59 | end 60 | -------------------------------------------------------------------------------- /bench/README: -------------------------------------------------------------------------------- 1 | Each benchmark requires a webserver be running on port 80 with a URL of http://127.0.0.1/zeros-2k 2 | 3 | Memory usage is reported using the rmem gem. 4 | 5 | 6 | === 7 | OSX you might experience very bad performance from libcurl... it is because of OpenSSL 8 | 9 | 10 | Running on my Linux machine: 11 | 12 | Processor: Intel(R) Core(TM)2 Quad CPU Q6600 @ 2.40GHz 13 | Memory: 3G 14 | OS: Linux spin 2.6.35.11-83.fc14.i686 #1 SMP Mon Feb 7 07:04:18 UTC 2011 i686 i686 i386 GNU/Linux 15 | libcurl: 7.21.0 16 | openssl: 1.0.0d 17 | ruby: ruby 1.9.2p136 (2010-12-25 revision 30365) [i686-linux] 18 | webserver: nginx/0.8.53 19 | 20 | Curl::Easy(5050) Duration: 0.6169 sec, Memory Usage: 27.71 KB - Memory Growth: 25.31 KB 21 | Curl::Multi(5050) Duration: 0.5969 sec, Memory Usage: 8.33 KB - Memory Growth: 5.94 KB 22 | Net::HTTP(5050) Duration: 1.9837 sec, Memory Usage: 73.90 KB - Memory Growth: 71.51 KB 23 | Patron(5050) Duration: 0.9795 sec, Memory Usage: 88.94 KB - Memory Growth: 86.56 KB 24 | Typhoeus::Hydra(5050) Duration: 0.2877 sec, Memory Usage: 8.90 KB - Memory Growth: 6.51 KB 25 | Typhoeus(5050) Duration: 1.2635 sec, Memory Usage: 18.78 KB - Memory Growth: 16.39 KB 26 | 27 | Running on my MacBook Pro 28 | 29 | Processor: 2.53 GHz Intel Core 2 Duo 30 | Memory: 4 GB 31 | OS: Darwin glazgo 10.6.0 Darwin Kernel Version 10.6.0: Wed Nov 10 18:13:17 PST 2010; root:xnu-1504.9.26~3/RELEASE_I386 i386 i386 MacBookPro5,1 Darwin 32 | libcurl: 7.19.7 33 | openssl: 0.9.8l 34 | ruby: ruby 1.8.7 (2010-04-19 patchlevel 253) [i686-darwin10.6.0], MBARI 0x6770, Ruby Enterprise Edition 2010.02 35 | webserver: nginx/0.7.65 36 | 37 | Curl::Easy(5050) Duration: 0.8870 sec, Memory Usage: 41900.00 KB - Memory Growth: 11068.00 KB 38 | Curl::Multi(5050) Duration: 0.9340 sec, Memory Usage: 41408.00 KB - Memory Growth: 10572.00 KB 39 | Net::HTTP(5050) Duration: 1.8687 sec, Memory Usage: 71660.00 KB - Memory Growth: 40828.00 KB 40 | Patron(5050) Duration: 1.2757 sec, Memory Usage: 38108.00 KB - Memory Growth: 7280.00 KB 41 | Typhoeus::Hydra(5050) Duration: 0.4511 sec, Memory Usage: 36932.00 KB - Memory Growth: 6100.00 KB 42 | Typhoeus(5050) Duration: 1.7582 sec, Memory Usage: 55812.00 KB - Memory Growth: 24984.00 KB 43 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2006 Ross Bamford (rosco AT roscopeco DOT co DOT uk). 2 | Curb is free software licensed under the following terms: 3 | 4 | 1. You may make and give away verbatim copies of the source form of the 5 | software without restriction, provided that you duplicate all of the 6 | original copyright notices and associated disclaimers. 7 | 8 | 2. You may modify your copy of the software in any way, provided that 9 | you do at least ONE of the following: 10 | 11 | a) place your modifications in the Public Domain or otherwise 12 | make them Freely Available, such as by posting said 13 | modifications to Usenet or an equivalent medium, or by allowing 14 | the author to include your modifications in the software. 15 | 16 | b) use the modified software only within your corporation or 17 | organization. 18 | 19 | c) give non-standard binaries non-standard names, with 20 | instructions on where to get the original software distribution. 21 | 22 | d) make other distribution arrangements with the author. 23 | 24 | 3. You may distribute the software in object code or binary form, 25 | provided that you do at least ONE of the following: 26 | 27 | a) distribute the binaries and library files of the software, 28 | together with instructions (in the manual page or equivalent) 29 | on where to get the original distribution. 30 | 31 | b) accompany the distribution with the machine-readable source of 32 | the software. 33 | 34 | c) give non-standard binaries non-standard names, with 35 | instructions on where to get the original software distribution. 36 | 37 | d) make other distribution arrangements with the author. 38 | 39 | 4. You may modify and include the part of the software into any other 40 | software (possibly commercial). 41 | 42 | 5. The scripts and library files supplied as input to or produced as 43 | output from the software do not automatically fall under the 44 | copyright of the software, but belong to whomever generated them, 45 | and may be sold commercially, and may be aggregated with this 46 | software. 47 | 48 | 6. THIS SOFTWARE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR 49 | IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED 50 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 51 | PURPOSE. 52 | -------------------------------------------------------------------------------- /samples/fiber_crawler.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | # Use the locally built curb (extension + libs) instead of any installed gem. 4 | $LOAD_PATH.unshift File.expand_path('../lib', __dir__) 5 | $LOAD_PATH.unshift File.expand_path('../ext', __dir__) 6 | 7 | require 'curl' 8 | 9 | # Demo parameters 10 | GROUPS = (ENV['GROUPS'] || 3).to_i 11 | PER_GROUP = (ENV['PER_GROUP'] || 5).to_i 12 | CONNECT_TIMEOUT = (ENV['CONNECT_TIMEOUT'] || 5).to_i 13 | TOTAL_TIMEOUT = (ENV['TOTAL_TIMEOUT'] || 15).to_i 14 | DELAY_S = (ENV['DELAY_S'] || 0.25).to_f 15 | 16 | require 'webrick' 17 | require 'async' 18 | 19 | # Start a small local HTTP server on loopback and wait until it’s listening. 20 | ready = Queue.new 21 | server = WEBrick::HTTPServer.new( 22 | BindAddress: '127.0.0.1', 23 | Port: (ENV['PORT'] || 0).to_i, # 0 chooses a free port 24 | Logger: WEBrick::Log.new($stderr, WEBrick::Log::FATAL), 25 | AccessLog: [], 26 | StartCallback: -> { ready << true } 27 | ) 28 | server.mount_proc('/test') do |_req, res| 29 | sleep DELAY_S 30 | res.status = 200 31 | res['Content-Type'] = 'text/plain' 32 | res.body = 'OK' 33 | end 34 | 35 | server_thread = Thread.new { server.start } 36 | ready.pop # wait until the server is bound 37 | bound_port = server.listeners.first.addr[1] 38 | URL = "http://127.0.0.1:#{bound_port}/test" 39 | 40 | puts "Async demo: groups=#{GROUPS} per_group=#{PER_GROUP} url=#{URL} delay=#{DELAY_S}s" 41 | 42 | def crawl(url) 43 | c = Curl::Easy.new 44 | c.url = url 45 | c.connect_timeout = CONNECT_TIMEOUT 46 | c.timeout = TOTAL_TIMEOUT 47 | start = Time.now 48 | code = nil 49 | error = nil 50 | begin 51 | c.perform 52 | code = c.response_code 53 | rescue => e 54 | code = -1 55 | error = "#{e.class}: #{e.message}" 56 | end 57 | { code: code, error: error, duration: (Time.now - start) } 58 | end 59 | 60 | results = [] 61 | started = Time.now 62 | if Async.respond_to?(:run) 63 | Async.run do |top| 64 | GROUPS.times do 65 | PER_GROUP.times do 66 | top.async do 67 | results << crawl(URL) 68 | end 69 | end 70 | end 71 | end 72 | else 73 | Async do |top| 74 | GROUPS.times do 75 | PER_GROUP.times do 76 | top.async do 77 | results << crawl(URL) 78 | end 79 | end 80 | end 81 | end 82 | end 83 | duration = Time.now - started 84 | 85 | ok = results.count { |r| r[:code] == 200 } 86 | fail = results.count { |r| r[:code] != 200 } 87 | puts "Completed #{results.size} requests in #{duration.round(3)}s (ok=#{ok}, fail=#{fail})" 88 | 89 | server.shutdown 90 | server_thread.join 91 | -------------------------------------------------------------------------------- /tests/tc_curl_download.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.join(File.dirname(__FILE__), 'helper')) 2 | 3 | class TestCurbCurlDownload < Test::Unit::TestCase 4 | include TestServerMethods 5 | 6 | def setup 7 | server_setup 8 | end 9 | 10 | def test_download_url_to_file_via_string 11 | dl_url = "http://127.0.0.1:9129/ext/curb_easy.c" 12 | dl_path = File.join(Dir::tmpdir, "dl_url_test.file") 13 | 14 | Curl::Easy.download(dl_url, dl_path) 15 | assert File.exist?(dl_path) 16 | assert_equal File.read(File.join(File.dirname(__FILE__), '..','ext','curb_easy.c')), File.read(dl_path) 17 | ensure 18 | File.unlink(dl_path) if File.exist?(dl_path) 19 | end 20 | 21 | def test_download_url_to_file_via_file_io 22 | dl_url = "http://127.0.0.1:9129/ext/curb_easy.c" 23 | dl_path = File.join(Dir::tmpdir, "dl_url_test.file") 24 | io = File.open(dl_path, 'wb') 25 | 26 | Curl::Easy.download(dl_url, io) 27 | assert io.closed? 28 | assert File.exist?(dl_path) 29 | assert_equal File.read(File.join(File.dirname(__FILE__), '..','ext','curb_easy.c')), File.read(dl_path) 30 | ensure 31 | File.unlink(dl_path) if File.exist?(dl_path) 32 | end 33 | 34 | def test_download_url_to_file_via_io 35 | omit('fork not available on this platform') if NO_FORK || WINDOWS 36 | dl_url = "http://127.0.0.1:9129/ext/curb_easy.c" 37 | dl_path = File.join(Dir::tmpdir, "dl_url_test.file") 38 | reader, writer = IO.pipe 39 | 40 | # Write to local file 41 | fork do 42 | begin 43 | writer.close 44 | File.open(dl_path, 'wb') { |file| file << reader.read } 45 | ensure 46 | reader.close rescue IOError # if the stream has already been closed 47 | end 48 | end 49 | 50 | # Download remote source 51 | begin 52 | reader.close 53 | Curl::Easy.download(dl_url, writer) 54 | Process.wait 55 | ensure 56 | writer.close rescue IOError # if the stream has already been closed, which occurs in Easy::download 57 | end 58 | 59 | assert File.exist?(dl_path) 60 | assert_equal File.read(File.join(File.dirname(__FILE__), '..','ext','curb_easy.c')), File.read(dl_path) 61 | ensure 62 | File.unlink(dl_path) if dl_path && File.exist?(dl_path) 63 | end 64 | 65 | def test_download_bad_url_gives_404 66 | dl_url = "http://127.0.0.1:9129/this_file_does_not_exist.html" 67 | dl_path = File.join(Dir::tmpdir, "dl_url_test.file") 68 | 69 | curb = Curl::Easy.download(dl_url, dl_path) 70 | assert_equal Curl::Easy, curb.class 71 | assert_equal 404, curb.response_code 72 | ensure 73 | File.unlink(dl_path) if File.exist?(dl_path) 74 | end 75 | 76 | end 77 | -------------------------------------------------------------------------------- /tests/bug_follow_redirect_288.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.join(File.dirname(__FILE__), 'helper')) 2 | 3 | class BugFollowRedirect288 < Test::Unit::TestCase 4 | include BugTestServerSetupTeardown 5 | 6 | def setup 7 | @port = 9999 8 | super 9 | @server.mount_proc("/redirect_to_test") do|req,res| 10 | res.set_redirect(WEBrick::HTTPStatus::TemporaryRedirect, "/test") 11 | end 12 | end 13 | 14 | def test_follow_redirect_with_no_redirect 15 | 16 | c = Curl::Easy.new('http://127.0.0.1:9999/test') 17 | did_call_redirect = false 18 | c.on_redirect do|x| 19 | did_call_redirect = true 20 | end 21 | c.perform 22 | 23 | assert !did_call_redirect, "should reach this point redirect should not have been called" 24 | 25 | c = Curl::Easy.new('http://127.0.0.1:9999/test') 26 | did_call_redirect = false 27 | c.on_redirect do|x| 28 | did_call_redirect = true 29 | end 30 | c.follow_location = true 31 | c.perform 32 | 33 | assert_equal 0, c.redirect_count 34 | assert !did_call_redirect, "should reach this point redirect should not have been called" 35 | 36 | c = Curl::Easy.new('http://127.0.0.1:9999/redirect_to_test') 37 | did_call_redirect = false 38 | c.on_redirect do|x| 39 | did_call_redirect = true 40 | end 41 | c.perform 42 | assert_equal 307, c.response_code 43 | 44 | assert did_call_redirect, "we should have called on_redirect" 45 | 46 | c = Curl::Easy.new('http://127.0.0.1:9999/redirect_to_test') 47 | did_call_redirect = false 48 | c.follow_location = true 49 | # NOTE: while this API is not supported by libcurl e.g. there is no redirect function callback in libcurl we could 50 | # add support in ruby for this by executing this callback if redirect_count is greater than 0 at the end of a request in curb_multi.c 51 | c.on_redirect do|x| 52 | did_call_redirect = true 53 | end 54 | c.perform 55 | assert_equal 1, c.redirect_count 56 | assert_equal 200, c.response_code 57 | 58 | assert did_call_redirect, "we should have called on_redirect" 59 | 60 | c.url = 'http://127.0.0.1:9999/test' 61 | c.perform 62 | assert_equal 0, c.redirect_count 63 | assert_equal 200, c.response_code 64 | 65 | puts "checking for raise support" 66 | did_raise = false 67 | begin 68 | c = Curl::Easy.new('http://127.0.0.1:9999/redirect_to_test') 69 | did_call_redirect = false 70 | c.on_redirect do|x| 71 | raise "raise" 72 | did_call_redirect = true 73 | end 74 | c.perform 75 | rescue => e 76 | did_raise = true 77 | end 78 | assert_equal 307, c.response_code 79 | assert did_raise 80 | 81 | end 82 | 83 | end 84 | -------------------------------------------------------------------------------- /curb.gemspec: -------------------------------------------------------------------------------- 1 | Gem::Specification.new do |s| 2 | s.name = "curb" 3 | s.authors = ["Ross Bamford", "Todd A. Fisher"] 4 | s.version = '1.2.2' 5 | s.date = '2025-09-18' 6 | s.description = %q{Curb (probably CUrl-RuBy or something) provides Ruby-language bindings for the libcurl(3), a fully-featured client-side URL transfer library. cURL and libcurl live at http://curl.haxx.se/} 7 | s.email = 'todd.fisher@gmail.com' 8 | s.extra_rdoc_files = ['LICENSE', 'README.md'] 9 | 10 | s.files = ["LICENSE", "README.md", "Rakefile", "doc.rb", "ext/extconf.rb", "lib/curb.rb", "lib/curl/easy.rb", "lib/curl/multi.rb", "lib/curl.rb", "ext/curb.c", "ext/curb_easy.c", "ext/curb_errors.c", "ext/curb_multi.c", "ext/curb_postfield.c", "ext/curb_upload.c", "ext/banned.h", "ext/curb.h", "ext/curb_easy.h", "ext/curb_errors.h", "ext/curb_macros.h", "ext/curb_multi.h", "ext/curb_postfield.h", "ext/curb_upload.h"] 11 | 12 | #### Load-time details 13 | s.require_paths = ['lib','ext'] 14 | s.summary = %q{Ruby libcurl bindings} 15 | s.test_files = ["tests/alltests.rb", "tests/bug_crash_on_debug.rb", "tests/bug_crash_on_progress.rb", "tests/bug_curb_easy_blocks_ruby_threads.rb", "tests/bug_curb_easy_post_with_string_no_content_length_header.rb", "tests/bug_follow_redirect_288.rb", "tests/bug_instance_post_differs_from_class_post.rb", "tests/bug_issue102.rb", "tests/bug_issue_noproxy.rb", "tests/bug_issue_post_redirect.rb", "tests/bug_issue_spnego.rb", "tests/bug_multi_segfault.rb", "tests/bug_postfields_crash.rb", "tests/bug_postfields_crash2.rb", "tests/bug_raise_on_callback.rb", "tests/bug_require_last_or_segfault.rb", "tests/bugtests.rb", "tests/helper.rb", "tests/mem_check.rb", "tests/require_last_or_segfault_script.rb", "tests/signals.rb", "tests/tc_curl.rb", "tests/tc_curl_download.rb", "tests/tc_curl_easy.rb", "tests/tc_curl_easy_cookielist.rb", "tests/tc_curl_easy_request_target.rb", "tests/tc_curl_easy_resolve.rb", "tests/tc_curl_easy_setopt.rb", "tests/tc_curl_maxfilesize.rb", "tests/tc_curl_multi.rb", "tests/tc_curl_postfield.rb", "tests/tc_curl_protocols.rb", "tests/tc_fiber_scheduler.rb", "tests/tc_ftp_options.rb", "tests/tc_gc_compact.rb", "tests/test_basic.rb", "tests/test_fiber_debug.rb", "tests/test_fiber_simple.rb", "tests/test_real_url.rb", "tests/test_simple_fiber.rb", "tests/timeout.rb", "tests/timeout_server.rb", "tests/unittests.rb"] 16 | 17 | s.extensions << 'ext/extconf.rb' 18 | 19 | 20 | #### Documentation and testing. 21 | s.homepage = 'https://github.com/taf2/curb' 22 | s.rdoc_options = ['--main', 'README.md'] 23 | 24 | s.metadata = { 25 | 'changelog_uri' => 'https://github.com/taf2/curb/blob/master/ChangeLog.md' 26 | } 27 | 28 | 29 | s.platform = Gem::Platform::RUBY 30 | 31 | s.licenses = ['Ruby'] 32 | end 33 | -------------------------------------------------------------------------------- /ext/curb_upload.c: -------------------------------------------------------------------------------- 1 | /* curb_upload.c - Curl upload handle 2 | * Copyright (c)2009 Todd A Fisher. 3 | * Licensed under the Ruby License. See LICENSE for details. 4 | */ 5 | #include "curb_upload.h" 6 | extern VALUE mCurl; 7 | VALUE cCurlUpload; 8 | 9 | #ifdef RDOC_NEVER_DEFINED 10 | mCurl = rb_define_module("Curl"); 11 | #endif 12 | 13 | static void curl_upload_mark(ruby_curl_upload *rbcu) { 14 | if (rbcu->stream && !NIL_P(rbcu->stream)) rb_gc_mark(rbcu->stream); 15 | } 16 | static void curl_upload_free(ruby_curl_upload *rbcu) { 17 | free(rbcu); 18 | } 19 | 20 | /* 21 | * call-seq: 22 | * internal class for sending large file uploads 23 | */ 24 | VALUE ruby_curl_upload_new(VALUE klass) { 25 | VALUE upload; 26 | ruby_curl_upload *rbcu = ALLOC(ruby_curl_upload); 27 | if (!rbcu) { 28 | rb_raise(rb_eNoMemError, "Failed to allocate memory for Curl::Upload"); 29 | } 30 | rbcu->stream = Qnil; 31 | rbcu->offset = 0; 32 | upload = Data_Wrap_Struct(klass, curl_upload_mark, curl_upload_free, rbcu); 33 | return upload; 34 | } 35 | 36 | /* 37 | * call-seq: 38 | * internal class for sending large file uploads 39 | */ 40 | VALUE ruby_curl_upload_stream_set(VALUE self, VALUE stream) { 41 | ruby_curl_upload *rbcu; 42 | Data_Get_Struct(self, ruby_curl_upload, rbcu); 43 | rbcu->stream = stream; 44 | return stream; 45 | } 46 | /* 47 | * call-seq: 48 | * internal class for sending large file uploads 49 | */ 50 | VALUE ruby_curl_upload_stream_get(VALUE self) { 51 | ruby_curl_upload *rbcu; 52 | Data_Get_Struct(self, ruby_curl_upload, rbcu); 53 | return rbcu->stream; 54 | } 55 | /* 56 | * call-seq: 57 | * internal class for sending large file uploads 58 | */ 59 | VALUE ruby_curl_upload_offset_set(VALUE self, VALUE offset) { 60 | ruby_curl_upload *rbcu; 61 | Data_Get_Struct(self, ruby_curl_upload, rbcu); 62 | rbcu->offset = NUM2LONG(offset); 63 | return offset; 64 | } 65 | /* 66 | * call-seq: 67 | * internal class for sending large file uploads 68 | */ 69 | VALUE ruby_curl_upload_offset_get(VALUE self) { 70 | ruby_curl_upload *rbcu; 71 | Data_Get_Struct(self, ruby_curl_upload, rbcu); 72 | return LONG2NUM(rbcu->offset); 73 | } 74 | 75 | /* =================== INIT LIB =====================*/ 76 | void init_curb_upload() { 77 | cCurlUpload = rb_define_class_under(mCurl, "Upload", rb_cObject); 78 | rb_undef_alloc_func(cCurlUpload); 79 | rb_define_singleton_method(cCurlUpload, "new", ruby_curl_upload_new, 0); 80 | rb_define_method(cCurlUpload, "stream=", ruby_curl_upload_stream_set, 1); 81 | rb_define_method(cCurlUpload, "stream", ruby_curl_upload_stream_get, 0); 82 | rb_define_method(cCurlUpload, "offset=", ruby_curl_upload_offset_set, 1); 83 | rb_define_method(cCurlUpload, "offset", ruby_curl_upload_offset_get, 0); 84 | } 85 | -------------------------------------------------------------------------------- /ext/curb_easy.h: -------------------------------------------------------------------------------- 1 | /* curb_easy.h - Curl easy mode 2 | * Copyright (c)2006 Ross Bamford. 3 | * Licensed under the Ruby License. See LICENSE for details. 4 | * 5 | * $Id: curb_easy.h 25 2006-12-07 23:38:25Z roscopeco $ 6 | */ 7 | #ifndef __CURB_EASY_H 8 | #define __CURB_EASY_H 9 | 10 | #include "curb.h" 11 | 12 | #include 13 | 14 | #ifdef CURL_VERSION_SSL 15 | #if LIBCURL_VERSION_NUM >= 0x070b00 16 | # if LIBCURL_VERSION_NUM <= 0x071004 17 | # define CURB_FTPSSL CURLOPT_FTP_SSL 18 | # define CURB_FTPSSL_ALL CURLFTPSSL_ALL 19 | # define CURB_FTPSSL_TRY CURLFTPSSL_TRY 20 | # define CURB_FTPSSL_CONTROL CURLFTPSSL_CONTROL 21 | # define CURB_FTPSSL_NONE CURLFTPSSL_NONE 22 | # else 23 | # define CURB_FTPSSL CURLOPT_USE_SSL 24 | # define CURB_FTPSSL_ALL CURLUSESSL_ALL 25 | # define CURB_FTPSSL_TRY CURLUSESSL_TRY 26 | # define CURB_FTPSSL_CONTROL CURLUSESSL_CONTROL 27 | # define CURB_FTPSSL_NONE CURLUSESSL_NONE 28 | # endif 29 | #endif 30 | #endif 31 | 32 | /* a lot of this *could* be kept in the handler itself, 33 | * but then we lose the ability to query it's status. 34 | */ 35 | typedef struct { 36 | /* The handler */ 37 | CURL *curl; 38 | 39 | /* Buffer for error details from CURLOPT_ERRORBUFFER */ 40 | char err_buf[CURL_ERROR_SIZE]; 41 | 42 | VALUE self; /* owning Ruby object */ 43 | VALUE opts; /* rather then allocate everything we might need to store, allocate a Hash and only store objects we actually use... */ 44 | VALUE multi; /* keep a multi handle alive for each easy handle not being used by a multi handle. This improves easy performance when not within a multi context */ 45 | 46 | /* Other opts */ 47 | unsigned short local_port; // 0 is no port 48 | unsigned short local_port_range; // " " " " 49 | unsigned short proxy_port; // " " " " 50 | int proxy_type; 51 | long http_auth_types; 52 | long proxy_auth_types; 53 | long max_redirs; 54 | unsigned long timeout; 55 | unsigned long timeout_ms; 56 | unsigned long connect_timeout; 57 | unsigned long connect_timeout_ms; 58 | long dns_cache_timeout; 59 | unsigned long ftp_response_timeout; 60 | long low_speed_limit; 61 | long low_speed_time; 62 | long max_send_speed_large; 63 | long max_recv_speed_large; 64 | long ssl_version; 65 | long use_ssl; 66 | long ftp_filemethod; 67 | unsigned short resolve_mode; 68 | 69 | /* bool flags */ 70 | char proxy_tunnel; 71 | char fetch_file_time; 72 | char ssl_verify_peer; 73 | char ssl_verify_host; 74 | char header_in_body; 75 | char use_netrc; 76 | char follow_location; 77 | char unrestricted_auth; 78 | char verbose; 79 | char multipart_form_post; 80 | char enable_cookies; 81 | char cookielist_engine_enabled; /* track if CURLOPT_COOKIELIST was used with a non-command to enable engine */ 82 | char ignore_content_length; 83 | char callback_active; 84 | 85 | struct curl_slist *curl_headers; 86 | struct curl_slist *curl_proxy_headers; 87 | struct curl_slist *curl_ftp_commands; 88 | struct curl_slist *curl_resolve; 89 | 90 | int last_result; /* last result code from multi loop */ 91 | 92 | } ruby_curl_easy; 93 | 94 | extern VALUE cCurlEasy; 95 | 96 | VALUE ruby_curl_easy_setup(ruby_curl_easy *rbce); 97 | VALUE ruby_curl_easy_cleanup(VALUE self, ruby_curl_easy *rbce); 98 | 99 | void init_curb_easy(); 100 | 101 | #endif 102 | -------------------------------------------------------------------------------- /tests/bug_issue_post_redirect.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.join(File.dirname(__FILE__), 'helper')) 2 | require 'json' 3 | 4 | class BugIssuePostRedirect < Test::Unit::TestCase 5 | include BugTestServerSetupTeardown 6 | 7 | def setup 8 | @port = 9998 9 | super 10 | # Mount a POST endpoint that returns a redirect 11 | @server.mount_proc("/post_redirect") do |req, res| 12 | if req.request_method == "POST" 13 | res.status = 302 14 | res['Location'] = "http://127.0.0.1:#{@port}/redirected" 15 | res.body = "Redirecting..." 16 | else 17 | res.status = 405 18 | res.body = "Method not allowed" 19 | end 20 | end 21 | 22 | # Mount the redirect target 23 | @server.mount_proc("/redirected") do |req, res| 24 | res.status = 200 25 | res['Content-Type'] = "text/plain" 26 | res.body = "You have been redirected" 27 | end 28 | end 29 | 30 | def test_post_with_max_redirects_zero_should_not_follow_redirect 31 | # Test case replicating the issue: POST with max_redirects=0 and follow_location=false 32 | # should NOT trigger on_redirect callback or follow the redirect 33 | 34 | redirect_called = false 35 | 36 | handle = Curl::Easy.new("http://127.0.0.1:#{@port}/post_redirect") do |curl| 37 | curl.max_redirects = 0 38 | curl.follow_location = false 39 | curl.on_redirect do |easy| 40 | redirect_called = true 41 | end 42 | curl.headers['Content-Type'] = 'application/json' 43 | curl.post_body = {test: "data"}.to_json 44 | end 45 | 46 | handle.http(:POST) 47 | 48 | # The response should be the redirect response (302) 49 | assert_equal 302, handle.response_code 50 | assert_match(/Redirecting/, handle.body_str) 51 | 52 | # on_redirect should NOT be called when follow_location is false 53 | assert !redirect_called, "on_redirect callback should not be called when follow_location is false" 54 | end 55 | 56 | def test_post_with_follow_location_true_triggers_redirect 57 | # Test that on_redirect IS called when follow_location is true 58 | redirect_called = false 59 | 60 | handle = Curl::Easy.new("http://127.0.0.1:#{@port}/post_redirect") do |curl| 61 | curl.follow_location = true 62 | curl.on_redirect do |easy| 63 | redirect_called = true 64 | end 65 | curl.headers['Content-Type'] = 'application/json' 66 | curl.post_body = {test: "data"}.to_json 67 | end 68 | 69 | handle.http(:POST) 70 | 71 | # Should follow the redirect and get the final response 72 | assert_equal 200, handle.response_code 73 | assert_match(/You have been redirected/, handle.body_str) 74 | 75 | # on_redirect SHOULD be called when follow_location is true 76 | assert redirect_called, "on_redirect callback should be called when follow_location is true" 77 | end 78 | 79 | def test_curl_post_class_method_respects_redirect_settings 80 | # Test that Curl.post (class method) respects redirect settings 81 | # According to the issue, this works correctly 82 | 83 | response = Curl.post("http://127.0.0.1:#{@port}/post_redirect", {test: "data"}.to_json) do |curl| 84 | curl.max_redirects = 0 85 | curl.follow_location = false 86 | curl.headers['Content-Type'] = 'application/json' 87 | end 88 | 89 | # Should get the redirect response, not follow it 90 | assert_equal 302, response.response_code 91 | assert_match(/Redirecting/, response.body_str) 92 | end 93 | end -------------------------------------------------------------------------------- /samples/delayed_wait.rb: -------------------------------------------------------------------------------- 1 | # DelayedEasy hits several domains and then interacts with the 2 | # responses. Each interaction will trigger a check to see if that 3 | # response has already come back and been cached. If it has not, 4 | # the interaction will be forced to wait for Curl::Easy#perform to 5 | # come back with the data. In this way, you could fire off several 6 | # requests (ala in a rails controller), and not need to wait until 7 | # the response data is needed (at the last possible second, like 8 | # a view). Using Curl::Easy means the requests are all truely 9 | # asynchronous, and time spent waiting for one request to finish 10 | # is not wasted, but could be used pulling down another request 11 | # in parallel. 12 | # 13 | 14 | # Use local curb 15 | # 16 | 17 | $:.unshift(File.expand_path(File.join(File.dirname(__FILE__), '..', 'ext'))) 18 | $:.unshift(File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib'))) 19 | require 'curb' 20 | 21 | # DelayedEasy class 22 | # 23 | # Fires Curb requests right away, then caches the result. If the 24 | # object has any access before the cache is populated, it waits 25 | # on the request to complete. 26 | # 27 | 28 | module Curl 29 | class DelayedEasy 30 | attr_accessor :response_thread, :delayed_response 31 | 32 | def initialize(*args, &blk) 33 | curl = nil 34 | self.response_thread = Thread.new { 35 | curl = Curl::Easy.new(*args, &blk) 36 | curl.perform 37 | @delayed_response = curl 38 | } 39 | self 40 | end 41 | 42 | def delayed_response 43 | @delayed_response or 44 | ( puts 'waiting...'; self.response_thread.join; self.delayed_response ) 45 | end 46 | 47 | def method_missing(method_name,*args,&blk) 48 | self.delayed_response.send(method_name, *args, &blk) 49 | end 50 | 51 | end 52 | end 53 | 54 | # An example for DelayedEasy 55 | # 56 | # Some URLs to smack in our demo. 57 | # 58 | 59 | urls = ["http://boingboing.net", 60 | "http://www.yahoo.com", 61 | "http://slashdot.org", 62 | "http://www.google.com", 63 | "http://www.yelp.com", 64 | "http://www.givereal.com", 65 | "http://www.google.co.uk/", 66 | "http://www.ruby-lang.org/"] 67 | 68 | # Fire a new DelayedEasy for each url. 69 | # 70 | 71 | requests = {} 72 | urls.each do |url| 73 | requests[url] = Curl::DelayedEasy.new(url) 74 | end 75 | 76 | # Now puts the response data. The output will show 77 | # that some interactions need to wait for the response, 78 | # while others proceed with no 'waiting' puts. 79 | # 80 | 81 | urls.each do |url| 82 | puts "[#{url} #{requests[url].total_time}] #{requests[url].body_str[0,30].gsub("\n", '')}..." 83 | puts 84 | end 85 | 86 | # Sample output: 87 | # 88 | # mixonic@pandora ~/Projects/curb $ time ruby samples/delayed_wait.rb 89 | # waiting... 90 | # [http://boingboing.net 1.36938] Yahoo!</t... 93 | # 94 | # [http://slashdot.org 1.007763] <!DOCTYPE HTML PUBLIC "-//W3C/... 95 | # 96 | # [http://www.google.com 0.796156] <!doctype html><html><head><me... 97 | # 98 | # waiting... 99 | # [http://www.yelp.com 1.365811] <!DOCTYPE HTML PUBLIC "-//W3C/... 100 | # 101 | # [http://www.givereal.com 0.803108] <!DOCTYPE html PUBLIC "-//W3C/... 102 | # 103 | # [http://www.google.co.uk/ 0.601572] <!doctype html><html><head><me... 104 | # 105 | # waiting... 106 | # [http://www.ruby-lang.org/ 0.628023] <html><body>302 Found</body></... 107 | # 108 | # 109 | # real 0m1.658s 110 | # user 0m0.043s 111 | # sys 0m0.030s 112 | # 113 | -------------------------------------------------------------------------------- /.github/workflows/CI.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | 9 | jobs: 10 | build: 11 | strategy: 12 | fail-fast: false 13 | matrix: 14 | os: [ubuntu-latest] 15 | ruby: 16 | - '2.6' 17 | - '2.7' 18 | - '3.0' 19 | - '3.1' 20 | - '3.2' 21 | - '3.3' 22 | - '3.4' 23 | include: 24 | - os: macos-latest 25 | ruby: '3.4' 26 | - os: windows-latest 27 | ruby: '3.4' 28 | runs-on: ${{ matrix.os }} 29 | name: ${{ matrix.os }} Ruby ${{ matrix.ruby }} 30 | continue-on-error: ${{ matrix.ruby == 'head' }} 31 | steps: 32 | - uses: actions/checkout@v4 33 | 34 | # Linux deps 35 | - name: Install libcurl (Linux) 36 | if: runner.os == 'Linux' 37 | run: | 38 | sudo apt-get update 39 | sudo apt-get install -y libcurl4 libcurl3-gnutls libcurl4-openssl-dev 40 | 41 | # macOS deps 42 | - name: Install libcurl (macOS) 43 | if: runner.os == 'macOS' 44 | run: | 45 | brew update 46 | brew install curl 47 | echo "$(brew --prefix curl)/bin" >> $GITHUB_PATH 48 | 49 | # Windows deps via MSYS2 (RubyInstaller DevKit) 50 | - name: Install libcurl (Windows) 51 | if: runner.os == 'Windows' 52 | run: | 53 | ridk exec pacman -S --noconfirm --needed mingw-w64-x86_64-curl 54 | 55 | - uses: ruby/setup-ruby@v1 56 | with: 57 | ruby-version: ${{ matrix.ruby }} 58 | bundler-cache: true 59 | 60 | - name: Run tests 61 | run: bundle exec rake 62 | 63 | coverage: 64 | runs-on: ubuntu-latest 65 | needs: build 66 | if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository 67 | 68 | steps: 69 | - uses: actions/checkout@v4 70 | 71 | - name: Install system dependencies 72 | run: | 73 | sudo apt-get update 74 | sudo apt-get install -y libcurl4 libcurl3-gnutls libcurl4-openssl-dev lcov 75 | 76 | - uses: ruby/setup-ruby@v1 77 | with: 78 | ruby-version: '3.2' 79 | bundler-cache: true 80 | 81 | - name: Run tests with full coverage 82 | run: | 83 | # Clean any previous coverage 84 | bundle exec rake coverage_clean 85 | 86 | # Compile with coverage flags 87 | bundle exec rake compile_coverage 88 | 89 | # Run tests with Ruby coverage 90 | COVERAGE=1 bundle exec rake test 91 | 92 | # Generate C coverage report 93 | bundle exec rake coverage_report 94 | 95 | - name: Upload Ruby coverage to Codecov 96 | uses: codecov/codecov-action@v4 97 | with: 98 | files: ./coverage/lcov/curb.lcov 99 | flags: ruby 100 | name: ruby-coverage 101 | fail_ci_if_error: false 102 | token: ${{ secrets.CODECOV_TOKEN }} 103 | 104 | - name: Upload C coverage to Codecov 105 | uses: codecov/codecov-action@v4 106 | with: 107 | files: ./coverage_c/coverage_filtered.info 108 | flags: c 109 | name: c-coverage 110 | fail_ci_if_error: false 111 | token: ${{ secrets.CODECOV_TOKEN }} 112 | 113 | - name: Archive coverage reports 114 | uses: actions/upload-artifact@v4 115 | if: always() 116 | with: 117 | name: coverage-reports 118 | path: | 119 | coverage/ 120 | coverage_c/ 121 | retention-days: 30 122 | -------------------------------------------------------------------------------- /tests/timeout.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.join(File.dirname(__FILE__), 'helper')) 2 | 3 | # Run server with: ruby -rubygems timeout_server.rb -p 9128 4 | 5 | # Note that curl requires all timeouts to be integers - 6 | # curl_easy_setopt does not have a provision for floating-point values 7 | 8 | class TestCurbTimeouts < Test::Unit::TestCase 9 | def test_no_timeout_by_default 10 | curl = Curl::Easy.new(wait_url(2)) 11 | start = Time.now 12 | assert_equal true, curl.http_get 13 | elapsed = Time.now - start 14 | assert elapsed > 2 15 | end 16 | 17 | def test_overall_timeout_on_dead_transfer 18 | curl = Curl::Easy.new(wait_url(2)) 19 | curl.timeout = 1 20 | exception = assert_raise(Curl::Err::TimeoutError) do 21 | curl.http_get 22 | end 23 | assert_match( 24 | /^Timeout was reached: Operation timed out after/, 25 | exception.message 26 | ) 27 | end 28 | 29 | def test_overall_timeout_ms_on_dead_transfer 30 | curl = Curl::Easy.new(wait_url(2)) 31 | curl.timeout_ms = 1000 32 | assert_raise(Curl::Err::TimeoutError) do 33 | curl.http_get 34 | end 35 | end 36 | 37 | def test_clearing_timeout 38 | curl = Curl::Easy.new(wait_url(2)) 39 | curl.timeout = 1 40 | curl.timeout = nil 41 | start = Time.now 42 | assert_equal true, curl.http_get 43 | elapsed = Time.now - start 44 | assert elapsed > 2 45 | end 46 | 47 | def test_overall_timeout_on_slow_transfer 48 | curl = Curl::Easy.new(serve_url(100, 2, 3)) 49 | curl.timeout = 1 50 | # transfer is aborted despite data being exchanged 51 | exception = assert_raise(Curl::Err::TimeoutError) do 52 | curl.http_get 53 | end 54 | assert_match( 55 | /^Timeout was reached: Operation timed out after/, 56 | exception.message 57 | ) 58 | end 59 | 60 | def test_low_speed_time_on_slow_transfer 61 | curl = Curl::Easy.new(serve_url(100, 1, 3)) 62 | curl.low_speed_time = 2 63 | # use default low_speed_limit of 1 64 | assert_equal true, curl.http_get 65 | end 66 | 67 | def test_low_speed_time_on_very_slow_transfer 68 | # send data slower than required 69 | curl = Curl::Easy.new(serve_url(10, 2, 3)) 70 | curl.low_speed_time = 1 71 | # XXX for some reason this test fails if low speed limit is not specified 72 | curl.low_speed_limit = 1 73 | # use default low_speed_limit of 1 74 | exception = assert_raise(Curl::Err::TimeoutError) do 75 | curl.http_get 76 | end 77 | assert_match( 78 | /^Timeout was reached: Operation too slow/, 79 | exception.message 80 | ) 81 | end 82 | 83 | def test_low_speed_limit_on_slow_transfer 84 | curl = Curl::Easy.new(serve_url(10, 1, 3)) 85 | curl.low_speed_time = 2 86 | curl.low_speed_limit = 1000 87 | exception = assert_raise(Curl::Err::TimeoutError) do 88 | curl.http_get 89 | end 90 | assert_match( 91 | /^Timeout was reached: Operation too slow/, 92 | exception.message 93 | ) 94 | end 95 | 96 | def test_clearing_low_speed_time 97 | curl = Curl::Easy.new(serve_url(100, 2, 3)) 98 | curl.low_speed_time = 1 99 | curl.low_speed_time = nil 100 | assert_equal true, curl.http_get 101 | end 102 | 103 | def test_clearing_low_speed_limit 104 | curl = Curl::Easy.new(serve_url(10, 1, 3)) 105 | curl.low_speed_time = 2 106 | curl.low_speed_limit = 1000 107 | curl.low_speed_limit = nil 108 | assert_equal true, curl.http_get 109 | end 110 | 111 | private 112 | 113 | def wait_url(time) 114 | "#{server_base}/wait/#{time}" 115 | end 116 | 117 | def serve_url(chunk_size, time, count) 118 | "#{server_base}/serve/#{chunk_size}/every/#{time}/for/#{count}" 119 | end 120 | 121 | def server_base 122 | 'http://127.0.0.1:9128' 123 | end 124 | end 125 | -------------------------------------------------------------------------------- /tasks/coverage.rake: -------------------------------------------------------------------------------- 1 | # Coverage tasks for both Ruby and C code 2 | 3 | desc "Run tests with full coverage (Ruby + C)" 4 | task :coverage => [:coverage_clean, :compile_coverage, :test_coverage, :coverage_report] 5 | 6 | desc "Clean coverage files" 7 | task :coverage_clean do 8 | # Clean C coverage files 9 | sh "rm -f ext/*.gcda ext/*.gcno ext/*.gcov" 10 | sh "rm -rf coverage/" 11 | sh "rm -rf coverage_c/" 12 | end 13 | 14 | desc "Compile extension with coverage flags" 15 | task :compile_coverage => :coverage_clean do 16 | # Clean previous build 17 | sh "cd ext && make clean" rescue nil 18 | sh "rm -f ext/Makefile" 19 | 20 | # Get existing flags 21 | existing_cflags = ENV['CFLAGS'] || "" 22 | existing_ldflags = ENV['LDFLAGS'] || "" 23 | 24 | # Set coverage flags for compilation - prepend to existing flags 25 | ENV['CFLAGS'] = "-fprofile-arcs -ftest-coverage -g -O0 #{existing_cflags}" 26 | ENV['LDFLAGS'] = "-fprofile-arcs -ftest-coverage #{existing_ldflags}" 27 | 28 | # Regenerate Makefile with coverage flags using special extconf 29 | Dir.chdir("ext") do 30 | sh "ruby extconf_coverage.rb" 31 | sh "make" 32 | end 33 | 34 | # Verify .gcno files were created 35 | gcno_count = Dir.glob("ext/*.gcno").size 36 | if gcno_count == 0 37 | puts "WARNING: No .gcno files generated. C coverage may not work." 38 | else 39 | puts "SUCCESS: Generated #{gcno_count} .gcno files for coverage." 40 | end 41 | end 42 | 43 | desc "Run tests with coverage enabled" 44 | task :test_coverage do 45 | ENV['COVERAGE'] = '1' 46 | Rake::Task['test'].invoke 47 | end 48 | 49 | desc "Generate C coverage report" 50 | task :coverage_report do 51 | puts "\n=== Generating C Coverage Report ===" 52 | 53 | # Create coverage directory 54 | sh "mkdir -p coverage_c" 55 | 56 | # Generate coverage info 57 | sh "cd ext && gcov *.c" 58 | 59 | # Check if lcov is available 60 | has_lcov = system("which lcov > /dev/null 2>&1") 61 | 62 | if has_lcov 63 | # Generate detailed HTML report with lcov 64 | sh "lcov --capture --directory ext --output-file coverage_c/coverage.info" 65 | 66 | # Check what patterns exist in the coverage data 67 | coverage_data = File.read("coverage_c/coverage.info") rescue "" 68 | 69 | # Build exclude patterns based on what's actually in the data 70 | exclude_patterns = [] 71 | exclude_patterns << "'/usr/*'" if coverage_data.include?("/usr/") 72 | exclude_patterns << "'*/ruby/*'" if coverage_data.match?(%r{/ruby/}) 73 | exclude_patterns << "'/Applications/*'" if coverage_data.include?("/Applications/") 74 | exclude_patterns << "'/opt/homebrew/*'" if coverage_data.include?("/opt/homebrew/") 75 | 76 | if exclude_patterns.any? 77 | sh "lcov --remove coverage_c/coverage.info #{exclude_patterns.join(' ')} --output-file coverage_c/coverage_filtered.info" 78 | else 79 | # No patterns to exclude, just copy 80 | sh "cp coverage_c/coverage.info coverage_c/coverage_filtered.info" 81 | end 82 | 83 | sh "genhtml coverage_c/coverage_filtered.info --output-directory coverage_c/html" 84 | 85 | puts "\n=== Coverage Reports Generated ===" 86 | puts "Ruby coverage: coverage/index.html" 87 | puts "C coverage: coverage_c/html/index.html" 88 | else 89 | # Basic text report 90 | puts "\n=== C Coverage Summary ===" 91 | Dir.glob("ext/*.gcov").each do |file| 92 | puts "\n#{File.basename(file)}:" 93 | coverage_data = File.read(file) 94 | total_lines = 0 95 | covered_lines = 0 96 | 97 | coverage_data.each_line do |line| 98 | if line =~ /^\s*(-|\d+):\s*\d+:/ 99 | total_lines += 1 100 | covered_lines += 1 if line =~ /^\s*\d+:/ 101 | end 102 | end 103 | 104 | percentage = (covered_lines.to_f / total_lines * 100).round(2) 105 | puts " Coverage: #{percentage}% (#{covered_lines}/#{total_lines} lines)" 106 | end 107 | 108 | puts "\nInstall lcov for detailed HTML reports: brew install lcov (macOS) or apt-get install lcov (Linux)" 109 | end 110 | end 111 | 112 | desc "Run tests with memory checking and coverage" 113 | task :test_memcheck_coverage => [:compile_coverage] do 114 | ENV['COVERAGE'] = '1' 115 | Rake::Task['test:valgrind'].invoke 116 | Rake::Task['coverage_report'].invoke 117 | end -------------------------------------------------------------------------------- /tests/tc_curl_postfield.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.join(File.dirname(__FILE__), 'helper')) 2 | 3 | class TestCurbCurlPostfield < Test::Unit::TestCase 4 | def test_private_new 5 | assert_raise(NoMethodError) { Curl::PostField.new } 6 | end 7 | 8 | def test_new_content_01 9 | pf = Curl::PostField.content('foo', 'bar') 10 | 11 | assert_equal 'foo', pf.name 12 | assert_equal 'bar', pf.content 13 | assert_nil pf.content_type 14 | assert_nil pf.local_file 15 | assert_nil pf.remote_file 16 | assert_nil pf.set_content_proc 17 | end 18 | 19 | def test_new_content_02 20 | pf = Curl::PostField.content('foo', 'bar', 'text/html') 21 | 22 | assert_equal 'foo', pf.name 23 | assert_equal 'bar', pf.content 24 | assert_equal 'text/html', pf.content_type 25 | assert_nil pf.local_file 26 | assert_nil pf.remote_file 27 | assert_nil pf.set_content_proc 28 | end 29 | 30 | def test_new_content_03 31 | l = lambda { |field| "never gets run" } 32 | pf = Curl::PostField.content('foo', &l) 33 | 34 | assert_equal 'foo', pf.name 35 | assert_nil pf.content 36 | assert_nil pf.content_type 37 | assert_nil pf.local_file 38 | assert_nil pf.remote_file 39 | 40 | # N.B. This doesn't just get the proc, but also removes it. 41 | assert_equal l, pf.set_content_proc 42 | end 43 | 44 | def test_new_content_04 45 | l = lambda { |field| "never gets run" } 46 | pf = Curl::PostField.content('foo', 'text/html', &l) 47 | 48 | assert_equal 'foo', pf.name 49 | assert_nil pf.content 50 | assert_equal 'text/html', pf.content_type 51 | assert_nil pf.local_file 52 | assert_nil pf.remote_file 53 | 54 | # N.B. This doesn't just get the proc, but also removes it. 55 | assert_equal l, pf.set_content_proc 56 | end 57 | 58 | 59 | def test_new_file_01 60 | pf = Curl::PostField.file('foo', 'localname') 61 | pf.content_type = 'text/super' 62 | 63 | assert_equal 'foo', pf.name 64 | assert_equal 'localname', pf.local_file 65 | assert_equal 'localname', pf.remote_file 66 | assert_nothing_raised { pf.to_s } 67 | assert_equal 'text/super', pf.content_type 68 | assert_nil pf.content 69 | assert_nil pf.set_content_proc 70 | end 71 | 72 | def test_new_file_02 73 | pf = Curl::PostField.file('foo', 'localname', 'remotename') 74 | 75 | assert_equal 'foo', pf.name 76 | assert_equal 'localname', pf.local_file 77 | assert_equal 'remotename', pf.remote_file 78 | assert_nil pf.content_type 79 | assert_nil pf.content 80 | assert_nil pf.set_content_proc 81 | end 82 | 83 | def test_new_file_03 84 | l = lambda { |field| "never gets run" } 85 | pf = Curl::PostField.file('foo', 'remotename', &l) 86 | 87 | assert_equal 'foo', pf.name 88 | assert_equal 'remotename', pf.remote_file 89 | assert_nil pf.local_file 90 | assert_nil pf.content_type 91 | assert_nil pf.content 92 | 93 | # N.B. This doesn't just get the proc, but also removes it. 94 | assert_equal l, pf.set_content_proc 95 | end 96 | 97 | def test_new_file_04 98 | assert_raise(ArgumentError) do 99 | # no local name, no block 100 | Curl::PostField.file('foo') 101 | end 102 | 103 | assert_raise(ArgumentError) do 104 | # no remote name with block 105 | Curl::PostField.file('foo') { |field| "never runs" } 106 | end 107 | end 108 | 109 | def test_new_file_05 110 | # local gets ignored when supplying a block, but remote 111 | # is still set up properly. 112 | l = lambda { |field| "never runs" } 113 | pf = Curl::PostField.file('foo', 'local', 'remote', &l) 114 | 115 | assert_equal 'foo', pf.name 116 | assert_equal 'remote', pf.remote_file 117 | assert_nil pf.local_file 118 | assert_nil pf.content_type 119 | assert_nil pf.content 120 | 121 | assert_equal l, pf.set_content_proc 122 | end 123 | 124 | def test_to_s_01 125 | pf = Curl::PostField.content('foo', 'bar') 126 | assert_equal "foo=bar", pf.to_s 127 | end 128 | 129 | def test_to_s_02 130 | pf = Curl::PostField.content('foo', 'bar ton') 131 | assert_equal "foo=bar%20ton", pf.to_s 132 | end 133 | 134 | def test_to_s_03 135 | pf = Curl::PostField.content('foo') { |field| field.name.upcase + "BAR" } 136 | assert_equal "foo=FOOBAR", pf.to_s 137 | end 138 | 139 | def test_to_s_04 140 | pf = Curl::PostField.file('foo.file', 'bar.file') 141 | assert_nothing_raised { pf.to_s } 142 | #assert_raise(Curl::Err::InvalidPostFieldError) { pf.to_s } 143 | end 144 | end 145 | -------------------------------------------------------------------------------- /ext/curb_errors.h: -------------------------------------------------------------------------------- 1 | /* curb_errors.h - Ruby exception types for curl errors 2 | * Copyright (c)2006 Ross Bamford. 3 | * Licensed under the Ruby License. See LICENSE for details. 4 | * 5 | * $Id: curb_errors.h 4 2006-11-17 18:35:31Z roscopeco $ 6 | */ 7 | #ifndef __CURB_ERRORS_H 8 | #define __CURB_ERRORS_H 9 | 10 | #include "curb.h" 11 | 12 | /* base errors */ 13 | extern VALUE cCurlErr; 14 | 15 | /* easy errors */ 16 | extern VALUE mCurlErr; 17 | extern VALUE eCurlErrError; 18 | extern VALUE eCurlErrFTPError; 19 | extern VALUE eCurlErrHTTPError; 20 | extern VALUE eCurlErrFileError; 21 | extern VALUE eCurlErrLDAPError; 22 | extern VALUE eCurlErrTelnetError; 23 | extern VALUE eCurlErrTFTPError; 24 | 25 | /* libcurl errors */ 26 | extern VALUE eCurlErrUnsupportedProtocol; 27 | extern VALUE eCurlErrFailedInit; 28 | extern VALUE eCurlErrMalformedURL; 29 | extern VALUE eCurlErrMalformedURLUser; 30 | extern VALUE eCurlErrProxyResolution; 31 | extern VALUE eCurlErrHostResolution; 32 | extern VALUE eCurlErrConnectFailed; 33 | extern VALUE eCurlErrFTPWeirdReply; 34 | extern VALUE eCurlErrFTPAccessDenied; 35 | extern VALUE eCurlErrFTPBadPassword; 36 | extern VALUE eCurlErrFTPWeirdPassReply; 37 | extern VALUE eCurlErrFTPWeirdUserReply; 38 | extern VALUE eCurlErrFTPWeirdPasvReply; 39 | extern VALUE eCurlErrFTPWeird227Format; 40 | extern VALUE eCurlErrFTPCantGetHost; 41 | extern VALUE eCurlErrFTPCantReconnect; 42 | extern VALUE eCurlErrFTPCouldntSetBinary; 43 | extern VALUE eCurlErrPartialFile; 44 | extern VALUE eCurlErrFTPCouldntRetrFile; 45 | extern VALUE eCurlErrFTPWrite; 46 | extern VALUE eCurlErrFTPQuote; 47 | extern VALUE eCurlErrHTTPFailed; 48 | extern VALUE eCurlErrWriteError; 49 | extern VALUE eCurlErrMalformedUser; 50 | extern VALUE eCurlErrFTPCouldntStorFile; 51 | extern VALUE eCurlErrReadError; 52 | extern VALUE eCurlErrOutOfMemory; 53 | extern VALUE eCurlErrTimeout; 54 | extern VALUE eCurlErrFTPCouldntSetASCII; 55 | extern VALUE eCurlErrFTPPortFailed; 56 | extern VALUE eCurlErrFTPCouldntUseRest; 57 | extern VALUE eCurlErrFTPCouldntGetSize; 58 | extern VALUE eCurlErrHTTPRange; 59 | extern VALUE eCurlErrHTTPPost; 60 | extern VALUE eCurlErrSSLConnectError; 61 | extern VALUE eCurlErrBadResume; 62 | extern VALUE eCurlErrFileCouldntRead; 63 | extern VALUE eCurlErrLDAPCouldntBind; 64 | extern VALUE eCurlErrLDAPSearchFailed; 65 | extern VALUE eCurlErrLibraryNotFound; 66 | extern VALUE eCurlErrFunctionNotFound; 67 | extern VALUE eCurlErrAbortedByCallback; 68 | extern VALUE eCurlErrBadFunctionArgument; 69 | extern VALUE eCurlErrBadCallingOrder; 70 | extern VALUE eCurlErrInterfaceFailed; 71 | extern VALUE eCurlErrBadPasswordEntered; 72 | extern VALUE eCurlErrTooManyRedirects; 73 | extern VALUE eCurlErrTelnetUnknownOption; 74 | extern VALUE eCurlErrTelnetBadOptionSyntax; 75 | extern VALUE eCurlErrObsolete; 76 | extern VALUE eCurlErrSSLPeerCertificate; 77 | extern VALUE eCurlErrGotNothing; 78 | extern VALUE eCurlErrSSLEngineNotFound; 79 | extern VALUE eCurlErrSSLEngineSetFailed; 80 | extern VALUE eCurlErrSendError; 81 | extern VALUE eCurlErrRecvError; 82 | extern VALUE eCurlErrShareInUse; 83 | extern VALUE eCurlErrSSLCertificate; 84 | extern VALUE eCurlErrSSLCipher; 85 | extern VALUE eCurlErrSSLCACertificate; 86 | extern VALUE eCurlErrBadContentEncoding; 87 | extern VALUE eCurlErrLDAPInvalidURL; 88 | extern VALUE eCurlErrFileSizeExceeded; 89 | extern VALUE eCurlErrFTPSSLFailed; 90 | extern VALUE eCurlErrSendFailedRewind; 91 | extern VALUE eCurlErrSSLEngineInitFailed; 92 | extern VALUE eCurlErrLoginDenied; 93 | extern VALUE eCurlErrTFTPNotFound; 94 | extern VALUE eCurlErrTFTPPermission; 95 | extern VALUE eCurlErrTFTPDiskFull; 96 | extern VALUE eCurlErrTFTPIllegalOperation; 97 | extern VALUE eCurlErrTFTPUnknownID; 98 | extern VALUE eCurlErrTFTPFileExists; 99 | extern VALUE eCurlErrTFTPNoSuchUser; 100 | extern VALUE eCurlErrConvFailed; 101 | extern VALUE eCurlErrConvReqd; 102 | extern VALUE eCurlErrSSLCacertBadfile; 103 | extern VALUE eCurlErrRemoteFileNotFound; 104 | extern VALUE eCurlErrSSH; 105 | extern VALUE eCurlErrSSLShutdownFailed; 106 | extern VALUE eCurlErrAgain; 107 | extern VALUE eCurlErrSSLCRLBadfile; 108 | extern VALUE eCurlErrSSLIssuerError; 109 | 110 | /* multi errors */ 111 | extern VALUE mCurlErrFailedInit; 112 | extern VALUE mCurlErrCallMultiPerform; 113 | extern VALUE mCurlErrBadHandle; 114 | extern VALUE mCurlErrBadEasyHandle; 115 | extern VALUE mCurlErrOutOfMemory; 116 | extern VALUE mCurlErrInternalError; 117 | extern VALUE mCurlErrBadSocket; 118 | extern VALUE mCurlErrUnknownOption; 119 | #if HAVE_CURLM_ADDED_ALREADY 120 | extern VALUE mCurlErrAddedAlready; 121 | #endif 122 | 123 | /* binding errors */ 124 | extern VALUE eCurlErrInvalidPostField; 125 | 126 | void init_curb_errors(); 127 | void raise_curl_easy_error_exception(CURLcode code); 128 | void raise_curl_multi_error_exception(CURLMcode code); 129 | VALUE rb_curl_easy_error(CURLcode code); 130 | VALUE rb_curl_multi_error(CURLMcode code); 131 | 132 | #endif 133 | -------------------------------------------------------------------------------- /tests/tc_fiber_scheduler.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.join(File.dirname(__FILE__), 'helper')) 2 | 3 | begin 4 | require 'async' 5 | rescue LoadError 6 | HAS_ASYNC = false 7 | else 8 | HAS_ASYNC = true 9 | end 10 | 11 | # This test verifies that Curl::Easy#perform cooperates with Ruby's Fiber scheduler 12 | # by running multiple requests concurrently in a single thread using the Async gem. 13 | class TestCurbFiberScheduler < Test::Unit::TestCase 14 | include BugTestServerSetupTeardown 15 | 16 | ITERS = 4 17 | MIN_S = 0.25 18 | # Each request sleeps 0.25s; two concurrent requests should be ~0.25–0.5s. 19 | # Allow some jitter in CI environments. 20 | THRESHOLD = ((MIN_S*(ITERS/2.0)) + (MIN_S/2.0)) # add more jitter for slower CI environments 21 | SERIAL_TIME_WOULD_BE_ABOUT = MIN_S * ITERS 22 | 23 | def setup 24 | @port = 9993 25 | 26 | @response_proc = lambda do |res| 27 | res['Content-Type'] = 'text/plain' 28 | sleep MIN_S 29 | res.body = '200' 30 | end 31 | super 32 | end 33 | 34 | def test_multi_is_scheduler_friendly 35 | if skip_no_async 36 | return 37 | end 38 | 39 | url = "http://127.0.0.1:#{@port}/test" 40 | 41 | started = Time.now 42 | results = [] 43 | 44 | 45 | async_run do 46 | m = Curl::Multi.new 47 | ITERS.times.each do 48 | c = Curl::Easy.new(url) 49 | c.on_complete { results << c.code } 50 | m.add(c) 51 | end 52 | m.perform 53 | end 54 | 55 | duration = Time.now - started 56 | 57 | assert duration < THRESHOLD, "Requests did not run concurrently under fiber scheduler (#{duration}s) which exceeds the expected threshold of: #{THRESHOLD} serial time would be about: #{SERIAL_TIME_WOULD_BE_ABOUT}" 58 | assert_equal ITERS, results.size 59 | assert_equal ITERS.times.map {200}, results 60 | end 61 | 62 | def test_easy_perform_is_scheduler_friendly 63 | if skip_no_async 64 | return 65 | end 66 | 67 | url = "http://127.0.0.1:#{@port}/test" 68 | 69 | started = Time.now 70 | results = [] 71 | 72 | async_run do |top| 73 | tasks = ITERS.times.map do 74 | top.async do 75 | #t = Time.now.to_i 76 | #puts "starting fiber [#{results.size}] -> #{t}" 77 | c = Curl.get(url) 78 | #puts "received result: #{results.size} -> #{Time.now.to_f - t.to_f}" 79 | results << c.code 80 | 81 | end 82 | end 83 | tasks.each(&:wait) 84 | end 85 | 86 | duration = Time.now - started 87 | 88 | assert duration < THRESHOLD, "Requests did not run concurrently under fiber scheduler (#{duration}s) which exceeds the expected threshold of: #{THRESHOLD} serial time would be about: #{SERIAL_TIME_WOULD_BE_ABOUT}" 89 | assert_equal ITERS, results.size 90 | assert_equal ITERS.times.map {200}, results 91 | end 92 | 93 | def test_multi_perform_yields_block_under_scheduler 94 | if skip_no_async 95 | return 96 | end 97 | 98 | url = "http://127.0.0.1:#{@port}/test" 99 | yielded = 0 100 | results = [] 101 | 102 | async_run do 103 | m = Curl::Multi.new 104 | ITERS.times do 105 | c = Curl::Easy.new(url) 106 | c.on_complete { results << c.code } 107 | m.add(c) 108 | end 109 | m.perform do 110 | yielded += 1 111 | end 112 | end 113 | 114 | assert_operator yielded, :>=, 1, 'perform did not yield block while waiting under scheduler' 115 | assert_equal ITERS, results.size 116 | assert_equal ITERS.times.map {200}, results 117 | end 118 | 119 | def test_multi_single_request_scheduler_path 120 | if skip_no_async 121 | return 122 | end 123 | 124 | url = "http://127.0.0.1:#{@port}/test" 125 | result = nil 126 | 127 | async_run do 128 | m = Curl::Multi.new 129 | c = Curl::Easy.new(url) 130 | c.on_complete { result = c.code } 131 | m.add(c) 132 | m.perform 133 | end 134 | 135 | assert_equal 200, result 136 | end 137 | 138 | def test_multi_reuse_after_scheduler_perform 139 | unless HAS_ASYNC 140 | warn 'Skipping fiber scheduler test (Async gem not available)' 141 | return 142 | end 143 | 144 | url = "http://127.0.0.1:#{@port}/test" 145 | results = [] 146 | 147 | async_run do 148 | m = Curl::Multi.new 149 | # First round 150 | c1 = Curl::Easy.new(url) 151 | c1.on_complete { results << c1.code } 152 | m.add(c1) 153 | m.perform 154 | 155 | # Second round on same multi 156 | c2 = Curl::Easy.new(url) 157 | c2.on_complete { results << c2.code } 158 | m.add(c2) 159 | m.perform 160 | end 161 | 162 | assert_equal [200, 200], results 163 | end 164 | 165 | private 166 | def skip_no_async 167 | if WINDOWS 168 | warn 'Skipping fiber scheduler tests on Windows' 169 | return true 170 | end 171 | unless HAS_ASYNC 172 | warn 'Skipping fiber scheduler test (Async gem not available)' 173 | return true 174 | end 175 | false 176 | end 177 | 178 | def async_run(&block) 179 | # Prefer newer Async.run to avoid deprecated scheduler.async path. 180 | if defined?(Async) && Async.respond_to?(:run) 181 | Async.run(&block) 182 | else 183 | Async(&block) 184 | end 185 | end 186 | end 187 | if Gem::Version.new(RUBY_VERSION) < Gem::Version.new('3.1') 188 | warn 'Skipping fiber scheduler tests on Ruby < 3.1' 189 | return 190 | end 191 | -------------------------------------------------------------------------------- /index.html: -------------------------------------------------------------------------------- 1 | <!DOCTYPE html> 2 | <html> 3 | <head> 4 | <meta http-equiv="Content-Type" content="text/html; charset=utf-8"/> 5 | <title>taf2/curb @ GitHub 6 | 30 | 31 | 32 | 33 | 34 | Fork me on GitHub 35 | 36 |
37 | 38 |
39 | 40 | 41 | 42 | 43 |
44 | 45 |

curb 46 | by taf2

47 | 48 |
49 | Curb (probably CUrl-RuBy or something) provides Ruby-language bindings for 50 | libcurl(3). 51 | Curb supports libcurl's 'easy' and 'multi' modes. 52 |
53 | 54 |

Dependencies

55 |

libcurl

56 |

Examples

57 |

Simple fetch via HTTP:

58 | 
 59 | c = Curl::Easy.perform("http://www.google.co.uk")
 60 | puts c.body_str
 61 |
62 |

Same thing, more manual

63 | 
 64 | c = Curl::Easy.new("http://www.google.co.uk")
 65 | c.perform
 66 | puts c.body_str
 67 |
68 |

Additional config

69 | 
 70 | Curl::Easy.perform("http://www.google.co.uk") do |curl| 
 71 |   curl.headers["User-Agent"] = "myapp-0.0"
 72 |   curl.verbose = true
 73 | end
 74 |
75 |

Same thing, more manual

76 | 
 77 | c = Curl::Easy.new("http://www.google.co.uk") do |curl| 
 78 |   curl.headers["User-Agent"] = "myapp-0.0"
 79 |   curl.verbose = true
 80 | end
 81 | 
 82 | c.perform
 83 |
84 | 85 |

Supplying custom handlers

86 | 
 87 | c = Curl::Easy.new("http://www.google.co.uk")
 88 | 
 89 | c.on_body { |data| print(data) }
 90 | c.on_header { |data| print(data) }
 91 | 
 92 | c.perform
 93 |
94 | 95 |

Reusing Curls

96 | 
 97 | c = Curl::Easy.new
 98 | 
 99 | ["http://www.google.co.uk", "http://www.ruby-lang.org/"].map do |url|
100 |   c.url = url
101 |   c.perform
102 |   c.body_str
103 | end
104 |
105 |

HTTP POST form

106 | 
107 | c = Curl::Easy.http_post("http://my.rails.box/thing/create",
108 |                          Curl::PostField.content('thing[name]', 'box',
109 |                          Curl::PostField.content('thing[type]', 'storage')
110 |
111 |

HTTP POST file upload

112 | 
113 | c = Curl::Easy.new("http://my.rails.box/files/upload")
114 | c.multipart_form_post = true
115 | c.http_post(Curl::PostField.file('myfile.rb'))
116 |
117 |

Multi Interface (Basic)

118 |

make multiple GET requests

119 | 
120 | easy_options = {:follow_location => true}
121 | multi_options = {:pipeline => true}
122 | 
123 | Curl::Multi.get('url1','url2','url3','url4','url5', easy_options, multi_options) do|easy|
124 |   # do something interesting with the easy response
125 |   puts easy.last_effective_url
126 | end
127 |
128 | 129 |

make multiple POST requests

130 | 
131 | easy_options = {:follow_location => true, :multipart_form_post => true}
132 | multi_options = {:pipeline => true}
133 | 
134 | url_fields = [
135 |   { :url => 'url1', :post_fields => {'f1' => 'v1'} },
136 |   { :url => 'url2', :post_fields => {'f1' => 'v1'} },
137 |   { :url => 'url3', :post_fields => {'f1' => 'v1'} }
138 | ]
139 | 
140 | Curl::Multi.post(url_fields, easy_options, multi_options) do|easy|
141 |   # do something interesting with the easy response
142 |   puts easy.last_effective_url
143 | end
144 |
145 | 146 | 147 | 148 |

Multi Interface (Advanced)

149 | 
150 | responses = {}
151 | requests = ["http://www.google.co.uk/", "http://www.ruby-lang.org/"]
152 | m = Curl::Multi.new
153 | # add a few easy handles
154 | requests.each do |url|
155 |   responses[url] = ""
156 |   c = Curl::Easy.new(url) do|curl|
157 |     curl.follow_location = true
158 |     curl.on_body{|data| responses[url] << data; data.size }
159 |   end
160 |   m.add(c)
161 | end
162 | 
163 | m.perform do
164 |   puts "idling... can do some work here, including add new requests"
165 | end
166 | 
167 | requests.each do|url|
168 |   puts responses[url]
169 | end
170 |
171 |

Install

172 |

gem install taf2-curb

173 |

Authors

174 |
    175 |
  • Ross Bamford
    • 176 |
  • Todd A. Fisher
    • 177 |
178 |

Contributors

179 |
    180 |
  • brycethornton
    • 181 |
  • Cheah Chu Yeow
    • 182 |
  • mobileAgent
    • 183 |
  • Ian MacLeod
    • 184 |
  • Ilya Grigorik
    • 185 |
  • Jeff Whitmire
    • 186 |
  • Matthew Beale
    • 187 |
  • Phi.Sanders
    • 188 |
  • Phillip Toland
    • 189 |
190 |

Contact

191 |

taf2 via github.com

192 |

Download

193 |

194 | You can download this project in either 195 | zip or 196 | tar formats. 197 |

198 |

You can also clone the project with Git 199 | by running: 200 | 

$ git clone git://github.com/taf2/curb
201 |

202 | 203 |
204 | get the source code on GitHub : taf2/curb 205 |
206 | 207 |
208 | 209 | 210 | 211 | 212 | -------------------------------------------------------------------------------- /ext/curb_macros.h: -------------------------------------------------------------------------------- 1 | /* Curb - helper macros for ruby integration 2 | * Copyright (c)2006 Ross Bamford. 3 | * Licensed under the Ruby License. See LICENSE for details. 4 | * 5 | * $Id: curb_macros.h 13 2006-11-23 23:54:25Z roscopeco $ 6 | */ 7 | 8 | #ifndef __CURB_MACROS_H 9 | #define __CURB_MACROS_H 10 | 11 | #define rb_easy_sym(sym) ID2SYM(rb_intern(sym)) 12 | #define rb_easy_hkey(key) ID2SYM(rb_intern(key)) 13 | #define rb_easy_set(key,val) rb_hash_aset(rbce->opts, rb_easy_hkey(key) , val) 14 | #define rb_easy_get(key) rb_hash_aref(rbce->opts, rb_easy_hkey(key)) 15 | #define rb_easy_del(key) rb_hash_delete(rbce->opts, rb_easy_hkey(key)) 16 | #define rb_easy_nil(key) (rb_hash_aref(rbce->opts, rb_easy_hkey(key)) == Qnil) 17 | #define rb_easy_type_check(key,type) (rb_type(rb_hash_aref(rbce->opts, rb_easy_hkey(key))) == type) 18 | 19 | // TODO: rb_sym_to_s may not be defined? 20 | #define rb_easy_get_str(key) \ 21 | RSTRING_PTR((rb_easy_type_check(key,T_STRING) ? rb_easy_get(key) : rb_str_to_str(rb_easy_get(key)))) 22 | 23 | /* getter/setter macros for various things */ 24 | /* setter for anything that stores a ruby VALUE in the struct */ 25 | #define CURB_OBJECT_SETTER(type, attr) \ 26 | type *ptr; \ 27 | \ 28 | Data_Get_Struct(self, type, ptr); \ 29 | ptr->attr = attr; \ 30 | \ 31 | return attr; 32 | 33 | /* getter for anything that stores a ruby VALUE */ 34 | #define CURB_OBJECT_GETTER(type, attr) \ 35 | type *ptr; \ 36 | \ 37 | Data_Get_Struct(self, type, ptr); \ 38 | return ptr->attr; 39 | 40 | /* setter for anything that stores a ruby VALUE in the struct opts hash */ 41 | #define CURB_OBJECT_HSETTER(type, attr) \ 42 | type *ptr; \ 43 | \ 44 | Data_Get_Struct(self, type, ptr); \ 45 | rb_hash_aset(ptr->opts, rb_easy_hkey(#attr), attr); \ 46 | \ 47 | return attr; 48 | 49 | /* getter for anything that stores a ruby VALUE in the struct opts hash */ 50 | #define CURB_OBJECT_HGETTER(type, attr) \ 51 | type *ptr; \ 52 | \ 53 | Data_Get_Struct(self, type, ptr); \ 54 | return rb_hash_aref(ptr->opts, rb_easy_hkey(#attr)); 55 | 56 | /* setter for bool flags */ 57 | #define CURB_BOOLEAN_SETTER(type, attr) \ 58 | type *ptr; \ 59 | Data_Get_Struct(self, type, ptr); \ 60 | \ 61 | if (attr == Qnil || attr == Qfalse) { \ 62 | ptr->attr = 0; \ 63 | } else { \ 64 | ptr->attr = 1; \ 65 | } \ 66 | \ 67 | return attr; 68 | 69 | /* getter for bool flags */ 70 | #define CURB_BOOLEAN_GETTER(type, attr) \ 71 | type *ptr; \ 72 | Data_Get_Struct(self, type, ptr); \ 73 | \ 74 | return((ptr->attr) ? Qtrue : Qfalse); 75 | 76 | /* special setter for on_event handlers that take a block */ 77 | #define CURB_HANDLER_PROC_SETTER(type, handler) \ 78 | type *ptr; \ 79 | VALUE oldproc; \ 80 | \ 81 | Data_Get_Struct(self, type, ptr); \ 82 | \ 83 | oldproc = ptr->handler; \ 84 | rb_scan_args(argc, argv, "0&", &ptr->handler); \ 85 | \ 86 | return oldproc; \ 87 | 88 | /* special setter for on_event handlers that take a block, same as above but stores int he opts hash */ 89 | #define CURB_HANDLER_PROC_HSETTER(type, handler) \ 90 | type *ptr; \ 91 | VALUE oldproc, newproc; \ 92 | \ 93 | Data_Get_Struct(self, type, ptr); \ 94 | \ 95 | oldproc = rb_hash_aref(ptr->opts, rb_easy_hkey(#handler)); \ 96 | rb_scan_args(argc, argv, "0&", &newproc); \ 97 | \ 98 | rb_hash_aset(ptr->opts, rb_easy_hkey(#handler), newproc); \ 99 | \ 100 | return oldproc; 101 | 102 | /* setter for numerics that are kept in c longs */ 103 | #define CURB_IMMED_SETTER(type, attr, nilval) \ 104 | type *ptr; \ 105 | \ 106 | Data_Get_Struct(self, type, ptr); \ 107 | if (attr == Qnil) { \ 108 | ptr->attr = nilval; \ 109 | } else { \ 110 | ptr->attr = NUM2LONG(attr); \ 111 | } \ 112 | \ 113 | return attr; \ 114 | 115 | /* setter for numerics that are kept in c longs */ 116 | #define CURB_IMMED_GETTER(type, attr, nilval) \ 117 | type *ptr; \ 118 | \ 119 | Data_Get_Struct(self, type, ptr); \ 120 | if (ptr->attr == nilval) { \ 121 | return Qnil; \ 122 | } else { \ 123 | return LONG2NUM(ptr->attr); \ 124 | } 125 | 126 | /* special setter for port / port ranges */ 127 | #define CURB_IMMED_PORT_SETTER(type, attr, msg) \ 128 | type *ptr; \ 129 | \ 130 | Data_Get_Struct(self, type, ptr); \ 131 | if (attr == Qnil) { \ 132 | ptr->attr = 0; \ 133 | } else { \ 134 | int port = NUM2INT(attr); \ 135 | \ 136 | if ((port) && ((port & 0xFFFF) == port)) { \ 137 | ptr->attr = port; \ 138 | } else { \ 139 | rb_raise(rb_eArgError, "Invalid " msg " %d (expected between 1 and 65535)", port); \ 140 | } \ 141 | } \ 142 | \ 143 | return attr; \ 144 | 145 | /* special getter for port / port ranges */ 146 | #define CURB_IMMED_PORT_GETTER(type, attr) \ 147 | type *ptr; \ 148 | \ 149 | Data_Get_Struct(self, type, ptr); \ 150 | if (ptr->attr == 0) { \ 151 | return Qnil; \ 152 | } else { \ 153 | return INT2NUM(ptr->attr); \ 154 | } 155 | 156 | #define CURB_DEFINE(name) \ 157 | rb_define_const(mCurl, #name, LONG2NUM(name)) 158 | 159 | /* copy and raise exception */ 160 | #define CURB_CHECK_RB_CALLBACK_RAISE(did_raise) \ 161 | VALUE exception = rb_hash_aref(did_raise, rb_easy_hkey("error")); \ 162 | if (FIX2INT(rb_hash_size(did_raise)) > 0 && exception != Qnil) { \ 163 | rb_hash_clear(did_raise); \ 164 | VALUE message = rb_funcall(exception, rb_intern("message"), 0); \ 165 | VALUE aborted_exception = rb_exc_new_str(eCurlErrAbortedByCallback, message); \ 166 | VALUE backtrace = rb_funcall(exception, rb_intern("backtrace"), 0); \ 167 | rb_funcall(aborted_exception, rb_intern("set_backtrace"), 1, backtrace); \ 168 | rb_exc_raise(aborted_exception); \ 169 | } 170 | 171 | #endif 172 | -------------------------------------------------------------------------------- /lib/curl/multi.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | module Curl 3 | class Multi 4 | class DownloadError < RuntimeError 5 | attr_accessor :errors 6 | end 7 | class << self 8 | # call-seq: 9 | # Curl::Multi.get(['url1','url2','url3','url4','url5'], :follow_location => true) do|easy| 10 | # easy 11 | # end 12 | # 13 | # Blocking call to fetch multiple url's in parallel. 14 | def get(urls, easy_options={}, multi_options={}, &blk) 15 | url_confs = [] 16 | urls.each do|url| 17 | url_confs << {:url => url, :method => :get}.merge(easy_options) 18 | end 19 | self.http(url_confs, multi_options) {|c,code,method| blk.call(c) if blk } 20 | end 21 | 22 | # call-seq: 23 | # 24 | # Curl::Multi.post([{:url => 'url1', :post_fields => {'field1' => 'value1', 'field2' => 'value2'}}, 25 | # {:url => 'url2', :post_fields => {'field1' => 'value1', 'field2' => 'value2'}}, 26 | # {:url => 'url3', :post_fields => {'field1' => 'value1', 'field2' => 'value2'}}], 27 | # { :follow_location => true, :multipart_form_post => true }, 28 | # {:pipeline => Curl::CURLPIPE_HTTP1}) do|easy| 29 | # easy_handle_on_request_complete 30 | # end 31 | # 32 | # Blocking call to POST multiple form's in parallel. 33 | # 34 | # urls_with_config: is a hash of url's pointing to the postfields to send 35 | # easy_options: are a set of common options to set on all easy handles 36 | # multi_options: options to set on the Curl::Multi handle 37 | # 38 | def post(urls_with_config, easy_options={}, multi_options={}, &blk) 39 | url_confs = [] 40 | urls_with_config.each do|uconf| 41 | url_confs << uconf.merge(:method => :post).merge(easy_options) 42 | end 43 | self.http(url_confs, multi_options) {|c,code,method| blk.call(c) } 44 | end 45 | 46 | # call-seq: 47 | # 48 | # Curl::Multi.put([{:url => 'url1', :put_data => "some message"}, 49 | # {:url => 'url2', :put_data => IO.read('filepath')}, 50 | # {:url => 'url3', :put_data => "maybe another string or socket?"], 51 | # {:follow_location => true}, 52 | # {:pipeline => Curl::CURLPIPE_HTTP1}) do|easy| 53 | # easy_handle_on_request_complete 54 | # end 55 | # 56 | # Blocking call to POST multiple form's in parallel. 57 | # 58 | # urls_with_config: is a hash of url's pointing to the postfields to send 59 | # easy_options: are a set of common options to set on all easy handles 60 | # multi_options: options to set on the Curl::Multi handle 61 | # 62 | def put(urls_with_config, easy_options={}, multi_options={}, &blk) 63 | url_confs = [] 64 | urls_with_config.each do|uconf| 65 | url_confs << uconf.merge(:method => :put).merge(easy_options) 66 | end 67 | self.http(url_confs, multi_options) {|c,code,method| blk.call(c) } 68 | end 69 | 70 | 71 | # call-seq: 72 | # 73 | # Curl::Multi.http( [ 74 | # { :url => 'url1', :method => :post, 75 | # :post_fields => {'field1' => 'value1', 'field2' => 'value2'} }, 76 | # { :url => 'url2', :method => :get, 77 | # :follow_location => true, :max_redirects => 3 }, 78 | # { :url => 'url3', :method => :put, :put_data => File.open('file.txt','rb') }, 79 | # { :url => 'url4', :method => :head } 80 | # ], {:pipeline => Curl::CURLPIPE_HTTP1}) 81 | # 82 | # Blocking call to issue multiple HTTP requests with varying verb's. 83 | # 84 | # urls_with_config: is a hash of url's pointing to the easy handle options as well as the special option :method, that can by one of [:get, :post, :put, :delete, :head], when no verb is provided e.g. :method => nil -> GET is used 85 | # multi_options: options for the multi handle 86 | # blk: a callback, that yeilds when a handle is completed 87 | # 88 | def http(urls_with_config, multi_options={}, &blk) 89 | m = Curl::Multi.new 90 | 91 | # maintain a sane number of easy handles 92 | multi_options[:max_connects] = max_connects = multi_options.key?(:max_connects) ? multi_options[:max_connects] : 10 93 | 94 | free_handles = [] # keep a list of free easy handles 95 | 96 | # configure the multi handle 97 | multi_options.each { |k,v| m.send("#{k}=", v) } 98 | callbacks = [:on_progress,:on_debug,:on_failure,:on_success,:on_redirect,:on_missing,:on_body,:on_header] 99 | 100 | add_free_handle = proc do|conf, easy| 101 | c = conf.dup # avoid being destructive to input 102 | url = c.delete(:url) 103 | method = c.delete(:method) 104 | headers = c.delete(:headers) 105 | 106 | easy = Curl::Easy.new if easy.nil? 107 | 108 | easy.url = url 109 | 110 | # assign callbacks 111 | callbacks.each do |cb| 112 | cbproc = c.delete(cb) 113 | easy.send(cb,&cbproc) if cbproc 114 | end 115 | 116 | case method 117 | when :post 118 | fields = c.delete(:post_fields) 119 | # set the post post using the url fields 120 | easy.post_body = fields.map{|f,k| "#{easy.escape(f)}=#{easy.escape(k)}"}.join('&') 121 | when :put 122 | easy.put_data = c.delete(:put_data) 123 | when :head 124 | easy.head = true 125 | when :delete 126 | easy.delete = true 127 | when :get 128 | else 129 | # XXX: nil is treated like a GET 130 | end 131 | 132 | # headers is a special key 133 | headers.each {|k,v| easy.headers[k] = v } if headers 134 | 135 | # 136 | # use the remaining options as specific configuration to the easy handle 137 | # bad options should raise an undefined method error 138 | # 139 | c.each { |k,v| easy.send("#{k}=",v) } 140 | 141 | easy.on_complete {|curl| 142 | free_handles << curl 143 | blk.call(curl,curl.response_code,method) if blk 144 | } 145 | m.add(easy) 146 | end 147 | 148 | max_connects.times do 149 | conf = urls_with_config.pop 150 | add_free_handle.call(conf, nil) if conf 151 | break if urls_with_config.empty? 152 | end 153 | 154 | consume_free_handles = proc do 155 | # as we idle consume free handles 156 | if urls_with_config.size > 0 && free_handles.size > 0 157 | easy = free_handles.pop 158 | conf = urls_with_config.pop 159 | add_free_handle.call(conf, easy) if conf 160 | end 161 | end 162 | 163 | if urls_with_config.empty? 164 | m.perform 165 | else 166 | until urls_with_config.empty? 167 | m.perform do 168 | consume_free_handles.call 169 | end 170 | consume_free_handles.call 171 | end 172 | free_handles = nil 173 | end 174 | 175 | end 176 | 177 | # call-seq: 178 | # 179 | # Curl::Multi.download(['http://example.com/p/a/t/h/file1.txt','http://example.com/p/a/t/h/file2.txt']){|c|} 180 | # 181 | # will create 2 new files file1.txt and file2.txt 182 | # 183 | # 2 files will be opened, and remain open until the call completes 184 | # 185 | # when using the :post or :put method, urls should be a hash, including the individual post fields per post 186 | # 187 | def download(urls,easy_options={},multi_options={},download_paths=nil,&blk) 188 | errors = [] 189 | procs = [] 190 | files = [] 191 | urls_with_config = [] 192 | url_to_download_paths = {} 193 | 194 | urls.each_with_index do|urlcfg,i| 195 | if urlcfg.is_a?(Hash) 196 | url = url[:url] 197 | else 198 | url = urlcfg 199 | end 200 | 201 | if download_paths and download_paths[i] 202 | download_path = download_paths[i] 203 | else 204 | download_path = File.basename(url) 205 | end 206 | 207 | file = lambda do|dp| 208 | file = File.open(dp,"wb") 209 | procs << (lambda {|data| file.write data; data.size }) 210 | files << file 211 | file 212 | end.call(download_path) 213 | 214 | if urlcfg.is_a?(Hash) 215 | urls_with_config << urlcfg.merge({:on_body => procs.last}.merge(easy_options)) 216 | else 217 | urls_with_config << {:url => url, :on_body => procs.last, :method => :get}.merge(easy_options) 218 | end 219 | url_to_download_paths[url] = {:path => download_path, :file => file} # store for later 220 | end 221 | 222 | if blk 223 | # when injecting the block, ensure file is closed before yielding 224 | Curl::Multi.http(urls_with_config, multi_options) do |c,code,method| 225 | info = url_to_download_paths[c.url] 226 | begin 227 | file = info[:file] 228 | files.reject!{|f| f == file } 229 | file.close 230 | rescue => e 231 | errors << e 232 | end 233 | blk.call(c,info[:path]) 234 | end 235 | else 236 | Curl::Multi.http(urls_with_config, multi_options) 237 | end 238 | 239 | ensure 240 | files.each {|f| 241 | begin 242 | f.close 243 | rescue => e 244 | errors << e 245 | end 246 | } 247 | if errors.any? 248 | de = Curl::Multi::DownloadError.new 249 | de.errors = errors 250 | raise de 251 | end 252 | end 253 | end 254 | 255 | def cancel! 256 | requests.each do |_,easy| 257 | remove(easy) 258 | end 259 | end 260 | 261 | def idle? 262 | requests.empty? 263 | end 264 | 265 | def requests 266 | @requests ||= {} 267 | end 268 | 269 | def add(easy) 270 | return self if requests[easy.object_id] 271 | requests[easy.object_id] = easy 272 | _add(easy) 273 | self 274 | end 275 | 276 | def remove(easy) 277 | return self if !requests[easy.object_id] 278 | requests.delete(easy.object_id) 279 | _remove(easy) 280 | self 281 | end 282 | 283 | def close 284 | requests.values.each {|easy| 285 | _remove(easy) 286 | } 287 | @requests = {} 288 | _close 289 | self 290 | end 291 | 292 | 293 | end 294 | end 295 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # $Id$ 2 | # 3 | require 'rake/clean' 4 | require 'rake/testtask' 5 | require "ruby_memcheck" 6 | begin 7 | require 'mixlib/shellout' 8 | rescue LoadError 9 | end 10 | 11 | CLEAN.include '**/*.o' 12 | CLEAN.include "**/*.#{(defined?(RbConfig) ? RbConfig : Config)::MAKEFILE_CONFIG['DLEXT']}" 13 | CLOBBER.include 'doc' 14 | CLOBBER.include '**/*.log' 15 | CLOBBER.include '**/Makefile' 16 | CLOBBER.include '**/extconf.h' 17 | 18 | # Load support ruby and rake files (in this order) 19 | Dir.glob('tasks/*.rb').each { |r| load r} 20 | Dir.glob('tasks/*.rake').each { |r| load r} 21 | 22 | desc 'Print Ruby major version (ie "2_5")' 23 | task :ruby_version do 24 | print current_ruby_major 25 | end 26 | 27 | def announce(msg='') 28 | $stderr.puts msg 29 | end 30 | 31 | desc "Default Task (Test project)" 32 | task :default => :test 33 | 34 | # Determine the current version of the software 35 | if File.read('ext/curb.h') =~ /\s*CURB_VERSION\s*['"](\d.+)['"]/ 36 | CURRENT_VERSION = $1 37 | else 38 | CURRENT_VERSION = "0.0.0" 39 | end 40 | 41 | if ENV['REL'] 42 | PKG_VERSION = ENV['REL'] 43 | else 44 | PKG_VERSION = CURRENT_VERSION 45 | end 46 | 47 | task :test_ver do 48 | puts PKG_VERSION 49 | end 50 | 51 | # Make tasks ----------------------------------------------------- 52 | make_program = (/mswin/ =~ RUBY_PLATFORM) ? 'nmake' : 'make' 53 | MAKECMD = ENV['MAKE_CMD'] || make_program 54 | MAKEOPTS = ENV['MAKE_OPTS'] || '' 55 | CURB_SO = "ext/curb_core.#{(defined?(RbConfig) ? RbConfig : Config)::MAKEFILE_CONFIG['DLEXT']}" 56 | 57 | file 'ext/Makefile' => 'ext/extconf.rb' do 58 | shell(['ruby', 'extconf.rb', ENV['EXTCONF_OPTS'].to_s], 59 | { :live_stdout => STDOUT , :cwd => "#{Dir.pwd}/ext" } 60 | ).error! 61 | end 62 | 63 | def make(target = '') 64 | shell(["#{MAKECMD}", "#{MAKEOPTS}", "#{target}"].reject(&:empty?), 65 | { :live_stdout => STDOUT, :cwd => "#{Dir.pwd}/ext" } 66 | ).error! 67 | end 68 | 69 | # Let make handle dependencies between c/o/so - we'll just run it. 70 | file CURB_SO => (['ext/Makefile'] + Dir['ext/*.c'] + Dir['ext/*.h']) do 71 | make 72 | end 73 | 74 | desc "Compile the shared object" 75 | task :compile => [CURB_SO] 76 | 77 | desc "Install to your site_ruby directory" 78 | task :install do 79 | make 'install' 80 | end 81 | 82 | # Test Tasks --------------------------------------------------------- 83 | task :ta => :alltests 84 | task :tu => :unittests 85 | task :test => [:rmpid,:unittests] 86 | 87 | task :rmpid do 88 | FileUtils.rm_rf Dir.glob("tests/server_lock-*") 89 | end 90 | 91 | if ENV['RELTEST'] 92 | announce "Release task testing - not running regression tests on alltests" 93 | task :alltests => [:unittests] 94 | else 95 | task :alltests => [:unittests, :bugtests] 96 | end 97 | 98 | RubyMemcheck.config(binary_name: 'curb_core') 99 | namespace :test do 100 | RubyMemcheck::TestTask.new(valgrind: :compile) do|t| 101 | t.test_files = FileList['tests/tc_*.rb'] 102 | t.verbose = false 103 | end 104 | end 105 | 106 | Rake::TestTask.new(:unittests) do |t| 107 | t.test_files = FileList['tests/tc_*.rb'] 108 | t.verbose = false 109 | end 110 | 111 | Rake::TestTask.new(:bugtests) do |t| 112 | t.test_files = FileList['tests/bug_*.rb'] 113 | t.verbose = false 114 | end 115 | 116 | #Rake::TestTask.new(:funtests) do |t| 117 | # t.test_files = FileList['test/func_*.rb'] 118 | #t.warning = true 119 | #t.warning = true 120 | #end 121 | 122 | task :unittests => :compile 123 | task :bugtests => :compile 124 | 125 | def has_gem?(file,name) 126 | begin 127 | require file 128 | has_http_persistent = true 129 | rescue LoadError => e 130 | puts "Skipping #{name}" 131 | end 132 | end 133 | 134 | desc "Benchmark curl against http://127.0.0.1/zeros-2k - will fail if /zeros-2k or 127.0.0.1 are missing" 135 | task :bench do 136 | sh "ruby bench/curb_easy.rb" 137 | sh "ruby bench/curb_multi.rb" 138 | sh "ruby bench/nethttp_test.rb" if has_gem?("net/http/persistent","net-http-persistent") 139 | sh "ruby bench/patron_test.rb" if has_gem?("patron","patron") 140 | sh "ruby bench/typhoeus_test.rb" if has_gem?("typhoeus","typhoeus") 141 | sh "ruby bench/typhoeus_hydra_test.rb" if has_gem?("typhoeus","typhoeus") 142 | end 143 | 144 | # RDoc Tasks --------------------------------------------------------- 145 | desc "Create the RDOC documentation" 146 | task :doc do 147 | ruby "doc.rb #{ENV['DOC_OPTS']}" 148 | end 149 | 150 | desc "Publish the RDoc documentation to project web site" 151 | task :doc_upload => [ :doc ] do 152 | begin 153 | require 'rdoc/task' 154 | rescue LoadError => e 155 | require 'rake/rdoctask' 156 | end 157 | 158 | if ENV['RELTEST'] 159 | announce "Release Task Testing, skipping doc upload" 160 | else 161 | unless ENV['RUBYFORGE_ACCT'] 162 | raise "Need to set RUBYFORGE_ACCT to your rubyforge.org user name (e.g. 'fred')" 163 | end 164 | 165 | require 'rake/contrib/sshpublisher' 166 | Rake::SshDirPublisher.new( 167 | "#{ENV['RUBYFORGE_ACCT']}@rubyforge.org", 168 | "/var/www/gforge-projects/curb", 169 | "doc" 170 | ).upload 171 | end 172 | end 173 | 174 | if ! defined?(Gem) 175 | warn "Package Target requires RubyGEMs" 176 | else 177 | desc 'Generate gem specification' 178 | task :gemspec do 179 | require 'erb' 180 | tspec = ERB.new(File.read(File.join(File.dirname(__FILE__),'lib','curb.gemspec.erb'))) 181 | File.open(File.join(File.dirname(__FILE__),'curb.gemspec'),'wb') do|f| 182 | f << tspec.result 183 | end 184 | end 185 | 186 | desc 'Build gem' 187 | task :package => :gemspec do 188 | require 'rubygems/package' 189 | spec_source = File.read File.join(File.dirname(__FILE__),'curb.gemspec') 190 | spec = nil 191 | # see: http://gist.github.com/16215 192 | Thread.new { spec = eval("#{spec_source}") }.join 193 | spec.validate 194 | Gem::Package.build(spec) 195 | end 196 | 197 | task :static do 198 | ENV['STATIC_BUILD'] = '1' 199 | end 200 | 201 | task :binary_gemspec => [:static, :compile] do 202 | require 'erb' 203 | ENV['BINARY_PACKAGE'] = '1' 204 | tspec = ERB.new(File.read(File.join(File.dirname(__FILE__),'lib','curb.gemspec.erb'))) 205 | 206 | File.open(File.join(File.dirname(__FILE__),'curb-binary.gemspec'),'wb') do|f| 207 | f << tspec.result 208 | end 209 | end 210 | 211 | desc 'Strip extra strings from Binary' 212 | task :binary_strip do 213 | strip = '/usr/bin/strip' 214 | if File.exist?(strip) and `#{strip} -h 2>&1`.match(/GNU/) 215 | sh "#{strip} #{CURB_SO}" 216 | end 217 | end 218 | 219 | desc 'Build gem' 220 | task :binary_package => [:binary_gemspec, :binary_strip] do 221 | require 'rubygems/specification' 222 | spec_source = File.read File.join(File.dirname(__FILE__),'curb-binary.gemspec') 223 | spec = nil 224 | # see: http://gist.github.com/16215 225 | Thread.new { spec = eval("$SAFE = 3\n#{spec_source}") }.join 226 | spec.validate 227 | Gem::Builder.new(spec).build 228 | end 229 | end 230 | 231 | # -------------------------------------------------------------------- 232 | # Creating a release 233 | desc "Make a new release (Requires SVN commit / webspace access)" 234 | task :release => [ 235 | :prerelease, 236 | :clobber, 237 | :alltests, 238 | :update_version, 239 | :package, 240 | :tag, 241 | :doc_upload] do 242 | 243 | announce 244 | announce "**************************************************************" 245 | announce "* Release #{PKG_VERSION} Complete." 246 | announce "* Packages ready to upload." 247 | announce "**************************************************************" 248 | announce 249 | end 250 | 251 | # Validate that everything is ready to go for a release. 252 | task :prerelease do 253 | announce 254 | announce "**************************************************************" 255 | announce "* Making RubyGem Release #{PKG_VERSION}" 256 | announce "* (current version #{CURRENT_VERSION})" 257 | announce "**************************************************************" 258 | announce 259 | 260 | # Is a release number supplied? 261 | unless ENV['REL'] 262 | fail "Usage: rake release REL=x.y.z [REUSE=tag_suffix]" 263 | end 264 | 265 | # Is the release different than the current release. 266 | # (or is REUSE set?) 267 | if PKG_VERSION == CURRENT_VERSION && ! ENV['REUSE'] 268 | fail "Current version is #{PKG_VERSION}, must specify REUSE=tag_suffix to reuse version" 269 | end 270 | 271 | # Are all source files checked in? 272 | if ENV['RELTEST'] 273 | announce "Release Task Testing, skipping checked-in file test" 274 | else 275 | announce "Checking for unchecked-in files..." 276 | data = `svn status` 277 | unless data =~ /^$/ 278 | fail "SVN status is not clean ... do you have unchecked-in files?" 279 | end 280 | announce "No outstanding checkins found ... OK" 281 | end 282 | 283 | announce "Doc will try to use GNU cpp if available" 284 | ENV['DOC_OPTS'] = "--cpp" 285 | end 286 | 287 | # Used during release packaging if a REL is supplied 288 | task :update_version do 289 | unless PKG_VERSION == CURRENT_VERSION 290 | pkg_vernum = PKG_VERSION.tr('.','').sub(/^0*/,'') 291 | pkg_vernum << '0' until pkg_vernum.length > 2 292 | 293 | File.open('ext/curb.h.new','w+') do |f| 294 | maj, min, mic, patch = /(\d+)\.(\d+)(?:\.(\d+))?(?:\.(\d+))?/.match(PKG_VERSION).captures 295 | f << File.read('ext/curb.h'). 296 | gsub(/CURB_VERSION\s+"(\d.+)"/) { "CURB_VERSION \"#{PKG_VERSION}\"" }. 297 | gsub(/CURB_VER_NUM\s+\d+/) { "CURB_VER_NUM #{pkg_vernum}" }. 298 | gsub(/CURB_VER_MAJ\s+\d+/) { "CURB_VER_MAJ #{maj}" }. 299 | gsub(/CURB_VER_MIN\s+\d+/) { "CURB_VER_MIN #{min}" }. 300 | gsub(/CURB_VER_MIC\s+\d+/) { "CURB_VER_MIC #{mic || 0}" }. 301 | gsub(/CURB_VER_PATCH\s+\d+/) { "CURB_VER_PATCH #{patch || 0}" } 302 | end 303 | mv('ext/curb.h.new', 'ext/curb.h') 304 | if ENV['RELTEST'] 305 | announce "Release Task Testing, skipping commiting of new version" 306 | else 307 | sh %{svn commit -m "Updated to version #{PKG_VERSION}" ext/curb.h} 308 | end 309 | end 310 | end 311 | 312 | # "Create a new SVN tag with the latest release number (REL=x.y.z)" 313 | task :tag => [:prerelease] do 314 | reltag = "curb-#{PKG_VERSION}" 315 | reltag << ENV['REUSE'] if ENV['REUSE'] 316 | announce "Tagging SVN with [#{reltag}]" 317 | if ENV['RELTEST'] 318 | announce "Release Task Testing, skipping SVN tagging" 319 | else 320 | # need to get current base URL 321 | s = `svn info` 322 | if s =~ /URL:\s*([^\n]*)\n/ 323 | svnroot = $1 324 | if svnroot =~ /^(.*)\/trunk/i 325 | svnbase = $1 326 | sh %{svn cp #{svnroot} #{svnbase}/TAGS/#{reltag} -m "Release #{PKG_VERSION}"} 327 | else 328 | fail "Please merge to trunk before making a release" 329 | end 330 | else 331 | fail "Unable to determine repository URL from 'svn info' - is this a working copy?" 332 | end 333 | end 334 | end 335 | -------------------------------------------------------------------------------- /tests/helper.rb: -------------------------------------------------------------------------------- 1 | # DO NOT REMOVE THIS COMMENT - PART OF TESTMODEL. 2 | # Copyright (c)2006 Ross Bamford. See LICENSE. 3 | $CURB_TESTING = true 4 | require 'uri' 5 | require 'stringio' 6 | require 'digest/md5' 7 | 8 | $TOPDIR = File.expand_path(File.join(File.dirname(__FILE__), '..')) 9 | $EXTDIR = File.join($TOPDIR, 'ext') 10 | $LIBDIR = File.join($TOPDIR, 'lib') 11 | $:.unshift($LIBDIR) 12 | $:.unshift($EXTDIR) 13 | 14 | # Setup SimpleCov for Ruby code coverage if COVERAGE env var is set 15 | if ENV['COVERAGE'] 16 | begin 17 | require 'simplecov' 18 | require 'simplecov-lcov' 19 | 20 | SimpleCov::Formatter::LcovFormatter.config.report_with_single_file = true 21 | SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter.new([ 22 | SimpleCov::Formatter::HTMLFormatter, 23 | SimpleCov::Formatter::LcovFormatter 24 | ]) 25 | 26 | SimpleCov.start do 27 | add_filter '/tests/' 28 | add_filter '/spec/' 29 | add_filter '/vendor/' 30 | add_filter '/.bundle/' 31 | add_group 'Library', 'lib' 32 | add_group 'Extensions', 'ext' 33 | 34 | # Track branch coverage if available 35 | enable_coverage :branch if respond_to?(:enable_coverage) 36 | end 37 | rescue LoadError 38 | puts "SimpleCov not available. Install it with: gem install simplecov simplecov-lcov" 39 | end 40 | end 41 | 42 | require 'curb' 43 | begin 44 | require 'test/unit' 45 | rescue LoadError 46 | gem 'test/unit' 47 | require 'test/unit' 48 | end 49 | require 'fileutils' 50 | require 'rbconfig' 51 | 52 | # Platform helpers 53 | WINDOWS = /mswin|msys|mingw|cygwin|bccwin|wince|emc|windows/i.match?(RbConfig::CONFIG['host_os']) 54 | NO_FORK = !Process.respond_to?(:fork) 55 | 56 | $TEST_URL = "file://#{'/' if RUBY_DESCRIPTION =~ /mswin|msys|mingw|cygwin|bccwin|wince|emc/}#{File.expand_path(__FILE__).tr('\\','/')}" 57 | 58 | require 'thread' 59 | require 'webrick' 60 | 61 | # set this to true to avoid testing with multiple threads 62 | # or to test with multiple threads set it to false 63 | # this is important since, some code paths will change depending 64 | # on the presence of multiple threads 65 | TEST_SINGLE_THREADED=false 66 | 67 | # keep webrick quiet 68 | ::WEBrick::HTTPServer.send(:remove_method,:access_log) if ::WEBrick::HTTPServer.instance_methods.include?(:access_log) 69 | ::WEBrick::BasicLog.send(:remove_method,:log) if ::WEBrick::BasicLog.instance_methods.include?(:log) 70 | 71 | ::WEBrick::HTTPServer.class_eval do 72 | def access_log(config, req, res) 73 | # nop 74 | end 75 | end 76 | ::WEBrick::BasicLog.class_eval do 77 | def log(level, data) 78 | # nop 79 | end 80 | end 81 | 82 | # 83 | # Simple test server to record number of times a request is sent/recieved of a specific 84 | # request type, e.g. GET,POST,PUT,DELETE 85 | # 86 | class TestServlet < WEBrick::HTTPServlet::AbstractServlet 87 | 88 | def self.port=(p) 89 | @port = p 90 | end 91 | 92 | def self.port 93 | @port ||= 9129 94 | end 95 | 96 | def self.path 97 | '/methods' 98 | end 99 | def self.url 100 | "http://127.0.0.1:#{port}#{path}" 101 | end 102 | 103 | def respond_with(method,req,res) 104 | res.body = method.to_s 105 | $auth_header = req['Authorization'] 106 | res['Content-Type'] = "text/plain" 107 | end 108 | 109 | def do_GET(req,res) 110 | if req.path.match(/redirect$/) 111 | res.status = 302 112 | res['Location'] = '/foo' 113 | elsif req.path.match(/not_here$/) 114 | res.status = 404 115 | elsif req.path.match(/error$/) 116 | res.status = 500 117 | elsif req.path.match(/get_cookies$/) 118 | res['Content-Type'] = "text/plain" 119 | res.body = req['Cookie'] 120 | return 121 | end 122 | respond_with("GET#{req.query_string}",req,res) 123 | end 124 | 125 | def do_HEAD(req,res) 126 | res['Location'] = "/nonexistent" 127 | respond_with("HEAD#{req.query_string}",req,res) 128 | end 129 | 130 | def do_POST(req,res) 131 | if req.path.match(/set_cookies$/) 132 | JSON.parse(req.body || '[]', symbolize_names: true).each do |hash| 133 | cookie = WEBrick::Cookie.new(hash.fetch(:name), hash.fetch(:value)) 134 | cookie.domain = hash[:domain] if hash.key?(:domain) 135 | cookie.expires = hash[:expires] if hash.key?(:expires) 136 | cookie.path = hash[:path] if hash.key?(:path) 137 | cookie.secure = hash[:secure] if hash.key?(:secure) 138 | cookie.max_age = hash[:max_age] if hash.key?(:max_age) 139 | res.cookies.push(cookie) 140 | end 141 | respond_with('OK', req, res) 142 | elsif req.query['filename'].nil? 143 | if req.body 144 | params = {} 145 | req.body.split('&').map{|s| k,v=s.split('='); params[k] = v } 146 | end 147 | if params and params['s'] == '500' 148 | res.status = 500 149 | elsif params and params['c'] 150 | cookie = URI.decode_www_form_component(params['c']).split('=') 151 | res.cookies << WEBrick::Cookie.new(*cookie) 152 | else 153 | respond_with("POST\n#{req.body}",req,res) 154 | end 155 | else 156 | respond_with(req.query['filename'],req,res) 157 | end 158 | end 159 | 160 | def do_PUT(req,res) 161 | res['X-Requested-Content-Type'] = req.content_type 162 | respond_with("PUT\n#{req.body}",req,res) 163 | end 164 | 165 | def do_DELETE(req,res) 166 | respond_with("DELETE#{req.query_string}",req,res) 167 | end 168 | 169 | def do_PURGE(req,res) 170 | respond_with("PURGE#{req.query_string}",req,res) 171 | end 172 | 173 | def do_COPY(req,res) 174 | respond_with("COPY#{req.query_string}",req,res) 175 | end 176 | 177 | def do_PATCH(req,res) 178 | respond_with("PATCH\n#{req.body}",req,res) 179 | end 180 | 181 | def do_OPTIONS(req,res) 182 | respond_with("OPTIONS#{req.query_string}",req,res) 183 | end 184 | 185 | end 186 | 187 | module BugTestServerSetupTeardown 188 | def setup 189 | @port ||= 9992 190 | @server = WEBrick::HTTPServer.new( :Port => @port ) 191 | @server.mount_proc("/test") do|req,res| 192 | if @response_proc 193 | @response_proc.call(res) 194 | else 195 | res.body = "hi" 196 | res['Content-Type'] = "text/html" 197 | end 198 | end 199 | 200 | @thread = Thread.new(@server) do|srv| 201 | srv.start 202 | end 203 | end 204 | 205 | def teardown 206 | while @server.status != :Shutdown 207 | @server.shutdown 208 | end 209 | @thread.join 210 | end 211 | end 212 | 213 | module TestServerMethods 214 | def locked_file 215 | File.join(File.dirname(__FILE__),"server_lock-#{@__port}") 216 | end 217 | 218 | def server_setup(port=9129,servlet=TestServlet) 219 | @__port = port 220 | if (@server ||= nil).nil? and !File.exist?(locked_file) 221 | File.open(locked_file,'w') {|f| f << 'locked' } 222 | if TEST_SINGLE_THREADED 223 | rd, wr = IO.pipe 224 | @__pid = fork do 225 | rd.close 226 | rd = nil 227 | 228 | # start up a webrick server for testing delete 229 | server = WEBrick::HTTPServer.new :Port => port, :DocumentRoot => File.expand_path(File.dirname(__FILE__)) 230 | 231 | server.mount(servlet.path, servlet) 232 | server.mount("/ext", WEBrick::HTTPServlet::FileHandler, File.join(File.dirname(__FILE__),'..','ext')) 233 | 234 | trap("INT") { server.shutdown } 235 | GC.start 236 | wr.flush 237 | wr.close 238 | server.start 239 | end 240 | wr.close 241 | rd.read 242 | rd.close 243 | else 244 | # start up a webrick server for testing delete 245 | @server = WEBrick::HTTPServer.new :Port => port, :DocumentRoot => File.expand_path(File.dirname(__FILE__)) 246 | 247 | @server.mount(servlet.path, servlet) 248 | @server.mount("/ext", WEBrick::HTTPServlet::FileHandler, File.join(File.dirname(__FILE__),'..','ext')) 249 | queue = Queue.new # synchronize the thread startup to the main thread 250 | 251 | @test_thread = Thread.new { queue << 1; @server.start } 252 | 253 | # wait for the queue 254 | value = queue.pop 255 | if !value 256 | STDERR.puts "Failed to startup test server!" 257 | exit(1) 258 | end 259 | 260 | end 261 | 262 | exit_code = lambda do 263 | begin 264 | if File.exist?(locked_file) 265 | File.unlink locked_file 266 | if TEST_SINGLE_THREADED 267 | Process.kill 'INT', @__pid 268 | else 269 | @server.shutdown unless @server.nil? 270 | end 271 | end 272 | #@server.shutdown unless @server.nil? 273 | rescue Object => e 274 | puts "Error #{__FILE__}:#{__LINE__}\n#{e.message}" 275 | end 276 | end 277 | 278 | trap("INT"){exit_code.call} 279 | at_exit{exit_code.call} 280 | 281 | end 282 | rescue Errno::EADDRINUSE 283 | end 284 | end 285 | 286 | 287 | 288 | # Backport for Ruby 1.8 289 | module Backports 290 | module Ruby18 291 | module URIFormEncoding 292 | TBLENCWWWCOMP_ = {} 293 | TBLDECWWWCOMP_ = {} 294 | 295 | def encode_www_form_component(str) 296 | if TBLENCWWWCOMP_.empty? 297 | 256.times do |i| 298 | TBLENCWWWCOMP_[i.chr] = '%%%02X' % i 299 | end 300 | TBLENCWWWCOMP_[' '] = '+' 301 | TBLENCWWWCOMP_.freeze 302 | end 303 | str.to_s.gsub( /([^*\-.0-9A-Z_a-z])/ ) {|*| TBLENCWWWCOMP_[$1] } 304 | end 305 | 306 | def decode_www_form_component(str) 307 | if TBLDECWWWCOMP_.empty? 308 | 256.times do |i| 309 | h, l = i>>4, i&15 310 | TBLDECWWWCOMP_['%%%X%X' % [h, l]] = i.chr 311 | TBLDECWWWCOMP_['%%%x%X' % [h, l]] = i.chr 312 | TBLDECWWWCOMP_['%%%X%x' % [h, l]] = i.chr 313 | TBLDECWWWCOMP_['%%%x%x' % [h, l]] = i.chr 314 | end 315 | TBLDECWWWCOMP_['+'] = ' ' 316 | TBLDECWWWCOMP_.freeze 317 | end 318 | 319 | raise ArgumentError, "invalid %-encoding (#{str.dump})" unless /\A(?:%[[:xdigit:]]{2}|[^%]+)*\z/ =~ str 320 | str.gsub( /(\+|%[[:xdigit:]]{2})/ ) {|*| TBLDECWWWCOMP_[$1] } 321 | end 322 | 323 | def encode_www_form( enum ) 324 | enum.map do |k,v| 325 | if v.nil? 326 | encode_www_form_component(k) 327 | elsif v.respond_to?(:to_ary) 328 | v.to_ary.map do |w| 329 | str = encode_www_form_component(k) 330 | unless w.nil? 331 | str << '=' 332 | str << encode_www_form_component(w) 333 | end 334 | end.join('&') 335 | else 336 | str = encode_www_form_component(k) 337 | str << '=' 338 | str << encode_www_form_component(v) 339 | end 340 | end.join('&') 341 | end 342 | 343 | WFKV_ = '(?:%\h\h|[^%#=;&])' 344 | def decode_www_form(str, _) 345 | return [] if str.to_s == '' 346 | 347 | unless /\A#{WFKV_}=#{WFKV_}(?:[;&]#{WFKV_}=#{WFKV_})*\z/ =~ str 348 | raise ArgumentError, "invalid data of application/x-www-form-urlencoded (#{str})" 349 | end 350 | ary = [] 351 | $&.scan(/([^=;&]+)=([^;&]*)/) do 352 | ary << [decode_www_form_component($1, enc), decode_www_form_component($2, enc)] 353 | end 354 | ary 355 | end 356 | end 357 | end 358 | end 359 | 360 | unless URI.methods.include?(:encode_www_form) 361 | URI.extend(Backports::Ruby18::URIFormEncoding) 362 | end 363 | -------------------------------------------------------------------------------- /tests/tc_curl_easy_cookielist.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.join(File.dirname(__FILE__), 'helper')) 2 | require 'json' 3 | 4 | class TestCurbCurlEasyCookielist < Test::Unit::TestCase 5 | def test_setopt_cookielist 6 | easy = Curl::Easy.new 7 | # DateTime handles time zone correctly 8 | expires = (Date.today + 2).to_datetime 9 | easy.setopt(Curl::CURLOPT_COOKIELIST, "Set-Cookie: c1=v1; domain=localhost; expires=#{expires.httpdate};") 10 | easy.setopt(Curl::CURLOPT_COOKIELIST, 'Set-Cookie: c2=v2; domain=localhost') 11 | easy.setopt(Curl::CURLOPT_COOKIELIST, "Set-Cookie: c3=v3; expires=#{expires.httpdate};") 12 | easy.setopt(Curl::CURLOPT_COOKIELIST, 'Set-Cookie: c4=v4;') 13 | easy.setopt(Curl::CURLOPT_COOKIELIST, "Set-Cookie: c5=v5; domain=127.0.0.1; expires=#{expires.httpdate};") 14 | easy.setopt(Curl::CURLOPT_COOKIELIST, 'Set-Cookie: c6=v6; domain=127.0.0.1;;') 15 | 16 | # Since 7.43.0 cookies that were imported in the Set-Cookie format without a domain name are not exported by this option. 17 | # So, before 7.43.0, c3 and c4 will be exported too; but that version is far too old for current curb version, so it's not handled here. 18 | if Curl::CURL_VERSION.to_f > 8 19 | expected_cookielist = [ 20 | ".127.0.0.1\tTRUE\t/\tFALSE\t#{expires.to_time.to_i}\tc5\tv5", 21 | ".127.0.0.1\tTRUE\t/\tFALSE\t0\tc6\tv6", 22 | ".localhost\tTRUE\t/\tFALSE\t#{expires.to_time.to_i}\tc1\tv1", 23 | ".localhost\tTRUE\t/\tFALSE\t0\tc2\tv2", 24 | ] 25 | else 26 | expected_cookielist = [ 27 | "127.0.0.1\tFALSE\t/\tFALSE\t#{expires.to_time.to_i}\tc5\tv5", 28 | "127.0.0.1\tFALSE\t/\tFALSE\t0\tc6\tv6", 29 | ".localhost\tTRUE\t/\tFALSE\t#{expires.to_time.to_i}\tc1\tv1", 30 | ".localhost\tTRUE\t/\tFALSE\t0\tc2\tv2", 31 | ] 32 | end 33 | assert_equal expected_cookielist, easy.cookielist 34 | 35 | easy.url = "#{TestServlet.url}/get_cookies" 36 | easy.perform 37 | assert_equal 'c6=v6; c5=v5; c4=v4; c3=v3', easy.body_str 38 | easy.url = "http://localhost:#{TestServlet.port}#{TestServlet.path}/get_cookies" 39 | easy.perform 40 | assert_equal 'c2=v2; c1=v1', easy.body_str 41 | end 42 | 43 | # libcurl documentation says: "This option also enables the cookie engine", but it's not tracked on the curb level 44 | def test_setopt_cookielist_enables_cookie_engine 45 | easy = Curl::Easy.new 46 | expires = (Date.today + 2).to_datetime 47 | easy.url = "http://localhost:#{TestServlet.port}#{TestServlet.path}/set_cookies" 48 | easy.setopt(Curl::CURLOPT_COOKIELIST, "Set-Cookie: c1=v1; domain=localhost; expires=#{expires.httpdate};") 49 | easy.post_body = JSON.generate([{ name: 'c2', value: 'v2', domain: 'localhost', expires: expires.httpdate, path: '/' }]) 50 | easy.perform 51 | easy.url = "http://localhost:#{TestServlet.port}#{TestServlet.path}/get_cookies" 52 | easy.post_body = nil 53 | easy.perform 54 | 55 | assert !easy.enable_cookies? 56 | assert_equal [".localhost\tTRUE\t/\tFALSE\t#{expires.to_time.to_i}\tc1\tv1", ".localhost\tTRUE\t/\tFALSE\t#{expires.to_time.to_i}\tc2\tv2"], easy.cookielist 57 | assert_equal 'c2=v2; c1=v1', easy.body_str 58 | end 59 | 60 | def test_setopt_cookielist_invalid_format 61 | easy = Curl::Easy.new 62 | easy.url = "http://localhost:#{TestServlet.port}#{TestServlet.path}/get_cookies" 63 | easy.setopt(Curl::CURLOPT_COOKIELIST, 'Not a cookie') 64 | assert_nil easy.cookielist 65 | easy.perform 66 | assert_equal '', easy.body_str 67 | end 68 | 69 | def test_setopt_cookielist_netscape_format 70 | easy = Curl::Easy.new 71 | expires = (Date.today + 2).to_datetime 72 | easy.url = "http://localhost:#{TestServlet.port}#{TestServlet.path}/get_cookies" 73 | # Note domain changes for include subdomains 74 | [ 75 | ['localhost', 'FALSE', '/', 'TRUE', 0, 'session_http_only', '42'].join("\t"), 76 | ['.localhost', 'TRUE', '/', 'FALSE', 0, 'session', '43'].join("\t"), 77 | ['localhost', 'TRUE', '/', 'FALSE', expires.to_time.to_i, 'permanent', '44'].join("\t"), 78 | ['.localhost', 'FALSE', '/', 'TRUE', expires.to_time.to_i, 'permanent_http_only', '45'].join("\t"), 79 | ].each { |cookie| easy.setopt(Curl::CURLOPT_COOKIELIST, cookie) } 80 | 81 | expected_cookielist = [ 82 | ['localhost', 'FALSE', '/', 'TRUE', 0, 'session_http_only', '42'].join("\t"), 83 | ['.localhost', 'TRUE', '/', 'FALSE', 0, 'session', '43'].join("\t"), 84 | ['.localhost', 'TRUE', '/', 'FALSE', expires.to_time.to_i, 'permanent', '44'].join("\t"), 85 | ['localhost', 'FALSE', '/', 'TRUE', expires.to_time.to_i, 'permanent_http_only', '45'].join("\t"), 86 | ] 87 | assert_equal expected_cookielist, easy.cookielist 88 | easy.perform 89 | assert_equal 'permanent_http_only=45; session_http_only=42; permanent=44; session=43', easy.body_str 90 | end 91 | 92 | # Multiple cookies and comments are not supported 93 | def test_setopt_cookielist_netscape_format_mutliline 94 | easy = Curl::Easy.new 95 | easy.url = "http://localhost:#{TestServlet.port}#{TestServlet.path}/get_cookies" 96 | easy.setopt( 97 | Curl::CURLOPT_COOKIELIST, 98 | [ 99 | '# Netscape HTTP Cookie File', 100 | ['.localhost', 'TRUE', '/', 'FALSE', 0, 'session', '42'].join("\t"), 101 | '', 102 | ].join("\n"), 103 | ) 104 | assert_nil easy.cookielist 105 | easy.perform 106 | assert_equal '', easy.body_str 107 | 108 | easy.setopt( 109 | Curl::CURLOPT_COOKIELIST, 110 | [ 111 | ['.localhost', 'TRUE', '/', 'FALSE', 0, 'session', '42'].join("\t"), 112 | ['.localhost', 'TRUE', '/', 'FALSE', 0, 'session2', '84'].join("\t"), 113 | '', 114 | ].join("\n"), 115 | ) 116 | # Only first cookie is set 117 | assert_equal [".localhost\tTRUE\t/\tFALSE\t0\tsession\t42"], easy.cookielist 118 | easy.perform 119 | assert_equal 'session=42', easy.body_str 120 | end 121 | 122 | # ALL erases all cookies held in memory 123 | # ALL was added in 7.14.1 124 | def test_setopt_cookielist_command_all 125 | expires = (Date.today + 2).to_datetime 126 | with_permanent_and_session_cookies(expires) do |easy| 127 | easy.setopt(Curl::CURLOPT_COOKIELIST, 'ALL') 128 | assert_nil easy.cookielist 129 | easy.perform 130 | assert_equal '', easy.body_str 131 | end 132 | end 133 | 134 | # SESS erases all session cookies held in memory 135 | # SESS was added in 7.15.4 136 | def test_setopt_cookielist_command_sess 137 | expires = (Date.today + 2).to_datetime 138 | with_permanent_and_session_cookies(expires) do |easy| 139 | easy.setopt(Curl::CURLOPT_COOKIELIST, 'SESS') 140 | assert_equal [".localhost\tTRUE\t/\tFALSE\t#{expires.to_time.to_i}\tpermanent\t42"], easy.cookielist 141 | easy.perform 142 | assert_equal 'permanent=42', easy.body_str 143 | end 144 | end 145 | 146 | # FLUSH writes all known cookies to the file specified by CURLOPT_COOKIEJAR 147 | # FLUSH was added in 7.17.1 148 | def test_setopt_cookielist_command_flush 149 | expires = (Date.today + 2).to_datetime 150 | with_permanent_and_session_cookies(expires) do |easy| 151 | cookiejar = File.join(Dir.tmpdir, 'curl_test_cookiejar') 152 | assert !File.exist?(cookiejar) 153 | begin 154 | easy.cookiejar = cookiejar 155 | # trick to actually set CURLOPT_COOKIEJAR 156 | easy.enable_cookies = true 157 | easy.perform 158 | assert !File.exist?(cookiejar) 159 | easy.setopt(Curl::CURLOPT_COOKIELIST, 'FLUSH') 160 | expected_cookiejar = <<~COOKIEJAR 161 | # Netscape HTTP Cookie File 162 | # https://curl.se/docs/http-cookies.html 163 | # This file was generated by libcurl! Edit at your own risk. 164 | 165 | .localhost TRUE / FALSE 0 session 420 166 | .localhost TRUE / FALSE #{expires.to_time.to_i} permanent 42 167 | COOKIEJAR 168 | assert_equal expected_cookiejar, File.read(cookiejar) 169 | ensure 170 | # Otherwise it'll create this file again 171 | easy.close 172 | File.unlink(cookiejar) if File.exist?(cookiejar) 173 | end 174 | end 175 | end 176 | 177 | # RELOAD loads all cookies from the files specified by CURLOPT_COOKIEFILE 178 | # RELOAD was added in 7.39.0 179 | def test_setopt_cookielist_command_reload 180 | expires = (Date.today + 2).to_datetime 181 | expires_file = (Date.today + 4).to_datetime 182 | with_permanent_and_session_cookies(expires) do |easy| 183 | cookiefile = File.join(Dir.tmpdir, 'curl_test_cookiefile') 184 | assert !File.exist?(cookiefile) 185 | begin 186 | cookielist = [ 187 | # Won't be updated, added instead 188 | ".localhost\tTRUE\t/\tFALSE\t#{expires_file.to_time.to_i}\tpermanent\t84", 189 | ".localhost\tTRUE\t/\tFALSE\t#{expires_file.to_time.to_i}\tpermanent_file\t84", 190 | # Won't be updated, added instead 191 | ".localhost\tTRUE\t/\tFALSE\t0\tsession\t840", 192 | ".localhost\tTRUE\t/\tFALSE\t0\tsession_file\t840", 193 | '', 194 | ] 195 | File.write(cookiefile, cookielist.join("\n")) 196 | easy.cookiefile = cookiefile 197 | # trick to actually set CURLOPT_COOKIEFILE 198 | easy.enable_cookies = true 199 | easy.perform 200 | easy.setopt(Curl::CURLOPT_COOKIELIST, 'RELOAD') 201 | expected_cookielist = [ 202 | ".localhost\tTRUE\t/\tFALSE\t#{expires.to_time.to_i}\tpermanent\t42", 203 | ".localhost\tTRUE\t/\tFALSE\t0\tsession\t420", 204 | ".localhost\tTRUE\t/\tFALSE\t#{expires_file.to_time.to_i}\tpermanent\t84", 205 | ".localhost\tTRUE\t/\tFALSE\t#{expires_file.to_time.to_i}\tpermanent_file\t84", 206 | ".localhost\tTRUE\t/\tFALSE\t0\tsession\t840", 207 | ".localhost\tTRUE\t/\tFALSE\t0\tsession_file\t840", 208 | ] 209 | assert_equal expected_cookielist, easy.cookielist 210 | easy.perform 211 | # Be careful, duplicates are not removed 212 | assert_equal 'permanent_file=84; session_file=840; permanent=84; session=840; permanent=42; session=420', easy.body_str 213 | ensure 214 | File.unlink(cookiefile) if File.exist?(cookiefile) 215 | end 216 | end 217 | end 218 | 219 | def test_commands_do_not_enable_cookie_engine 220 | %w[ALL SESS FLUSH RELOAD].each do |command| 221 | easy = Curl::Easy.new 222 | expires = (Date.today + 2).to_datetime 223 | easy.url = "http://localhost:#{TestServlet.port}#{TestServlet.path}/set_cookies" 224 | easy.setopt(Curl::CURLOPT_COOKIELIST, command) 225 | easy.post_body = JSON.generate([{ name: 'c2', value: 'v2', domain: 'localhost', expires: expires.httpdate, path: '/' }]) 226 | easy.perform 227 | easy.url = "http://localhost:#{TestServlet.port}#{TestServlet.path}/get_cookies" 228 | easy.post_body = nil 229 | easy.perform 230 | 231 | assert !easy.enable_cookies? 232 | assert_nil easy.cookielist 233 | assert_equal '', easy.body_str 234 | end 235 | end 236 | 237 | 238 | def test_strings_without_cookie_enable_cookie_engine 239 | [ 240 | '', 241 | '# Netscape HTTP Cookie File', 242 | 'no_a_cookie', 243 | ].each do |command| 244 | easy = Curl::Easy.new 245 | expires = (Date.today + 2).to_datetime 246 | easy.url = "http://localhost:#{TestServlet.port}#{TestServlet.path}/set_cookies" 247 | easy.setopt(Curl::CURLOPT_COOKIELIST, command) 248 | easy.post_body = JSON.generate([{ name: 'c2', value: 'v2', domain: 'localhost', expires: expires.httpdate, path: '/' }]) 249 | easy.perform 250 | easy.url = "http://localhost:#{TestServlet.port}#{TestServlet.path}/get_cookies" 251 | easy.post_body = nil 252 | easy.perform 253 | 254 | assert !easy.enable_cookies? 255 | assert_equal [".localhost\tTRUE\t/\tFALSE\t#{expires.to_time.to_i}\tc2\tv2"], easy.cookielist 256 | assert_equal 'c2=v2', easy.body_str 257 | end 258 | end 259 | 260 | def with_permanent_and_session_cookies(expires) 261 | easy = Curl::Easy.new 262 | easy.url = "http://localhost:#{TestServlet.port}#{TestServlet.path}/get_cookies" 263 | easy.setopt(Curl::CURLOPT_COOKIELIST, "Set-Cookie: permanent=42; domain=localhost; expires=#{expires.httpdate};") 264 | easy.setopt(Curl::CURLOPT_COOKIELIST, 'Set-Cookie: session=420; domain=localhost;') 265 | assert_equal [".localhost\tTRUE\t/\tFALSE\t#{expires.to_time.to_i}\tpermanent\t42", ".localhost\tTRUE\t/\tFALSE\t0\tsession\t420"], easy.cookielist 266 | easy.perform 267 | assert_equal 'permanent=42; session=420', easy.body_str 268 | 269 | yield easy 270 | end 271 | 272 | include TestServerMethods 273 | 274 | def setup 275 | server_setup 276 | end 277 | end 278 | -------------------------------------------------------------------------------- /tasks/docker.rake: -------------------------------------------------------------------------------- 1 | # ==== About Docker 2 | # 3 | # Docker needs to be installed and support mounting host volumes for tasks to work as expected. 4 | # 5 | # macOS and Windows may require additonal configuration that is not covered here. Refer to the 6 | # official documentation: 7 | # 8 | # * https://docs.docker.com/docker-for-mac/#file-sharing 9 | # * https://docs.docker.com/docker-for-windows/#shared-drives 10 | # 11 | # Additionally, on Linux, having enabled and configured user namespaces is recommended to maintain 12 | # file permissions. Not having user-ns will make all new files belonging to root and some tasks may 13 | # stop working outside of docker environment. 14 | # 15 | # See the oficial docummentation how to enable user-ns: 16 | # 17 | # https://docs.docker.com/engine/security/userns-remap/ 18 | # 19 | # WARNING: Enabling user namespaces will have global impact and some containers/images will not work 20 | # in this configuration. Do read the documentation carefully. 21 | # 22 | # Disabling user namespaces will restore the original docker behavior and fix any errors 23 | # related to user-ns. 24 | # 25 | # ==== Summary 26 | # 27 | # This namespace provides Rake tasks to manage development environment in docker. 28 | # 29 | # With minimal requirements it allows to run all tests across all supported rubies and provides a 30 | # real time feedback. 31 | # 32 | # Start with "docker:environment" task (can be ran multiple times) and then run "docker:test". 33 | # 34 | # Use 'docker:list_rubies' task to list supported ruby images. 35 | # 36 | # Use 'docker:test' task to run a test suite in docker. 37 | # 38 | # To run tests on different ruby set DOCKER_RUBY_IMAGE environment variable to a supported ruby. 39 | # 40 | # This has been tested in Linux with Docker v18.06, with user namespaces enabed and running in 41 | # non-privileged mode. 42 | namespace :docker do 43 | desc "Run the test suite in docker environment (#{Docker.ruby_image})" 44 | task :test => :recompile do 45 | run_in_docker(Docker.ruby_image, 'bundle', 'exec', 'rake', 'tu', { :live_stdout => STDOUT }).error! 46 | end 47 | 48 | desc "Run the test suite against all rubies" 49 | task :test_all do 50 | Docker::DOCKER_IMAGES.keys.each do |image_name| 51 | ENV['DOCKER_RUBY_IMAGE'] = image_name 52 | Rake::Task['docker:recompile'].reenable 53 | invoke_and_reenable('docker:test') 54 | end 55 | end 56 | 57 | desc 'List supported ruby images (for use as DOCKER_RUBY_IMAGE env)' 58 | task :list_rubies do 59 | puts "Supported images:" 60 | Docker::DOCKER_IMAGES.keys.each do |image_name| 61 | puts " #{image_name}" 62 | end 63 | end 64 | 65 | # This task automates the environment configuration for developing Curb in docker. 66 | # 67 | # It pulls necessary images, configures volumes, networks etc... 68 | # 69 | # It also collects the following diagnostic information: 70 | # 71 | # * docker binary client & server version (docker system info) 72 | # * docker network the containers will use (docker network inspect ) 73 | # * all named docker volumes (docker volume inspect ) 74 | # * all pulled docker images (docker image inspect ) 75 | # * ruby environment of each container (bundle env) 76 | # * curl version (curl -V) 77 | # * last 3 commits info (without diffs) 78 | # 79 | # All the collected data is stored in 'build/docker/*'. 80 | desc 'Docker docker environment for developing Curb.' 81 | task :environment => Docker::BUILD_DIRECTORIES do 82 | Rake::Task['build/docker/entrypoint_ruby1.8.sh'].invoke 83 | Rake::Task['build/docker/docker.json'].invoke 84 | Rake::Task['build/docker/network.json'].invoke 85 | Rake::Task['build/docker/git_curb_info.txt'].invoke 86 | 87 | Docker::DOCKER_IMAGES.each do |name, conf| 88 | # create all named volumes 89 | conf[:volumes].each { |vol| Rake::Task[vol[:filepath]].invoke } 90 | 91 | Rake::Task[conf[:filepath]].invoke 92 | Rake::Task[conf[:bundle_env_filepath]].invoke if conf[:bundle_env_filepath] 93 | Rake::Task[conf[:curl_filepath]].invoke if conf[:curl_filepath] 94 | end 95 | end 96 | 97 | # Tasks listed below should be considered private and not invoked directly. These tasks do not 98 | # have "desc" to make Rake not list them. 99 | # 100 | # Rake will ignore the "private" statement, it's added here to show the intention. 101 | private 102 | 103 | file 'build/docker/_compiled_to_ruby.txt' do 104 | Rake::Task['docker:recompile'].invoke 105 | end 106 | 107 | file 'build/docker/git_curb_info.txt' do 108 | Rake::Task['docker:docker_git_curb_info'].invoke 109 | end 110 | 111 | file 'build/docker/docker.json' do 112 | Rake::Task['docker:docker_binary'].invoke 113 | end 114 | 115 | file 'build/docker/network.json' do 116 | Rake::Task['docker:docker_network'].invoke 117 | end 118 | 119 | file 'build/docker/entrypoint_ruby1.8.sh' do 120 | entrypoint = 'build/docker/entrypoint_ruby1.8.sh' 121 | File.write(entrypoint, <<-ENTRYPOINT) 122 | #!/bin/bash -e 123 | 124 | # Ruby 1.8 docker image does not include libcurl development libraries 125 | apt-get update -qq || true 126 | apt-get install -y -q --no-install-recommends libcurl4-openssl-dev 127 | 128 | exec "$@" 129 | ENTRYPOINT 130 | File.chmod(0775, entrypoint) 131 | end 132 | # This section generates file tasks based on the configuration (see 'tasks/rake_helpers.rb'). 133 | # 134 | # This code is being evaluated (loaded) early in the Rakefile, before any task is executed. 135 | # By the time Rake executes a task the "file" resources defined here are available. 136 | Docker::BUILD_DIRECTORIES.each { |dir| directory(dir) } 137 | Docker::DOCKER_IMAGES.each do |full_name, conf| 138 | file conf[:filepath] do 139 | invoke_and_reenable('docker:docker_pull', full_name) 140 | end 141 | 142 | file conf[:bundle_env_filepath] do 143 | invoke_and_reenable('docker:bundle_install', full_name) 144 | end if conf[:bundle_env_filepath] 145 | 146 | (conf[:volumes] || []).each do |vol_conf| 147 | file vol_conf[:filepath] do 148 | invoke_and_reenable('docker:docker_volume', vol_conf[:name], vol_conf[:filepath]) 149 | end 150 | end 151 | 152 | file conf[:curl_filepath] do 153 | invoke_and_reenable('docker:docker_curl', full_name) 154 | end if conf[:curl_filepath] 155 | end 156 | 157 | ################# 158 | # 159 | # Tasks defined below perform actual work and should not be invoked individually. 160 | # 161 | # These tasks are implementation detail and can change at any time without warning. 162 | # 163 | # This is the heart of docker.rake 164 | ################## 165 | 166 | # This task is a direct dependency of 'docker:test'. It has 2 purposes: 167 | # 168 | # * detect if curb has been compiled against current ruby 169 | # * recompile only if needed 170 | # 171 | # To successfuly recompile we need to clobber first, otherwise the built-in 'compile' task will 172 | # do nothing because it can't detect changes to the system libraries. 173 | # 174 | # This task triggers full recompilation on ruby version change, but when only source files are 175 | # changed it does nothing. In such case the default 'compile' target will run recompilation on 176 | # changed files only. 177 | task :recompile do 178 | # Find out what ruby version are we running in docker 179 | cmd_ruby_v_docker = run_in_docker(Docker.ruby_image, 'rake', 'ruby_version') 180 | cmd_ruby_v_docker.error! 181 | ruby_v_docker = cmd_ruby_v_docker.stdout 182 | 183 | # Check what Ruby version was Curb compiled against and set it to UNKNOWN if the file doesn't 184 | # exist to trigger recompilation (we can't know if it was compiled with correct Ruby). 185 | ruby_v_filepath = 'build/docker/_compiled_to_ruby.txt' 186 | ruby_v_file = File.exists?(ruby_v_filepath) ? File.read(ruby_v_filepath) : 'UNKNOWN' 187 | 188 | if ruby_v_file != ruby_v_docker 189 | run_in_docker(Docker.ruby_image, 'rake', 'clobber').error! 190 | run_in_docker(Docker.ruby_image, 'rake', 'compile', { :live_stdout => nil }).error! 191 | end 192 | 193 | File.write(ruby_v_filepath, ruby_v_docker) 194 | end 195 | 196 | # Creates persistent named volumes for containers to store gems installed by bundler. 197 | # 198 | # Docker containers have ephermal file system and can't persist anything. Any changes to the disk 199 | # will be deleted when the container stops. As a result we'd need to do a fresh 'bundle install' 200 | # each time we want to run tests. 201 | # 202 | # To avoid 'bundle install' completely we will mount a persistent volume to '/usr/local/bundle'. 203 | task :docker_volume, [:volume_name, :filepath] do |_, args| 204 | abort('volume_name argument is required.') unless args.volume_name 205 | abort('filepath argument is required.') unless args.filepath 206 | 207 | # This should never fail, even if the volume exists docker return success, but if it does for 208 | # whatever reason we can't do anything about it so we will just explode. 209 | create_cmd = shell(['docker', 'volume', 'create', args.volume_name]) 210 | abort('failed to create docker volume') if create_cmd.error? 211 | 212 | # This could fail if the volume (for any reason) does not exist. 213 | inspect_cmd = shell(['docker', 'volume', 'inspect', args.volume_name]) 214 | inspect_cmd.error! 215 | 216 | File.write(args.filepath, inspect_cmd.stdout) 217 | end 218 | 219 | # Run bundle install on the container and dump bundle env to file. The gems will be installed to 220 | # the shared volume for later reuse. 221 | # 222 | # Each image has it's own volume and different ruby versions are completely isolated. 223 | task :bundle_install, [:name] do |_, args| 224 | config = Docker::DOCKER_IMAGES[args.name] 225 | 226 | bundle_install = run_in_docker(args.name, 'bundle', 'install') 227 | bundle_install.error! 228 | 229 | bundle_env = run_in_docker(args.name, 'bundle', 'env') 230 | bundle_env.error! 231 | 232 | File.write(config[:bundle_env_filepath], bundle_env.stdout) 233 | end 234 | 235 | # Pull a docker image from Dockerhub. 236 | # 237 | # This will pull up-to-date images, it's common otherwise to have local outdated images in the 238 | # cache. We always want to work on the most recent versions. 239 | task :docker_pull, [:name] do |_, args| 240 | conf = Docker::DOCKER_IMAGES[args.name] 241 | 242 | pull_cmd = shell(['docker', 'pull', "#{conf[:name]}:#{conf[:tag]}"]) 243 | abort('docker pull failed.') if pull_cmd.error? 244 | 245 | inspect_cmd = shell(['docker', 'image', 'inspect', "#{conf[:name]}:#{conf[:tag]}"]) 246 | inspect_cmd.error! 247 | 248 | File.write(conf[:filepath], inspect_cmd.stdout) 249 | end 250 | 251 | # Make sure the docker binary is available. 252 | task :docker_binary do 253 | cmd = shell(['docker', 'system', 'info', '--format', '{{json .}}']) 254 | if cmd.error? 255 | abort('docker binary not found on the system. Make sure docker is installed.') 256 | end 257 | 258 | File.write('build/docker/docker.json', cmd.stdout) 259 | end 260 | 261 | # Create docker network `curb`. 262 | # 263 | # Using a named network will allow us to run supporting services in docker. It's not in active 264 | # use currently and it's intended for the future. 265 | # 266 | # The intention is to allow easy testing against different configurations with little manual 267 | # setup. 268 | # 269 | # Some possible examples: 270 | # 271 | # * test with nginx or apache 272 | # * test different protocols CURL supports (ie SMTP) 273 | # * test against backend application written in any language 274 | # 275 | # With initial setup in place, and even if the feature is not used in the codebase it's handy for 276 | # local development. 277 | task :docker_network do 278 | shell(['docker', 'network', 'create', 'curb']) 279 | 280 | inspect_cmd = shell(['docker', 'network', 'inspect', 'curb']) 281 | inspect_cmd.error! 282 | 283 | File.write('build/docker/network.json', inspect_cmd.stdout) 284 | end 285 | 286 | # Make sure curl is available 287 | task :docker_curl, [:full_name] do |_, args| 288 | conf = Docker::DOCKER_IMAGES[args.full_name] 289 | 290 | curl_info = run_in_docker(args.full_name, 'curl', '-V') 291 | curl_info.error! 292 | 293 | File.write(conf[:curl_filepath], curl_info.stdout) 294 | end 295 | 296 | # Grab last 3 commits and save info to the file. 297 | # 298 | # '--no-pager' is required, otherwise git would block waiting on user input when the commit is 299 | # too long to display on one screen. Since we don't connect stdin it would hang indefinitely. 300 | task :docker_git_curb_info do 301 | git_cmd = run_in_docker(Docker.ruby_image, 'git', '--no-pager', 'log', '--graph', '-n', '3', '--no-color') 302 | git_cmd.error! 303 | 304 | File.write('build/docker/git_curb_info.txt', git_cmd.stdout) 305 | end 306 | end 307 | -------------------------------------------------------------------------------- /tasks/rake_helpers.rb: -------------------------------------------------------------------------------- 1 | module Curb 2 | module RakeHelpers 3 | 4 | # This module is being required in the top level Rakefile and it provides all-around helpers. 5 | module DSL 6 | 7 | # Invoke a Rake task by name and enable it again after completion. 8 | # 9 | # Rake tries to be smarter than Make when it comes to dependency graph resolution and it will 10 | # invoke any task only once, even if it's called multiple times with different parameters. 11 | # 12 | # It's not a bug, it's how Rake is designed to work. 13 | # 14 | # To workaround this limitation a wrapper method will invoke and then re-enable the task 15 | # afterwards. No exception handling is necessary as the Rake will just abort everything. 16 | def invoke_and_reenable(task_name, *args) 17 | rake_task = Rake::Task[task_name] 18 | rake_task.invoke(*args) 19 | rake_task.reenable # allow the task to be executed again 20 | rake_task 21 | end 22 | 23 | # Returns the current ruby major version. 24 | # 25 | # Ruby uses the following version schema: 26 | # 27 | # .. 28 | # 29 | # Ruby maintains API compatibility on level and change to means breaking 30 | # changes. 31 | # 32 | # When compiling sources we will persist the information about ruby version used and when it 33 | # changes it's a signal to recompile. 34 | def current_ruby_major 35 | RUBY_VERSION.match(/(\d).(\d).\d/)[1..2].join('_') # => 2_5 36 | end 37 | 38 | # Helper method for shelling out. 39 | # 40 | # It's easier than Rake's 'sh' and it's backed by Mixlib::Shellout if available with fallback 41 | # to pure ruby implementation compatible down to Ruby 1.8. 42 | def shell(args, mix_options = {}) 43 | puts "> #{args.join(' ')}" 44 | 45 | _shell_wrapper(args, mix_options).run_command 46 | end 47 | 48 | def run_in_docker(*args) 49 | Docker.run_in_docker(*args) 50 | end 51 | 52 | private 53 | 54 | # Mixlib won't be available on every ruby we support. This handles selecting appropriate 55 | # backed for the command. 56 | def _shell_wrapper(args, shell_options) 57 | if defined?(Mixlib::ShellOut) # we're on "modern" ruby that supports mixlib 58 | Mixlib::ShellOut.new(args, shell_options) 59 | else # ruby 1.8 60 | ShellWrapper.new(args, shell_options) 61 | end 62 | end 63 | end 64 | 65 | # This module encapsulates configuration and helper methods to interact with docker. 66 | module Docker 67 | DEFAULT_DOCKER_IMAGE = 'ruby:2.5' 68 | BUILD_DIRECTORIES = %w(build/docker) 69 | 70 | DOCKER_VOLUMES = { 71 | 'ruby:1.8' => [ { :name => 'curb_ruby_1_8', :mount_path => '/usr/local/bundle', :filepath => 'build/docker/volume_ruby_1_8.json' } ], 72 | 'ruby:2.0' => [ { :name => 'curb_ruby_2_0', :mount_path => '/usr/local/bundle', :filepath => 'build/docker/volume_ruby_2_0.json' } ], 73 | 'ruby:2.1' => [ { :name => 'curb_ruby_2_1', :mount_path => '/usr/local/bundle', :filepath => 'build/docker/volume_ruby_2_1.json' } ], 74 | 'ruby:2.2' => [ { :name => 'curb_ruby_2_2', :mount_path => '/usr/local/bundle', :filepath => 'build/docker/volume_ruby_2_2.json' } ], 75 | 'ruby:2.3' => [ { :name => 'curb_ruby_2_3', :mount_path => '/usr/local/bundle', :filepath => 'build/docker/volume_ruby_2_3.json' } ], 76 | 'ruby:2.4' => [ { :name => 'curb_ruby_2_4', :mount_path => '/usr/local/bundle', :filepath => 'build/docker/volume_ruby_2_4.json' } ], 77 | 'ruby:2.5' => [ { :name => 'curb_ruby_2_5', :mount_path => '/usr/local/bundle', :filepath => 'build/docker/volume_ruby_2_5.json' } ], 78 | 'ruby:2.6' => [ { :name => 'curb_ruby_2_6', :mount_path => '/usr/local/bundle', :filepath => 'build/docker/volume_ruby_2_6.json' } ], 79 | 'ruby:2.7' => [ { :name => 'curb_ruby_2_7', :mount_path => '/usr/local/bundle', :filepath => 'build/docker/volume_ruby_2_7.json' } ], 80 | 'ruby:3.0' => [ { :name => 'curb_ruby_3_0', :mount_path => '/usr/local/bundle', :filepath => 'build/docker/volume_ruby_3_0.json' } ], 81 | 'ruby:3.1' => [ { :name => 'curb_ruby_3_1', :mount_path => '/usr/local/bundle', :filepath => 'build/docker/volume_ruby_3_1.json' } ], 82 | 'ruby:3.2' => [ { :name => 'curb_ruby_3_2', :mount_path => '/usr/local/bundle', :filepath => 'build/docker/volume_ruby_3_2.json' } ], 83 | } 84 | 85 | DOCKER_IMAGES = { 86 | 'ruby:1.8' => { :name => 'phusion/passenger-ruby18', :tag => 'latest', 87 | :filepath => 'build/docker/image_ruby_1_8.json', 88 | :gemfile => 'Gemfile.ruby-1.8', 89 | :entrypoint => '/code/build/docker/entrypoint_ruby1.8.sh', 90 | :curl_filepath => 'build/docker/curl_ruby_1_8.txt', 91 | :bundle_env_filepath => 'build/docker/bundle_env_ruby_1_8.md', 92 | :volumes => DOCKER_VOLUMES['ruby:1.8'] }, 93 | 'ruby:2.0' => { :name => 'ruby', :tag => '2.0', 94 | :filepath => 'build/docker/image_ruby_2_0.json', 95 | :gemfile => 'Gemfile.ruby-2.1', 96 | :curl_filepath => 'build/docker/curl_ruby_2_0.txt', 97 | :bundle_env_filepath => 'build/docker/bundle_env_ruby_2_0.md', 98 | :volumes => DOCKER_VOLUMES['ruby:2.0'] }, 99 | 'ruby:2.1' => { :name => 'ruby', :tag => '2.1', 100 | :gemfile => 'Gemfile.ruby-2.1', 101 | :filepath => 'build/docker/image_ruby_2_1.json', 102 | :curl_filepath => 'build/docker/curl_ruby_2_1.txt', 103 | :bundle_env_filepath => 'build/docker/bundle_env_ruby_2_1.md', 104 | :volumes => DOCKER_VOLUMES['ruby:2.1'] }, 105 | 'ruby:2.2' => { :name => 'ruby', :tag => '2.2', 106 | :filepath => 'build/docker/image_ruby_2_2.json', 107 | :curl_filepath => 'build/docker/curl_ruby_2_2.txt', 108 | :bundle_env_filepath => 'build/docker/bundle_env_ruby_2_2.md', 109 | :volumes => DOCKER_VOLUMES['ruby:2.2'] }, 110 | 'ruby:2.3' => { :name => 'ruby', :tag => '2.3', 111 | :filepath => 'build/docker/image_ruby_2_3.json', 112 | :curl_filepath => 'build/docker/curl_ruby_2_3.txt', 113 | :bundle_env_filepath => 'build/docker/bundle_env_ruby_2_3.md', 114 | :volumes => DOCKER_VOLUMES['ruby:2.3'] }, 115 | 'ruby:2.4' => { :name => 'ruby', :tag => '2.4', 116 | :filepath => 'build/docker/image_ruby_2_4.json', 117 | :curl_filepath => 'build/docker/curl_ruby_2_4.txt', 118 | :bundle_env_filepath => 'build/docker/bundle_env_ruby_2_4.md', 119 | :volumes => DOCKER_VOLUMES['ruby:2.4'] }, 120 | 'ruby:2.5' => { :name => 'ruby', :tag => '2.5', 121 | :filepath => 'build/docker/image_ruby_2_5.json', 122 | :curl_filepath => 'build/docker/curl_ruby_2_5.txt', 123 | :bundle_env_filepath => 'build/docker/bundle_env_ruby_2_5.md', 124 | :volumes => DOCKER_VOLUMES['ruby:2.5'] }, 125 | 'ruby:2.6' => { :name => 'ruby', :tag => '2.6', 126 | :filepath => 'build/docker/image_ruby_2_6.json', 127 | :curl_filepath => 'build/docker/curl_ruby_2_6.txt', 128 | :bundle_env_filepath => 'build/docker/bundle_env_ruby_2_6.md', 129 | :volumes => DOCKER_VOLUMES['ruby:2.6'] }, 130 | 'ruby:2.7' => { :name => 'ruby', :tag => '2.7', 131 | :filepath => 'build/docker/image_ruby_2_7.json', 132 | :curl_filepath => 'build/docker/curl_ruby_2_7.txt', 133 | :bundle_env_filepath => 'build/docker/bundle_env_ruby_2_7.md', 134 | :volumes => DOCKER_VOLUMES['ruby:2.7'] }, 135 | 'ruby:3.0' => { :name => 'ruby', :tag => '3.0', 136 | :filepath => 'build/docker/image_ruby_3_0.json', 137 | :curl_filepath => 'build/docker/curl_ruby_3_0.txt', 138 | :bundle_env_filepath => 'build/docker/bundle_env_ruby_3_0.md', 139 | :volumes => DOCKER_VOLUMES['ruby:3.0'] }, 140 | 'ruby:3.1' => { :name => 'ruby', :tag => '3.1', 141 | :filepath => 'build/docker/image_ruby_3_1.json', 142 | :curl_filepath => 'build/docker/curl_ruby_3_1.txt', 143 | :bundle_env_filepath => 'build/docker/bundle_env_ruby_3_1.md', 144 | :volumes => DOCKER_VOLUMES['ruby:3.1'] }, 145 | 'ruby:3.2' => { :name => 'ruby', :tag => '3.2', 146 | :filepath => 'build/docker/image_ruby_3_2.json', 147 | :curl_filepath => 'build/docker/curl_ruby_3_2.txt', 148 | :bundle_env_filepath => 'build/docker/bundle_env_ruby_3_2.md', 149 | :volumes => DOCKER_VOLUMES['ruby:3.2'] }, 150 | } 151 | 152 | # Returns current docker image name with tag for other Rake tasks to use. 153 | # 154 | # Task 'docker:test_all' overwrites the ENV to run a test suite for every configuration, but 155 | # otherwise it's not overridden anywhere. 156 | # 157 | # 'docker:test' uses this to select appropriate image to test on. 158 | # 159 | # Example: 160 | # 161 | # DOCKER_RUBY_IMAGE=ruby:2.2 rake docker:test # run a test suite on Ruby 2.2 162 | # 163 | def self.ruby_image 164 | ENV['DOCKER_RUBY_IMAGE'] || DEFAULT_DOCKER_IMAGE 165 | end 166 | 167 | # Execute a command in docker container. 168 | # 169 | # If the last argument is a Hash it will be used as options for the Ruby shell and not to 170 | # passed to docker as part of the command. 171 | # 172 | # See Mixlib::ShellOut documentation for supported options. 173 | # See Curb::RakeHelpers::ShellWrapper (defined in 'tasks/utils.rb') for supported options. 174 | # 175 | # The command will be passed to the Mixlib library if available, otherwise (old rubies) it 176 | # will be passed through ShellWrapper. 177 | # 178 | # If this method is invoked from inside the docker container it will raise an exception. We 179 | # will not support running docker in docker. 180 | # 181 | # Given an example call: 182 | # 183 | # Curb::Docker.run_in_docker('ruby:2.5', 'rake', 'recompile') 184 | # 185 | # It's being translated to the following shell command: 186 | # 187 | # docker run --rm -t -w /code 188 | # --mount type=bind,source=$PWD,target=/code 189 | # --mount type=volume,source=curb_ruby_2_5,target=/usr/local/bundle 190 | # --network curb 191 | # ruby:2.5 rake compile 192 | # 193 | def self.run_in_docker(config_name, *cmd) 194 | conf = DOCKER_IMAGES[config_name] 195 | 196 | # The docker-in-docker detection is as simple as checking existence of '/.dockerenv' file. 197 | # '.dockerenv' and '.dockerinit' have been historically used with the LXC execution driver, 198 | # but it has been completely removed from docker in version 1.10.0 and the files are no 199 | # longer used for anything, yet remain widely used. 200 | # 201 | # Official ruby images ship with an empty '.dockerenv' file. 202 | docker_in_docker = File.exists?('/.dockerenv') 203 | abort("Can't run docker inside docker") if docker_in_docker 204 | 205 | # If last argument of cmd is a hash do not pass it to docker, but pass it to shell. 206 | shell_options = cmd.last.is_a?(Hash) ? cmd.pop : {} 207 | 208 | # Setup mount points for named volumes. 209 | # 210 | # These volumes are used to persist gems installed with 'bundle install' across runs. 211 | # 212 | # As of version 17.06 standalone containers support '--mount' flag that is almost 213 | # equivalent to '-v'. 214 | # 215 | # When '-v' flag is used to mount a host volume and the local path does not exist it will 216 | # create an empty directory and then mount it. 217 | # 218 | # When '--mount' flag is used with a non-existent path it will raise an error. 219 | # 220 | # '--mount' is more verbose than '-v', but it's more explicit, easier to understand, 221 | # and it's recommended over '-v' in the documentation: 222 | # 223 | # New users should use the --mount syntax. Experienced users may be more familiar 224 | # with the -v or --volume syntax, but are encouraged to use --mount, because research has 225 | # shown it to be easier to use. 226 | # 227 | # Source: https://docs.docker.com/storage/bind-mounts/ 228 | volumes = conf[:volumes].map do |conf| 229 | ['--mount', "type=volume,source=#{conf[:name]},target=#{conf[:mount_path]}"] 230 | end 231 | 232 | # Inject custom gemfiles over the original ones. 233 | # 234 | # Bundler supports BUNDLE_GEMFILE environment variable that points to non-standard Gemfile, 235 | # but it doesn't work reliably across rubies and compilation crashes on 2.1. 236 | # 237 | # This will +shadow+ the original Gemfile and Gemfile.lock with a custom one if defined in 238 | # the configuration. 239 | # 240 | # It will crash if custom Gemfile or Gemfile.lock does not exist. 241 | if conf[:gemfile] 242 | volumes << ['--mount', "type=bind,source=#{Dir.pwd}/#{conf[:gemfile]},target=/code/Gemfile"] 243 | volumes << ['--mount', "type=bind,source=#{Dir.pwd}/#{conf[:gemfile]}.lock,target=/code/Gemfile.lock"] 244 | end 245 | 246 | # Mount project directory to '/code'. Any changes made the local files will be reflected 247 | # in the container. 248 | # 249 | # It allows 'rake docker:test' to see the changes instantly and enables real-time feedback 250 | # loop for developer's convenience. 251 | volumes << ['--mount', "type=bind,source=#{Dir.pwd},target=/code"] 252 | 253 | # The docker entrypoint will be executed instead of command and the command will be passed 254 | # as arguments. This allows us to hook into the container and install/reconfigure 255 | # dependencies as needed, etc. 256 | entrypoint = [ '--entrypoint', conf[:entrypoint]] if conf[:entrypoint] 257 | 258 | image_name = "#{conf[:name]}:#{conf[:tag]}" 259 | 260 | args = [ 'docker', 261 | 'run', 262 | '--rm', 263 | '-t', # make docker color the output :) 264 | '-w', '/code', 265 | '--network', 'curb', 266 | volumes, 267 | entrypoint, 268 | image_name, 269 | cmd]. 270 | flatten. # volumes 271 | compact # entrypoint 272 | 273 | shell(args, shell_options) 274 | end 275 | end 276 | end 277 | end 278 | 279 | # Because the file is loaded rather than required this will be evaluated by Ruby 280 | # and the DSL will be extended automatically. 281 | include Curb::RakeHelpers::DSL 282 | 283 | # Handy shortcut 284 | abort("FATAL: Docker constant already loaded.") if defined?(Docker) 285 | Docker = Curb::RakeHelpers::Docker 286 | --------------------------------------------------------------------------------