├── .gitignore ├── Gemfile ├── Gemfile.lock ├── README.md ├── Rakefile ├── anvil-cli.gemspec ├── bin └── anvil └── lib ├── anvil.rb └── anvil ├── builder.rb ├── cli.rb ├── engine.rb ├── helpers.rb ├── manifest.rb ├── okjson.rb └── version.rb /.gitignore: -------------------------------------------------------------------------------- 1 | pkg 2 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | gemspec 4 | 5 | group :development do 6 | gem "rake" 7 | end 8 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | PATH 2 | remote: . 3 | specs: 4 | anvil-cli (0.16.1) 5 | progress (~> 2.4.0) 6 | rest-client (~> 1.6.7) 7 | thor (~> 0.15.2) 8 | 9 | GEM 10 | remote: https://rubygems.org/ 11 | specs: 12 | mime-types (1.21) 13 | progress (2.4.0) 14 | rake (0.9.6) 15 | rest-client (1.6.7) 16 | mime-types (>= 1.16) 17 | thor (0.15.4) 18 | 19 | PLATFORMS 20 | ruby 21 | 22 | DEPENDENCIES 23 | anvil-cli! 24 | rake 25 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # anvil 2 | 3 | Builds as a service. 4 | 5 | ## Installation 6 | 7 | $ gem install anvil-cli 8 | 9 | ## Usage 10 | 11 | #### Build an application from a local directory 12 | 13 | $ anvil build . 14 | Building ... 15 | Success, slug is https://api.anvilworks.org/slugs/000.tgz 16 | 17 | #### Build from a public-accessible git repository 18 | 19 | $ anvil build https://github.com/ddollar/anvil.git 20 | 21 | #### Specify a buildpack 22 | 23 | # specify a buildpack url 24 | $ anvil build https://github.com/ddollar/anvil.git -b https://github.com/heroku/heroku-buildpack-nodejs.git 25 | 26 | # specify a buildpack from https://buildkits.heroku.com/ 27 | $ anvil build https://github.com/ddollar/anvil.git -b heroku/nodejs 28 | 29 | #### Iterate on buildpacks without pushing to Github 30 | 31 | # test a known app against the local buildpack code 32 | $ anvil build https://github.com/me/mybuildpack-testapp -b ~/mybuildpack 33 | 34 | # can also use a local app 35 | $ anvil build ~/mybuildpack/test/app -b ~/mybuildpack 36 | 37 | #### Build using a shell script from a URL 38 | 39 | You can use this combination to host build scripts in gists. [Example](https://gist.github.com/ddollar/a2ceb7b9699f05303170) 40 | 41 | $ anvil build \ 42 | http://downloads.sourceforge.net/project/squashfs/squashfs/squashfs4.2/squashfs4.2.tar.gz \ 43 | -b https://gist.github.com/ddollar/a2ceb7b9699f05303170/raw/build-squashfs.sh 44 | 45 | #### Use a gist as a buildpack 46 | 47 | # build mercurial 48 | $ anvil build http://mercurial.selenic.com/release/mercurial-2.7.1.tar.gz -b https://gist.github.com/ddollar/07d579a6621b3ddd7b6b/raw/gistfile1.txt 49 | 50 | #### Use the pipelining feature to build complex deploy workflows 51 | 52 | This example requires the [heroku-anvil](https://github.com/ddollar/heroku-anvil) plugin. 53 | 54 | #!/usr/bin/env bash 55 | 56 | # fail fast 57 | set -o errexit 58 | set -o pipefail 59 | 60 | # build a slug of the app 61 | slug=$(anvil build https://github.com/my/project.git -p) 62 | 63 | # release the slug to staging 64 | heroku release $slug -a myapp-staging 65 | 66 | # run tests using `heroku run` 67 | heroku run bin/tests -a myapp-staging 68 | 69 | # test that the app responds via http 70 | curl https://myapp-staging.herokuapp.com/test 71 | 72 | # release to production 73 | heroku release $slug -a myapp-production 74 | 75 | ## Advanced Usage 76 | 77 | #### anvil build 78 | 79 | Usage: anvil build [SOURCE] 80 | 81 | build software on an anvil build server 82 | 83 | if SOURCE is a local directory, the contents of the directory will be built 84 | if SOURCE is a git URL, the contents of the repo will be built 85 | if SOURCE is a tarball URL, the contents of the tarball will be built 86 | 87 | SOURCE will default to "." 88 | 89 | -b, --buildpack URL # use a custom buildpack 90 | -p, --pipeline # pipe build output to stderr and only put the slug url on stdout 91 | -r, --release # release the slug to an app 92 | 93 | ## License 94 | 95 | MIT 96 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | def base_files 2 | Dir[File.expand_path("../{bin,data,lib}/**/*", __FILE__)].select do |file| 3 | File.file?(file) 4 | end 5 | end 6 | 7 | def pkg(filename) 8 | FileUtils.mkdir_p("pkg") 9 | File.expand_path("../pkg/#{filename}", __FILE__) 10 | end 11 | 12 | def version 13 | require "anvil/version" 14 | Anvil::VERSION 15 | end 16 | 17 | file pkg("anvil-cli-#{version}.gem") => base_files do |t| 18 | sh "gem build anvil-cli.gemspec" 19 | sh "mv anvil-cli-#{version}.gem #{t.name}" 20 | end 21 | 22 | task "gem:build" => pkg("anvil-cli-#{version}.gem") 23 | 24 | task "gem:clean" do 25 | clean pkg("anvil-cli-#{version}.gem") 26 | end 27 | 28 | task "gem:release" => "gem:build" do |t| 29 | sh "gem push #{pkg("anvil-cli-#{version}.gem")}" 30 | sh "git tag v#{version}" 31 | sh "git push origin master --tags" 32 | end 33 | -------------------------------------------------------------------------------- /anvil-cli.gemspec: -------------------------------------------------------------------------------- 1 | $:.unshift File.expand_path("../lib", __FILE__) 2 | require "anvil/version" 3 | 4 | Gem::Specification.new do |gem| 5 | gem.name = "anvil-cli" 6 | gem.version = Anvil::VERSION 7 | 8 | gem.author = "David Dollar" 9 | gem.email = "david@dollar.io" 10 | gem.homepage = "http://github.com/ddollar/anvil-cli" 11 | gem.summary = "Alternate Heroku build workflow" 12 | 13 | gem.description = gem.summary 14 | 15 | gem.executables = "anvil" 16 | gem.files = Dir["**/*"].select { |d| d =~ %r{^(README|bin/|data/|ext/|lib/|spec/|test/)} } 17 | 18 | gem.add_dependency "progress", "~> 2.4.0" 19 | gem.add_dependency "rest-client", "~> 1.6.7" 20 | gem.add_dependency "thor", "~> 0.15.2" 21 | end 22 | -------------------------------------------------------------------------------- /bin/anvil: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | $:.unshift File.expand_path("../../lib", __FILE__) 4 | 5 | require "anvil/cli" 6 | 7 | Anvil::CLI.start 8 | -------------------------------------------------------------------------------- /lib/anvil.rb: -------------------------------------------------------------------------------- 1 | require "anvil/version" 2 | 3 | module Anvil 4 | 5 | def self.agent 6 | @@agent ||= "anvil-cli/#{Anvil::VERSION}" 7 | end 8 | 9 | def self.append_agent(str) 10 | @@agent = self.agent + " " + str 11 | end 12 | 13 | def self.headers 14 | @headers ||= {} 15 | end 16 | 17 | end 18 | -------------------------------------------------------------------------------- /lib/anvil/builder.rb: -------------------------------------------------------------------------------- 1 | require "anvil" 2 | require "anvil/helpers" 3 | require "net/http" 4 | require "net/https" 5 | require "rest_client" 6 | 7 | class Anvil::Builder 8 | 9 | include Anvil::Helpers 10 | 11 | class BuildError < StandardError; end 12 | 13 | attr_reader :source 14 | 15 | def initialize(source) 16 | @source = source 17 | end 18 | 19 | def build(options={}) 20 | uri = URI.parse("#{anvil_host}/build") 21 | 22 | if uri.scheme == "https" 23 | proxy = https_proxy 24 | else 25 | proxy = http_proxy 26 | end 27 | 28 | if proxy 29 | proxy_uri = URI.parse(proxy) 30 | http = Net::HTTP.new(uri.host, uri.port, proxy_uri.host, proxy_uri.port, proxy_uri.user, proxy_uri.password) 31 | else 32 | http = Net::HTTP.new(uri.host, uri.port) 33 | end 34 | 35 | if uri.scheme == "https" 36 | http.use_ssl = true 37 | http.verify_mode = OpenSSL::SSL::VERIFY_NONE 38 | end 39 | 40 | req = Net::HTTP::Post.new uri.request_uri 41 | 42 | req.initialize_http_header "User-Agent" => Anvil.agent 43 | 44 | Anvil.headers.each do |name, val| 45 | next if name.to_s.strip == "" 46 | req.initialize_http_header name => val.to_s 47 | end 48 | 49 | req.set_form_data({ 50 | "buildpack" => options[:buildpack], 51 | "cache" => options[:cache], 52 | "env" => json_encode(options[:env] || {}), 53 | "source" => source, 54 | "type" => options[:type] 55 | }) 56 | 57 | slug_url = nil 58 | 59 | http.request(req) do |res| 60 | slug_url = res["x-slug-url"] 61 | 62 | begin 63 | res.read_body do |chunk| 64 | yield chunk 65 | end 66 | rescue EOFError 67 | puts 68 | raise BuildError, "terminated unexpectedly" 69 | end 70 | 71 | manifest_id = [res["x-manifest-id"]].flatten.first 72 | code = Integer(String.new(anvil["/exit/#{manifest_id}"].get.to_s)) 73 | raise BuildError, "exited #{code}" unless code.zero? 74 | end 75 | 76 | slug_url 77 | end 78 | 79 | private 80 | 81 | def anvil 82 | @anvil ||= RestClient::Resource.new(anvil_host, :headers => anvil_headers) 83 | end 84 | 85 | def anvil_headers 86 | { "User-Agent" => Anvil.agent } 87 | end 88 | 89 | def anvil_host 90 | ENV["ANVIL_HOST"] || "https://api.anvilworks.org" 91 | end 92 | 93 | def http_proxy 94 | proxy = ENV['HTTP_PROXY'] || ENV['http_proxy'] 95 | if proxy && !proxy.empty? 96 | unless /^[^:]+:\/\// =~ proxy 97 | proxy = "http://" + proxy 98 | end 99 | proxy 100 | else 101 | nil 102 | end 103 | end 104 | 105 | def https_proxy 106 | proxy = ENV['HTTPS_PROXY'] || ENV['https_proxy'] || ENV["HTTP_PROXY"] || ENV["http_proxy"] 107 | if proxy && !proxy.empty? 108 | unless /^[^:]+:\/\// =~ proxy 109 | proxy = "https://" + proxy 110 | end 111 | proxy 112 | else 113 | nil 114 | end 115 | end 116 | 117 | end 118 | -------------------------------------------------------------------------------- /lib/anvil/cli.rb: -------------------------------------------------------------------------------- 1 | require "anvil" 2 | require "anvil/builder" 3 | require "anvil/engine" 4 | require "anvil/manifest" 5 | require "anvil/version" 6 | require "progress" 7 | require "thor" 8 | require "uri" 9 | 10 | class Anvil::CLI < Thor 11 | 12 | map ["-v", "--version"] => :version 13 | 14 | desc "build [SOURCE]", "Build an application" 15 | 16 | method_option :buildpack, :type => :string, :aliases => "-b", :desc => "Use a specific buildpack" 17 | method_option :pipeline, :type => :boolean, :aliases => "-p", :desc => "Pipe build output to stderr and put the slug url on stdout" 18 | method_option :type, :type => :string, :aliases => "-t", :desc => "Build a specific slug type (tgz, deb)" 19 | 20 | def build(source=nil) 21 | Anvil::Engine.build(source, options) 22 | rescue Anvil::Builder::BuildError => ex 23 | error "Build Error: #{ex.message}" 24 | end 25 | 26 | desc "version", "Display Anvil version" 27 | 28 | def version 29 | Anvil::Engine.version 30 | end 31 | 32 | end 33 | -------------------------------------------------------------------------------- /lib/anvil/engine.rb: -------------------------------------------------------------------------------- 1 | require "anvil" 2 | require "anvil/builder" 3 | require "anvil/helpers" 4 | require "anvil/manifest" 5 | require "anvil/version" 6 | require "progress" 7 | require "thread" 8 | require "uri" 9 | 10 | class Anvil::Engine 11 | 12 | extend Anvil::Helpers 13 | 14 | def self.build(source, options={}) 15 | if options[:pipeline] 16 | old_stdout = $stdout.dup 17 | $stdout = $stderr 18 | end 19 | 20 | source ||= "." 21 | 22 | buildpack = options[:buildpack] || read_anvil_metadata(source, "buildpack") 23 | 24 | build_options = { 25 | :buildpack => prepare_buildpack(buildpack, options), 26 | :type => options[:type] || "tgz" 27 | } 28 | 29 | builder = if is_url?(source) 30 | Anvil::Builder.new(source) 31 | else 32 | manifest = Anvil::Manifest.new(File.expand_path(source), 33 | :cache => read_anvil_metadata(source, "cache"), 34 | :ignore => options[:ignore]) 35 | upload_missing manifest 36 | 37 | manifest 38 | end 39 | 40 | slug_url = builder.build(build_options) do |chunk| 41 | print chunk 42 | end 43 | 44 | unless is_url?(source) 45 | write_anvil_metadata source, "buildpack", buildpack 46 | write_anvil_metadata source, "cache", manifest.cache_url 47 | end 48 | 49 | old_stdout.puts slug_url if options[:pipeline] 50 | 51 | slug_url 52 | end 53 | 54 | def self.version 55 | puts Anvil::VERSION 56 | end 57 | 58 | def self.prepare_buildpack(buildpack, options = {}) 59 | buildpack = buildpack.to_s 60 | if buildpack == "" 61 | buildpack 62 | elsif is_url?(buildpack) 63 | buildpack 64 | elsif buildpack =~ /\A\w+\/\w+\Z/ 65 | "https://s3-external-1.amazonaws.com/codon-buildpacks/buildpacks/#{buildpack}.tgz" 66 | elsif File.exists?(buildpack) && File.directory?(buildpack) 67 | manifest = Anvil::Manifest.new(buildpack, :ignore => options[:ignore]) 68 | upload_missing manifest, "buildpack" 69 | manifest.save 70 | else 71 | raise Anvil::Builder::BuildError.new("unrecognized buildpack specification: #{buildpack}") 72 | end 73 | end 74 | 75 | def self.upload_missing(manifest, title="app") 76 | print "Checking for #{title} files to sync... " 77 | missing = manifest.missing 78 | puts "done, #{missing.length} files needed" 79 | 80 | return if missing.length.zero? 81 | 82 | queue = Queue.new 83 | total_size = missing.map { |hash, file| file["size"].to_i }.inject(&:+) 84 | 85 | display = Thread.new do 86 | Progress.start "Uploading", total_size 87 | while (msg = queue.pop).first != :done 88 | case msg.first 89 | when :step then Progress.step msg.last.to_i 90 | end 91 | end 92 | Progress.stop 93 | end 94 | 95 | if missing.length > 0 96 | manifest.upload(missing.keys) do |file| 97 | queue << [:step, file["size"].to_i] 98 | end 99 | queue << [:done, nil] 100 | end 101 | 102 | display.join 103 | end 104 | 105 | end 106 | -------------------------------------------------------------------------------- /lib/anvil/helpers.rb: -------------------------------------------------------------------------------- 1 | require "anvil" 2 | require "anvil/okjson" 3 | 4 | module Anvil::Helpers 5 | 6 | def json_encode(obj) 7 | Anvil::OkJson.encode(obj) 8 | end 9 | 10 | def json_decode(str) 11 | Anvil::OkJson.decode(str) 12 | end 13 | 14 | def anvil_metadata_dir(root) 15 | dir = File.join(root, ".anvil") 16 | FileUtils.mkdir_p(dir) 17 | dir 18 | end 19 | 20 | def is_url?(string) 21 | URI.parse(string).scheme rescue nil 22 | end 23 | 24 | def read_anvil_metadata(root, name) 25 | return nil if is_url?(root) 26 | File.open(File.join(anvil_metadata_dir(root), name)).read.chomp rescue nil 27 | end 28 | 29 | def write_anvil_metadata(root, name, data) 30 | return if is_url?(root) 31 | File.open(File.join(anvil_metadata_dir(root), name), "w") do |file| 32 | file.puts data 33 | end 34 | end 35 | 36 | end 37 | -------------------------------------------------------------------------------- /lib/anvil/manifest.rb: -------------------------------------------------------------------------------- 1 | require "anvil/builder" 2 | require "anvil/helpers" 3 | require "net/http" 4 | require "net/https" 5 | require "pathname" 6 | require "rest_client" 7 | require "find" 8 | 9 | class Anvil::Manifest 10 | 11 | include Anvil::Helpers 12 | 13 | PUSH_THREAD_COUNT = 40 14 | 15 | attr_reader :cache_url 16 | attr_reader :dir 17 | attr_reader :manifest 18 | 19 | def initialize(dir=nil, options={}) 20 | @dir = dir 21 | @ignore = options[:ignore] || [] 22 | @manifest = @dir ? directory_manifest(@dir, :ignore => @ignore) : {} 23 | @cache_url = options[:cache] 24 | end 25 | 26 | def build(options={}) 27 | uri = URI.parse("#{anvil_host}/manifest/build") 28 | 29 | if uri.scheme == "https" 30 | proxy = https_proxy 31 | else 32 | proxy = http_proxy 33 | end 34 | 35 | if proxy 36 | proxy_uri = URI.parse(proxy) 37 | http = Net::HTTP.new(uri.host, uri.port, proxy_uri.host, proxy_uri.port, proxy_uri.user, proxy_uri.password) 38 | else 39 | http = Net::HTTP.new(uri.host, uri.port) 40 | end 41 | 42 | if uri.scheme == "https" 43 | http.use_ssl = true 44 | http.verify_mode = OpenSSL::SSL::VERIFY_NONE 45 | end 46 | 47 | if uri.scheme == "https" 48 | http.use_ssl = true 49 | http.verify_mode = OpenSSL::SSL::VERIFY_NONE 50 | end 51 | 52 | req = Net::HTTP::Post.new uri.request_uri 53 | 54 | env = options[:env] || {} 55 | 56 | req.initialize_http_header "User-Agent" => Anvil.agent 57 | req["User-Agent"] = Anvil.agent 58 | 59 | Anvil.headers.each do |name, val| 60 | next if name.to_s.strip == "" 61 | req[name] = val.to_s 62 | end 63 | 64 | req.set_form_data({ 65 | "buildpack" => options[:buildpack], 66 | "cache" => @cache_url, 67 | "env" => json_encode(options[:env] || {}), 68 | "keepalive" => "1", 69 | "manifest" => self.to_json, 70 | "type" => options[:type] 71 | }) 72 | 73 | slug_url = nil 74 | 75 | http.request(req) do |res| 76 | slug_url = res["x-slug-url"] 77 | @cache_url = res["x-cache-url"] 78 | 79 | begin 80 | res.read_body do |chunk| 81 | yield chunk.gsub("\000\000\000", "") 82 | end 83 | rescue EOFError 84 | puts 85 | raise Anvil::Builder::BuildError, "terminated unexpectedly" 86 | end 87 | 88 | code = if res["x-exit-code"].nil? 89 | manifest_id = Array(res["x-manifest-id"]).first 90 | Integer(String.new(anvil["/exit/#{manifest_id}"].get.to_s)) 91 | else 92 | res["x-exit-code"].first.to_i 93 | end 94 | 95 | raise Anvil::Builder::BuildError, "exited #{code}" unless code.zero? 96 | end 97 | 98 | slug_url 99 | end 100 | 101 | def save 102 | res = anvil["/manifest"].post(:manifest => self.to_json) 103 | res.headers[:location] 104 | end 105 | 106 | def manifest_by_hash(manifest) 107 | manifest.inject({}) do |ax, (name, file)| 108 | ax.update file["hash"] => file.merge("name" => name) 109 | end 110 | end 111 | 112 | def missing 113 | mbh = manifest_by_hash(@manifest) 114 | json_decode(anvil["/manifest/diff"].post(:manifest => self.to_json).to_s).inject({}) do |ax, hash| 115 | ax.update hash => mbh[hash] 116 | end 117 | end 118 | 119 | def upload(missing, &blk) 120 | upload_hashes missing, &blk 121 | missing.length 122 | end 123 | 124 | def to_json 125 | json_encode(@manifest) 126 | end 127 | 128 | def add(filename) 129 | @manifest[filename] = file_manifest(filename) 130 | end 131 | 132 | private 133 | 134 | def anvil 135 | @anvil ||= RestClient::Resource.new(anvil_host, :headers => anvil_headers) 136 | end 137 | 138 | def anvil_headers 139 | { "User-Agent" => Anvil.agent } 140 | end 141 | 142 | def anvil_host 143 | ENV["ANVIL_HOST"] || "https://api.anvilworks.org" 144 | end 145 | 146 | def directory_manifest(dir, options={}) 147 | root = Pathname.new(dir) 148 | ignore = (options[:ignore] || []) + [".anvil", ".git"] 149 | 150 | if File.exists?("#{dir}/.slugignore") 151 | File.read("#{dir}/.slugignore").split("\n").each do |match| 152 | Dir["#{dir}/**/#{match}"].each do |ignored_file| 153 | ignore.push Pathname.new(ignored_file).relative_path_from(root).to_s 154 | end 155 | end 156 | end 157 | 158 | manifest = {} 159 | Find.find(dir) do |path| 160 | relative = Pathname.new(path).relative_path_from(root).to_s 161 | if !File.symlink?(path) && File.directory?(path) 162 | Find.prune if ignore.include?(relative) || ignore.include?(relative + "/") 163 | next 164 | end 165 | next if ignore.include?(relative) 166 | next if %w( . .. ).include?(File.basename(path)) 167 | next if File.pipe?(path) 168 | next if path =~ /\.swp$/ 169 | next unless path =~ /^[A-Za-z0-9\-\_\.\/]*$/ 170 | manifest[relative] = file_manifest(path) 171 | end 172 | manifest 173 | end 174 | 175 | def file_manifest(file) 176 | stat = File.stat(file) 177 | manifest = { 178 | "mtime" => stat.mtime.to_i, 179 | "mode" => "%o" % stat.mode, 180 | "size" => stat.size.to_s 181 | } 182 | if File.symlink?(file) 183 | manifest["link"] = File.readlink(file) 184 | else 185 | manifest["hash"] = calculate_hash(file) 186 | end 187 | manifest 188 | end 189 | 190 | def calculate_hash(filename) 191 | Digest::SHA2.hexdigest(File.open(filename, "rb").read) 192 | end 193 | 194 | def upload_file(filename, hash=nil) 195 | hash ||= calculate_hash(filename) 196 | anvil["/file/#{hash}"].post :data => File.new(filename, "rb") 197 | hash 198 | rescue RestClient::Forbidden => ex 199 | raise "error uploading #{filename}: #{ex.http_body}" 200 | end 201 | 202 | def upload_hashes(hashes, &blk) 203 | mbh = manifest_by_hash(@manifest) 204 | filenames_by_hash = @manifest.inject({}) do |ax, (name, file_manifest)| 205 | ax.update file_manifest["hash"] => File.join(@dir.to_s, name) 206 | end 207 | bucket_hashes = hashes.inject({}) do |ax, hash| 208 | index = hash.hash % PUSH_THREAD_COUNT 209 | ax[index] ||= [] 210 | ax[index] << hash 211 | ax 212 | end 213 | threads = bucket_hashes.values.map do |hashes| 214 | Thread.new do 215 | hashes.each do |hash| 216 | upload_file filenames_by_hash[hash], hash 217 | blk.call mbh[hash] 218 | end 219 | end 220 | end 221 | threads.each(&:join) 222 | end 223 | 224 | def http_proxy 225 | proxy = ENV['HTTP_PROXY'] || ENV['http_proxy'] 226 | if proxy && !proxy.empty? 227 | unless /^[^:]+:\/\// =~ proxy 228 | proxy = "http://" + proxy 229 | end 230 | proxy 231 | else 232 | nil 233 | end 234 | end 235 | 236 | def https_proxy 237 | proxy = ENV['HTTPS_PROXY'] || ENV['https_proxy'] || ENV["HTTP_PROXY"] || ENV["http_proxy"] 238 | if proxy && !proxy.empty? 239 | unless /^[^:]+:\/\// =~ proxy 240 | proxy = "https://" + proxy 241 | end 242 | proxy 243 | else 244 | nil 245 | end 246 | end 247 | 248 | end 249 | -------------------------------------------------------------------------------- /lib/anvil/okjson.rb: -------------------------------------------------------------------------------- 1 | # encoding: UTF-8 2 | # 3 | # Copyright 2011, 2012 Keith Rarick 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in 13 | # all copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | # THE SOFTWARE. 22 | 23 | # See https://github.com/kr/okjson for updates. 24 | 25 | require 'stringio' 26 | 27 | # Some parts adapted from 28 | # http://golang.org/src/pkg/json/decode.go and 29 | # http://golang.org/src/pkg/utf8/utf8.go 30 | module Anvil 31 | module OkJson 32 | extend self 33 | 34 | 35 | # Decodes a json document in string s and 36 | # returns the corresponding ruby value. 37 | # String s must be valid UTF-8. If you have 38 | # a string in some other encoding, convert 39 | # it first. 40 | # 41 | # String values in the resulting structure 42 | # will be UTF-8. 43 | def decode(s) 44 | ts = lex(s) 45 | v, ts = textparse(ts) 46 | if ts.length > 0 47 | raise Error, 'trailing garbage' 48 | end 49 | v 50 | end 51 | 52 | 53 | # Parses a "json text" in the sense of RFC 4627. 54 | # Returns the parsed value and any trailing tokens. 55 | # Note: this is almost the same as valparse, 56 | # except that it does not accept atomic values. 57 | def textparse(ts) 58 | if ts.length < 0 59 | raise Error, 'empty' 60 | end 61 | 62 | typ, _, val = ts[0] 63 | case typ 64 | when '{' then objparse(ts) 65 | when '[' then arrparse(ts) 66 | else 67 | raise Error, "unexpected #{val.inspect}" 68 | end 69 | end 70 | 71 | 72 | # Parses a "value" in the sense of RFC 4627. 73 | # Returns the parsed value and any trailing tokens. 74 | def valparse(ts) 75 | if ts.length < 0 76 | raise Error, 'empty' 77 | end 78 | 79 | typ, _, val = ts[0] 80 | case typ 81 | when '{' then objparse(ts) 82 | when '[' then arrparse(ts) 83 | when :val,:str then [val, ts[1..-1]] 84 | else 85 | raise Error, "unexpected #{val.inspect}" 86 | end 87 | end 88 | 89 | 90 | # Parses an "object" in the sense of RFC 4627. 91 | # Returns the parsed value and any trailing tokens. 92 | def objparse(ts) 93 | ts = eat('{', ts) 94 | obj = {} 95 | 96 | if ts[0][0] == '}' 97 | return obj, ts[1..-1] 98 | end 99 | 100 | k, v, ts = pairparse(ts) 101 | obj[k] = v 102 | 103 | if ts[0][0] == '}' 104 | return obj, ts[1..-1] 105 | end 106 | 107 | loop do 108 | ts = eat(',', ts) 109 | 110 | k, v, ts = pairparse(ts) 111 | obj[k] = v 112 | 113 | if ts[0][0] == '}' 114 | return obj, ts[1..-1] 115 | end 116 | end 117 | end 118 | 119 | 120 | # Parses a "member" in the sense of RFC 4627. 121 | # Returns the parsed values and any trailing tokens. 122 | def pairparse(ts) 123 | (typ, _, k), ts = ts[0], ts[1..-1] 124 | if typ != :str 125 | raise Error, "unexpected #{k.inspect}" 126 | end 127 | ts = eat(':', ts) 128 | v, ts = valparse(ts) 129 | [k, v, ts] 130 | end 131 | 132 | 133 | # Parses an "array" in the sense of RFC 4627. 134 | # Returns the parsed value and any trailing tokens. 135 | def arrparse(ts) 136 | ts = eat('[', ts) 137 | arr = [] 138 | 139 | if ts[0][0] == ']' 140 | return arr, ts[1..-1] 141 | end 142 | 143 | v, ts = valparse(ts) 144 | arr << v 145 | 146 | if ts[0][0] == ']' 147 | return arr, ts[1..-1] 148 | end 149 | 150 | loop do 151 | ts = eat(',', ts) 152 | 153 | v, ts = valparse(ts) 154 | arr << v 155 | 156 | if ts[0][0] == ']' 157 | return arr, ts[1..-1] 158 | end 159 | end 160 | end 161 | 162 | 163 | def eat(typ, ts) 164 | if ts[0][0] != typ 165 | raise Error, "expected #{typ} (got #{ts[0].inspect})" 166 | end 167 | ts[1..-1] 168 | end 169 | 170 | 171 | # Scans s and returns a list of json tokens, 172 | # excluding white space (as defined in RFC 4627). 173 | def lex(s) 174 | ts = [] 175 | while s.length > 0 176 | typ, lexeme, val = tok(s) 177 | if typ == nil 178 | raise Error, "invalid character at #{s[0,10].inspect}" 179 | end 180 | if typ != :space 181 | ts << [typ, lexeme, val] 182 | end 183 | s = s[lexeme.length..-1] 184 | end 185 | ts 186 | end 187 | 188 | 189 | # Scans the first token in s and 190 | # returns a 3-element list, or nil 191 | # if s does not begin with a valid token. 192 | # 193 | # The first list element is one of 194 | # '{', '}', ':', ',', '[', ']', 195 | # :val, :str, and :space. 196 | # 197 | # The second element is the lexeme. 198 | # 199 | # The third element is the value of the 200 | # token for :val and :str, otherwise 201 | # it is the lexeme. 202 | def tok(s) 203 | case s[0] 204 | when ?{ then ['{', s[0,1], s[0,1]] 205 | when ?} then ['}', s[0,1], s[0,1]] 206 | when ?: then [':', s[0,1], s[0,1]] 207 | when ?, then [',', s[0,1], s[0,1]] 208 | when ?[ then ['[', s[0,1], s[0,1]] 209 | when ?] then [']', s[0,1], s[0,1]] 210 | when ?n then nulltok(s) 211 | when ?t then truetok(s) 212 | when ?f then falsetok(s) 213 | when ?" then strtok(s) 214 | when Spc then [:space, s[0,1], s[0,1]] 215 | when ?\t then [:space, s[0,1], s[0,1]] 216 | when ?\n then [:space, s[0,1], s[0,1]] 217 | when ?\r then [:space, s[0,1], s[0,1]] 218 | else numtok(s) 219 | end 220 | end 221 | 222 | 223 | def nulltok(s); s[0,4] == 'null' ? [:val, 'null', nil] : [] end 224 | def truetok(s); s[0,4] == 'true' ? [:val, 'true', true] : [] end 225 | def falsetok(s); s[0,5] == 'false' ? [:val, 'false', false] : [] end 226 | 227 | 228 | def numtok(s) 229 | m = /-?([1-9][0-9]+|[0-9])([.][0-9]+)?([eE][+-]?[0-9]+)?/.match(s) 230 | if m && m.begin(0) == 0 231 | if m[3] && !m[2] 232 | [:val, m[0], Integer(m[1])*(10**Integer(m[3][1..-1]))] 233 | elsif m[2] 234 | [:val, m[0], Float(m[0])] 235 | else 236 | [:val, m[0], Integer(m[0])] 237 | end 238 | else 239 | [] 240 | end 241 | end 242 | 243 | 244 | def strtok(s) 245 | m = /"([^"\\]|\\["\/\\bfnrt]|\\u[0-9a-fA-F]{4})*"/.match(s) 246 | if ! m 247 | raise Error, "invalid string literal at #{abbrev(s)}" 248 | end 249 | [:str, m[0], unquote(m[0])] 250 | end 251 | 252 | 253 | def abbrev(s) 254 | t = s[0,10] 255 | p = t['`'] 256 | t = t[0,p] if p 257 | t = t + '...' if t.length < s.length 258 | '`' + t + '`' 259 | end 260 | 261 | 262 | # Converts a quoted json string literal q into a UTF-8-encoded string. 263 | # The rules are different than for Ruby, so we cannot use eval. 264 | # Unquote will raise an error if q contains control characters. 265 | def unquote(q) 266 | q = q[1...-1] 267 | a = q.dup # allocate a big enough string 268 | rubydoesenc = false 269 | # In ruby >= 1.9, a[w] is a codepoint, not a byte. 270 | if a.class.method_defined?(:force_encoding) 271 | a.force_encoding('UTF-8') 272 | rubydoesenc = true 273 | end 274 | r, w = 0, 0 275 | while r < q.length 276 | c = q[r] 277 | case true 278 | when c == ?\\ 279 | r += 1 280 | if r >= q.length 281 | raise Error, "string literal ends with a \"\\\": \"#{q}\"" 282 | end 283 | 284 | case q[r] 285 | when ?",?\\,?/,?' 286 | a[w] = q[r] 287 | r += 1 288 | w += 1 289 | when ?b,?f,?n,?r,?t 290 | a[w] = Unesc[q[r]] 291 | r += 1 292 | w += 1 293 | when ?u 294 | r += 1 295 | uchar = begin 296 | hexdec4(q[r,4]) 297 | rescue RuntimeError => e 298 | raise Error, "invalid escape sequence \\u#{q[r,4]}: #{e}" 299 | end 300 | r += 4 301 | if surrogate? uchar 302 | if q.length >= r+6 303 | uchar1 = hexdec4(q[r+2,4]) 304 | uchar = subst(uchar, uchar1) 305 | if uchar != Ucharerr 306 | # A valid pair; consume. 307 | r += 6 308 | end 309 | end 310 | end 311 | if rubydoesenc 312 | a[w] = '' << uchar 313 | w += 1 314 | else 315 | w += ucharenc(a, w, uchar) 316 | end 317 | else 318 | raise Error, "invalid escape char #{q[r]} in \"#{q}\"" 319 | end 320 | when c == ?", c < Spc 321 | raise Error, "invalid character in string literal \"#{q}\"" 322 | else 323 | # Copy anything else byte-for-byte. 324 | # Valid UTF-8 will remain valid UTF-8. 325 | # Invalid UTF-8 will remain invalid UTF-8. 326 | # In ruby >= 1.9, c is a codepoint, not a byte, 327 | # in which case this is still what we want. 328 | a[w] = c 329 | r += 1 330 | w += 1 331 | end 332 | end 333 | a[0,w] 334 | end 335 | 336 | 337 | # Encodes unicode character u as UTF-8 338 | # bytes in string a at position i. 339 | # Returns the number of bytes written. 340 | def ucharenc(a, i, u) 341 | case true 342 | when u <= Uchar1max 343 | a[i] = (u & 0xff).chr 344 | 1 345 | when u <= Uchar2max 346 | a[i+0] = (Utag2 | ((u>>6)&0xff)).chr 347 | a[i+1] = (Utagx | (u&Umaskx)).chr 348 | 2 349 | when u <= Uchar3max 350 | a[i+0] = (Utag3 | ((u>>12)&0xff)).chr 351 | a[i+1] = (Utagx | ((u>>6)&Umaskx)).chr 352 | a[i+2] = (Utagx | (u&Umaskx)).chr 353 | 3 354 | else 355 | a[i+0] = (Utag4 | ((u>>18)&0xff)).chr 356 | a[i+1] = (Utagx | ((u>>12)&Umaskx)).chr 357 | a[i+2] = (Utagx | ((u>>6)&Umaskx)).chr 358 | a[i+3] = (Utagx | (u&Umaskx)).chr 359 | 4 360 | end 361 | end 362 | 363 | 364 | def hexdec4(s) 365 | if s.length != 4 366 | raise Error, 'short' 367 | end 368 | (nibble(s[0])<<12) | (nibble(s[1])<<8) | (nibble(s[2])<<4) | nibble(s[3]) 369 | end 370 | 371 | 372 | def subst(u1, u2) 373 | if Usurr1 <= u1 && u1 < Usurr2 && Usurr2 <= u2 && u2 < Usurr3 374 | return ((u1-Usurr1)<<10) | (u2-Usurr2) + Usurrself 375 | end 376 | return Ucharerr 377 | end 378 | 379 | 380 | def surrogate?(u) 381 | Usurr1 <= u && u < Usurr3 382 | end 383 | 384 | 385 | def nibble(c) 386 | case true 387 | when ?0 <= c && c <= ?9 then c.ord - ?0.ord 388 | when ?a <= c && c <= ?z then c.ord - ?a.ord + 10 389 | when ?A <= c && c <= ?Z then c.ord - ?A.ord + 10 390 | else 391 | raise Error, "invalid hex code #{c}" 392 | end 393 | end 394 | 395 | 396 | # Encodes x into a json text. It may contain only 397 | # Array, Hash, String, Numeric, true, false, nil. 398 | # (Note, this list excludes Symbol.) 399 | # X itself must be an Array or a Hash. 400 | # No other value can be encoded, and an error will 401 | # be raised if x contains any other value, such as 402 | # Nan, Infinity, Symbol, and Proc, or if a Hash key 403 | # is not a String. 404 | # Strings contained in x must be valid UTF-8. 405 | def encode(x) 406 | case x 407 | when Hash then objenc(x) 408 | when Array then arrenc(x) 409 | else 410 | raise Error, 'root value must be an Array or a Hash' 411 | end 412 | end 413 | 414 | 415 | def valenc(x) 416 | case x 417 | when Hash then objenc(x) 418 | when Array then arrenc(x) 419 | when String then strenc(x) 420 | when Numeric then numenc(x) 421 | when true then "true" 422 | when false then "false" 423 | when nil then "null" 424 | else 425 | raise Error, "cannot encode #{x.class}: #{x.inspect}" 426 | end 427 | end 428 | 429 | 430 | def objenc(x) 431 | '{' + x.map{|k,v| keyenc(k) + ':' + valenc(v)}.join(',') + '}' 432 | end 433 | 434 | 435 | def arrenc(a) 436 | '[' + a.map{|x| valenc(x)}.join(',') + ']' 437 | end 438 | 439 | 440 | def keyenc(k) 441 | case k 442 | when String then strenc(k) 443 | else 444 | raise Error, "Hash key is not a string: #{k.inspect}" 445 | end 446 | end 447 | 448 | 449 | def strenc(s) 450 | t = StringIO.new 451 | t.putc(?") 452 | r = 0 453 | 454 | # In ruby >= 1.9, s[r] is a codepoint, not a byte. 455 | rubydoesenc = s.class.method_defined?(:encoding) 456 | 457 | while r < s.length 458 | case s[r] 459 | when ?" then t.print('\\"') 460 | when ?\\ then t.print('\\\\') 461 | when ?\b then t.print('\\b') 462 | when ?\f then t.print('\\f') 463 | when ?\n then t.print('\\n') 464 | when ?\r then t.print('\\r') 465 | when ?\t then t.print('\\t') 466 | else 467 | c = s[r] 468 | case true 469 | when rubydoesenc 470 | begin 471 | c.ord # will raise an error if c is invalid UTF-8 472 | t.write(c) 473 | rescue 474 | t.write(Ustrerr) 475 | end 476 | when Spc <= c && c <= ?~ 477 | t.putc(c) 478 | else 479 | n = ucharcopy(t, s, r) # ensure valid UTF-8 output 480 | r += n - 1 # r is incremented below 481 | end 482 | end 483 | r += 1 484 | end 485 | t.putc(?") 486 | t.string 487 | end 488 | 489 | 490 | def numenc(x) 491 | if ((x.nan? || x.infinite?) rescue false) 492 | raise Error, "Numeric cannot be represented: #{x}" 493 | end 494 | "#{x}" 495 | end 496 | 497 | 498 | # Copies the valid UTF-8 bytes of a single character 499 | # from string s at position i to I/O object t, and 500 | # returns the number of bytes copied. 501 | # If no valid UTF-8 char exists at position i, 502 | # ucharcopy writes Ustrerr and returns 1. 503 | def ucharcopy(t, s, i) 504 | n = s.length - i 505 | raise Utf8Error if n < 1 506 | 507 | c0 = s[i].ord 508 | 509 | # 1-byte, 7-bit sequence? 510 | if c0 < Utagx 511 | t.putc(c0) 512 | return 1 513 | end 514 | 515 | raise Utf8Error if c0 < Utag2 # unexpected continuation byte? 516 | 517 | raise Utf8Error if n < 2 # need continuation byte 518 | c1 = s[i+1].ord 519 | raise Utf8Error if c1 < Utagx || Utag2 <= c1 520 | 521 | # 2-byte, 11-bit sequence? 522 | if c0 < Utag3 523 | raise Utf8Error if ((c0&Umask2)<<6 | (c1&Umaskx)) <= Uchar1max 524 | t.putc(c0) 525 | t.putc(c1) 526 | return 2 527 | end 528 | 529 | # need second continuation byte 530 | raise Utf8Error if n < 3 531 | 532 | c2 = s[i+2].ord 533 | raise Utf8Error if c2 < Utagx || Utag2 <= c2 534 | 535 | # 3-byte, 16-bit sequence? 536 | if c0 < Utag4 537 | u = (c0&Umask3)<<12 | (c1&Umaskx)<<6 | (c2&Umaskx) 538 | raise Utf8Error if u <= Uchar2max 539 | t.putc(c0) 540 | t.putc(c1) 541 | t.putc(c2) 542 | return 3 543 | end 544 | 545 | # need third continuation byte 546 | raise Utf8Error if n < 4 547 | c3 = s[i+3].ord 548 | raise Utf8Error if c3 < Utagx || Utag2 <= c3 549 | 550 | # 4-byte, 21-bit sequence? 551 | if c0 < Utag5 552 | u = (c0&Umask4)<<18 | (c1&Umaskx)<<12 | (c2&Umaskx)<<6 | (c3&Umaskx) 553 | raise Utf8Error if u <= Uchar3max 554 | t.putc(c0) 555 | t.putc(c1) 556 | t.putc(c2) 557 | t.putc(c3) 558 | return 4 559 | end 560 | 561 | raise Utf8Error 562 | rescue Utf8Error 563 | t.write(Ustrerr) 564 | return 1 565 | end 566 | 567 | 568 | class Utf8Error < ::StandardError 569 | end 570 | 571 | 572 | class Error < ::StandardError 573 | end 574 | 575 | 576 | Utagx = 0x80 # 1000 0000 577 | Utag2 = 0xc0 # 1100 0000 578 | Utag3 = 0xe0 # 1110 0000 579 | Utag4 = 0xf0 # 1111 0000 580 | Utag5 = 0xF8 # 1111 1000 581 | Umaskx = 0x3f # 0011 1111 582 | Umask2 = 0x1f # 0001 1111 583 | Umask3 = 0x0f # 0000 1111 584 | Umask4 = 0x07 # 0000 0111 585 | Uchar1max = (1<<7) - 1 586 | Uchar2max = (1<<11) - 1 587 | Uchar3max = (1<<16) - 1 588 | Ucharerr = 0xFFFD # unicode "replacement char" 589 | Ustrerr = "\xef\xbf\xbd" # unicode "replacement char" 590 | Usurrself = 0x10000 591 | Usurr1 = 0xd800 592 | Usurr2 = 0xdc00 593 | Usurr3 = 0xe000 594 | 595 | Spc = ' '[0] 596 | Unesc = {?b=>?\b, ?f=>?\f, ?n=>?\n, ?r=>?\r, ?t=>?\t} 597 | end 598 | end 599 | -------------------------------------------------------------------------------- /lib/anvil/version.rb: -------------------------------------------------------------------------------- 1 | module Anvil 2 | VERSION = "0.16.1" 3 | end 4 | --------------------------------------------------------------------------------