├── .github └── workflows │ └── ruby.yml ├── .gitignore ├── Gemfile ├── LICENSE.txt ├── README.md ├── Rakefile ├── faraday-encoding.gemspec ├── lib ├── faraday-encoding.rb └── faraday │ └── encoding.rb └── spec ├── faraday └── encoding_spec.rb └── spec_helper.rb /.github/workflows/ruby.yml: -------------------------------------------------------------------------------- 1 | # This workflow uses actions that are not certified by GitHub. 2 | # They are provided by a third-party and are governed by 3 | # separate terms of service, privacy policy, and support 4 | # documentation. 5 | # This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake 6 | # For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby 7 | 8 | name: Ruby 9 | 10 | on: 11 | push: 12 | branches: [ master ] 13 | pull_request: 14 | 15 | jobs: 16 | test: 17 | 18 | runs-on: ubuntu-latest 19 | strategy: 20 | matrix: 21 | ruby: [ '2.7', '3.0', '3.1', '3.2', '3.3' ] 22 | 23 | steps: 24 | - uses: actions/checkout@v2 25 | - name: Set up Ruby 26 | # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby, 27 | # change this to (see https://github.com/ruby/setup-ruby#versioning): 28 | uses: ruby/setup-ruby@v1 29 | with: 30 | ruby-version: ${{ matrix.ruby }} 31 | bundler-cache: true # runs 'bundle install' and caches installed gems automatically 32 | - name: Run tests 33 | run: bundle exec rake 34 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.bundle/ 2 | /.yardoc 3 | /Gemfile.lock 4 | /_yardoc/ 5 | /coverage/ 6 | /doc/ 7 | /pkg/ 8 | /spec/reports/ 9 | /tmp/ 10 | *.bundle 11 | *.so 12 | *.o 13 | *.a 14 | mkmf.log 15 | /vendor 16 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # Specify your gem's dependencies in faraday-encoding.gemspec 4 | gemspec 5 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015 Takayuki Matsubara 2 | 3 | MIT License 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Faraday::Encoding 2 | 3 | [![Gem Version](https://badge.fury.io/rb/faraday-encoding.svg)](http://badge.fury.io/rb/faraday-encoding) 4 | [![Build Status](https://github.com/ma2gedev/faraday-encoding/workflows/Ruby/badge.svg?branch=master)](https://github.com/ma2gedev/faraday-encoding/actions?query=workflow%3ARuby) 5 | 6 | A Faraday Middleware sets body encoding when specified by server. 7 | 8 | ## Motivation 9 | 10 | Response body's encoding is set always ASCII-8BIT using with net/http adapter. 11 | Net::HTTP doesn't handle encoding when server specifies encoding in content-type header. 12 | Sometimes we caught an Error such as the following: 13 | 14 | ```ruby 15 | body = Faraday.new(url: 'https://example.com').get('/').body 16 | # body contains utf-8 string. ex: "赤坂" 17 | body.to_json 18 | # => raise Encoding::UndefinedConversionError: "\xE8" from ASCII-8BIT to UTF-8 19 | ``` 20 | 21 | That's why I wrote Farday::Encoding gem. 22 | 23 | SEE ALSO: [response.body is ASCII-8BIT when Content-Type is text/xml; charset=utf-8](https://github.com/lostisland/faraday/issues/139) 24 | 25 | ## Installation 26 | 27 | Add this line to your application's Gemfile: 28 | 29 | ```ruby 30 | gem 'faraday-encoding' 31 | ``` 32 | 33 | And then execute: 34 | 35 | $ bundle 36 | 37 | Or install it yourself as: 38 | 39 | $ gem install faraday-encoding 40 | 41 | ## Usage 42 | 43 | ```ruby 44 | require 'faraday/encoding' 45 | 46 | conn = Faraday.new do |connection| 47 | connection.response :encoding # use Faraday::Encoding middleware 48 | connection.adapter Faraday.default_adapter # net/http 49 | end 50 | 51 | response = conn.get '/nya.html' # content-type is specified as 'text/plain; charset=utf-8' 52 | response.body.encoding 53 | # => # 54 | ``` 55 | 56 | ## Contributing 57 | 58 | 1. Fork it ( https://github.com/ma2gedev/faraday-encoding/fork ) 59 | 2. Create your feature branch (`git checkout -b my-new-feature`) 60 | 3. Commit your changes (`git commit -am 'Add some feature'`) 61 | 4. Push to the branch (`git push origin my-new-feature`) 62 | 5. Create a new Pull Request 63 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | require "rspec/core/rake_task" 3 | 4 | RSpec::Core::RakeTask.new(:spec) 5 | 6 | task :default => :spec 7 | -------------------------------------------------------------------------------- /faraday-encoding.gemspec: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | lib = File.expand_path('../lib', __FILE__) 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 4 | 5 | Gem::Specification.new do |spec| 6 | spec.name = "faraday-encoding" 7 | spec.version = "0.0.6" 8 | spec.authors = ["Takayuki Matsubara"] 9 | spec.email = ["takayuki.1229@gmail.com"] 10 | spec.summary = %q{A Faraday Middleware sets body encoding when specified by server.} 11 | spec.description = %q{A Faraday Middleware sets body encoding when specified by server.} 12 | spec.homepage = "https://github.com/ma2gedev/faraday-encoding" 13 | spec.license = "MIT" 14 | 15 | spec.files = `git ls-files -z`.split("\x0") 16 | spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) } 17 | spec.test_files = spec.files.grep(%r{^(test|spec|features)/}) 18 | spec.require_paths = ["lib"] 19 | 20 | spec.add_development_dependency "rake", ">= 0" 21 | spec.add_development_dependency "rspec" 22 | spec.add_development_dependency 'faraday_middleware', '~> 0.10' 23 | 24 | spec.add_runtime_dependency "faraday" 25 | end 26 | -------------------------------------------------------------------------------- /lib/faraday-encoding.rb: -------------------------------------------------------------------------------- 1 | require 'faraday/encoding' 2 | -------------------------------------------------------------------------------- /lib/faraday/encoding.rb: -------------------------------------------------------------------------------- 1 | require "faraday" 2 | 3 | module Faraday 4 | class Faraday::Encoding < Faraday::Middleware 5 | def self.mappings 6 | { 7 | 'utf8' => 'utf-8' 8 | } 9 | end 10 | 11 | def call(environment) 12 | @app.call(environment).on_complete do |env| 13 | @env = env 14 | if encoding = content_charset 15 | env[:body] = env[:body].dup if env[:body].frozen? 16 | env[:body].force_encoding(encoding) if !env[:body].nil? 17 | end 18 | end 19 | end 20 | 21 | private 22 | 23 | # @return [Encoding|NilClass] returns Encoding or nil 24 | def content_charset 25 | ::Encoding.find encoding_name rescue nil 26 | end 27 | 28 | # @return [String] returns a string representing encoding name if it is find in the CONTENT TYPE header 29 | def encoding_name 30 | if /charset=([^;|$]+)/.match(content_type) 31 | mapped_encoding(Regexp.last_match(1)) 32 | end 33 | end 34 | 35 | # @param [String] encoding_name 36 | # @return [String] tries to find a mapping for the encoding name 37 | # ex: returns 'utf-8' for encoding_name 'utf8' 38 | # if mapping is not found - return the same input parameter `encoding_name` 39 | # Look at `self.mappings` to see which mappings are available 40 | def mapped_encoding(encoding_name) 41 | self.class.mappings.fetch(encoding_name, encoding_name) 42 | end 43 | 44 | # @return [String] 45 | def content_type 46 | @env[:response_headers][:content_type] 47 | end 48 | end 49 | end 50 | 51 | Faraday::Response.register_middleware encoding: Faraday::Encoding 52 | -------------------------------------------------------------------------------- /spec/faraday/encoding_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Faraday::Encoding do 4 | let(:client) do 5 | Faraday.new do |connection| 6 | connection.use FaradayMiddleware::FollowRedirects, limit: 10 7 | connection.response :encoding 8 | connection.adapter :test, stubs 9 | end 10 | end 11 | 12 | let(:stubs) do 13 | Faraday::Adapter::Test::Stubs.new do |stub| 14 | stub.get('/') do 15 | [response_status, response_headers, response_body] 16 | end 17 | stub.get('/redirected') do 18 | [redirected_status, redirected_headers, redirected_body] 19 | end 20 | stub.post('/post') do 21 | [post_response_status, post_response_headers, post_response_body] 22 | end 23 | end 24 | end 25 | 26 | let(:response_status) { 200 } 27 | let(:response_headers) do 28 | { 'content-type' => "text/plain; charset=#{response_encoding}" } 29 | end 30 | 31 | context 'http adapter return binary encoded body' do 32 | let(:response_encoding) do 33 | 'utf-8' 34 | end 35 | let(:response_body) do 36 | 'ねこ'.force_encoding(Encoding::ASCII_8BIT) 37 | end 38 | 39 | it 'set encoding specified by http header' do 40 | response = client.get('/') 41 | expect(response.body.encoding).to eq(Encoding::UTF_8) 42 | end 43 | end 44 | 45 | context 'deal correctly with frozen strings' do 46 | let(:response_encoding) do 47 | 'utf-8' 48 | end 49 | let(:response_body) do 50 | 'abc'.force_encoding(Encoding::ASCII_8BIT).freeze 51 | end 52 | 53 | it 'set encoding to utf-8' do 54 | response = client.get('/') 55 | expect(response.body.encoding).to eq(Encoding::UTF_8) 56 | end 57 | end 58 | 59 | context 'deal correctly with a non standard encoding names' do 60 | context 'utf8' do 61 | let(:response_encoding) do 62 | 'utf8' 63 | end 64 | let(:response_body) do 65 | 'abc'.force_encoding(Encoding::ASCII_8BIT) 66 | end 67 | 68 | it 'set encoding to utf-8' do 69 | response = client.get('/') 70 | expect(response.body.encoding).to eq(Encoding::UTF_8) 71 | end 72 | end 73 | 74 | context 'for unknown encoding' do 75 | let(:response_encoding) do 76 | 'unknown-encoding' 77 | end 78 | let(:response_body) do 79 | 'abc'.force_encoding(Encoding::ASCII_8BIT) 80 | end 81 | 82 | it 'does not enforce encoding' do 83 | response = client.get('/') 84 | expect(response.body.encoding).to eq(Encoding::ASCII_8BIT) 85 | end 86 | end 87 | end 88 | 89 | context 'deal correctly with redirects that specify an encoding' do 90 | # First request response with a redirect + a charset in the content-type header 91 | let(:response_status) { 301 } 92 | let(:response_headers) do 93 | { 94 | 'content-type' => "text/plain; charset=utf-8", 95 | 'location' => '/redirected' 96 | } 97 | end 98 | let(:response_body) { 'Redirected to /redirected' } 99 | 100 | # The second request does not specify a charset in the content-type 101 | let(:redirected_status) { 200 } 102 | let(:redirected_headers) do 103 | { 104 | 'content-type' => "text/html" 105 | } 106 | end 107 | let(:redirected_body) do 108 | 'abc'.force_encoding(Encoding::ASCII_8BIT) 109 | end 110 | it 'does not memoize redirect encoding' do 111 | response = client.get('/') 112 | expect(response.body.encoding).to eq(Encoding::ASCII_8BIT) 113 | end 114 | end 115 | 116 | context 'ignore with an empty body' do 117 | let(:post_response_status) { 204 } 118 | let(:post_response_headers) do 119 | { 'content-type' => "text/plain; charset=utf-8" } 120 | end 121 | let(:post_response_body) do 122 | nil 123 | end 124 | 125 | it 'skips `force_encoding`' do 126 | expect { client.post('/post') }.not_to raise_error 127 | end 128 | end 129 | end 130 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | $LOAD_PATH.unshift File.expand_path("../../lib", __FILE__) 2 | require "faraday-encoding" 3 | require "faraday_middleware" 4 | 5 | RSpec.configure do |config| 6 | end 7 | --------------------------------------------------------------------------------