├── .gitignore ├── .rspec ├── .ruby-gemset ├── .ruby-version ├── .travis.yml ├── Appraisals ├── Gemfile ├── LICENSE.txt ├── README.md ├── Rakefile ├── demoji.gemspec ├── gemfiles ├── rails_4.gemfile ├── rails_4.gemfile.lock ├── rails_5.gemfile └── rails_5.gemfile.lock ├── lib ├── demoji.rb └── demoji │ └── version.rb └── spec ├── config └── database.example.yml ├── models └── test_user_spec.rb ├── spec_helper.rb └── support └── test_user.rb /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | *.rbc 3 | .bundle 4 | .config 5 | .yardoc 6 | Gemfile.lock 7 | InstalledFiles 8 | _yardoc 9 | coverage 10 | doc/ 11 | lib/bundler/man 12 | pkg 13 | rdoc 14 | spec/reports 15 | test/tmp 16 | test/version_tmp 17 | tmp 18 | spec/config/database.yml 19 | .byebug_history 20 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --color 2 | -------------------------------------------------------------------------------- /.ruby-gemset: -------------------------------------------------------------------------------- 1 | demoji -------------------------------------------------------------------------------- /.ruby-version: -------------------------------------------------------------------------------- 1 | 2.4.1 -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: ruby 2 | rvm: 3 | - 1.9.3-p551 4 | - 2.0.0-p648 5 | - 2.1.9 6 | - 2.2.7 7 | - 2.3.4 8 | - 2.4.1 9 | gemfile: 10 | - gemfiles/rails_5.gemfile 11 | - gemfiles/rails_4.gemfile 12 | matrix: 13 | exclude: 14 | - rvm: 1.9.3-p551 15 | gemfile: gemfiles/rails_5.gemfile 16 | - rvm: 2.0.0-p648 17 | gemfile: gemfiles/rails_5.gemfile 18 | - rvm: 2.1.9 19 | gemfile: gemfiles/rails_5.gemfile 20 | before_install: 21 | - mysql -e 'CREATE DATABASE demoji_test;' 22 | before_script: 23 | - cp spec/config/database.example.yml spec/config/database.yml 24 | sudo: false 25 | -------------------------------------------------------------------------------- /Appraisals: -------------------------------------------------------------------------------- 1 | appraise "rails-4" do 2 | gem "activesupport", "4.2.9" 3 | gem "activerecord", "4.2.9" 4 | end 5 | 6 | appraise "rails-5" do 7 | gem "activesupport", "5.1.2" 8 | gem "activerecord", "5.1.2" 9 | end 10 | 11 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # Specify your gem's dependencies in demoji.gemspec 4 | gemspec 5 | 6 | gem 'mysql2' 7 | gem 'rspec' 8 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014 David Jairala 2 | 3 | MIT License 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![](https://travis-ci.org/taskrabbit/demoji.svg?branch=master) 2 | 3 | # Demoji 4 | 5 | MySQL configured with utf-8 encoding blows up when trying to save text rows containing emojis, etc., to address this, Demoji rescues from that specific exception and replaces the culprit chars with empty spaces. 6 | 7 | This is a workaround until Rails adds support for UTF8MB4 in migrations, schema, etc. 8 | 9 | ## Installation 10 | 11 | Add this line to your application's Gemfile: 12 | 13 | gem 'demoji' 14 | 15 | And then execute: 16 | 17 | $ bundle 18 | 19 | Or install it yourself as: 20 | 21 | $ gem install demoji 22 | 23 | ## Usage 24 | 25 | Write an initializer in: `config/initializers/demoji.rb`: 26 | 27 | ```ruby 28 | ActiveRecord::Base.send :include, Demoji 29 | ``` 30 | 31 | ## Contributing 32 | 33 | 1. Fork it 34 | 2. Create your feature branch (`git checkout -b my-new-feature`) 35 | 3. Commit your changes (`git commit -am 'Add some feature'`) 36 | 4. Push to the branch (`git push origin my-new-feature`) 37 | 5. Create new Pull Request 38 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | require "rspec/core/rake_task" 3 | 4 | RSpec::Core::RakeTask.new(:spec) 5 | 6 | task default: :spec 7 | -------------------------------------------------------------------------------- /demoji.gemspec: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | lib = File.expand_path('../lib', __FILE__) 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 4 | require 'demoji/version' 5 | 6 | Gem::Specification.new do |spec| 7 | spec.name = "demoji" 8 | spec.version = Demoji::VERSION 9 | spec.authors = ["David Jairala"] 10 | spec.email = ["davidjairala@gmail.com"] 11 | spec.description = %q{Replace emojis as to not blow up utf8 MySQL} 12 | spec.summary = %q{MySQL configured with utf-8 encoding blows up when trying to save text rows containing emojis, etc., to address this, Demoji rescues from that specific exception and replaces the culprit chars with empty spaces. This is a workaround until Rails adds support for UTF8MB4 in migrations, schema, etc.} 13 | spec.homepage = "https://github.com/taskrabbit/demoji" 14 | spec.license = "MIT" 15 | 16 | spec.files = `git ls-files`.split($/) 17 | spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) } 18 | spec.test_files = spec.files.grep(%r{^(test|spec|features)/}) 19 | spec.require_paths = ["lib"] 20 | 21 | spec.add_development_dependency "bundler", ">= 1.3" 22 | spec.add_development_dependency "rake" 23 | spec.add_development_dependency "activesupport" 24 | spec.add_development_dependency "activerecord" 25 | spec.add_development_dependency "appraisal" 26 | end 27 | -------------------------------------------------------------------------------- /gemfiles/rails_4.gemfile: -------------------------------------------------------------------------------- 1 | # This file was generated by Appraisal 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "mysql2" 6 | gem "rspec" 7 | gem "activesupport", "4.2.9" 8 | gem "activerecord", "4.2.9" 9 | 10 | gemspec path: "../" 11 | -------------------------------------------------------------------------------- /gemfiles/rails_4.gemfile.lock: -------------------------------------------------------------------------------- 1 | PATH 2 | remote: .. 3 | specs: 4 | demoji (0.0.7) 5 | 6 | GEM 7 | remote: https://rubygems.org/ 8 | specs: 9 | activemodel (4.2.9) 10 | activesupport (= 4.2.9) 11 | builder (~> 3.1) 12 | activerecord (4.2.9) 13 | activemodel (= 4.2.9) 14 | activesupport (= 4.2.9) 15 | arel (~> 6.0) 16 | activesupport (4.2.9) 17 | i18n (~> 0.7) 18 | minitest (~> 5.1) 19 | thread_safe (~> 0.3, >= 0.3.4) 20 | tzinfo (~> 1.1) 21 | appraisal (2.2.0) 22 | bundler 23 | rake 24 | thor (>= 0.14.0) 25 | arel (6.0.4) 26 | builder (3.2.3) 27 | diff-lcs (1.3) 28 | i18n (0.8.6) 29 | minitest (5.10.3) 30 | mysql2 (0.4.8) 31 | rake (12.0.0) 32 | rspec (3.6.0) 33 | rspec-core (~> 3.6.0) 34 | rspec-expectations (~> 3.6.0) 35 | rspec-mocks (~> 3.6.0) 36 | rspec-core (3.6.0) 37 | rspec-support (~> 3.6.0) 38 | rspec-expectations (3.6.0) 39 | diff-lcs (>= 1.2.0, < 2.0) 40 | rspec-support (~> 3.6.0) 41 | rspec-mocks (3.6.0) 42 | diff-lcs (>= 1.2.0, < 2.0) 43 | rspec-support (~> 3.6.0) 44 | rspec-support (3.6.0) 45 | thor (0.19.4) 46 | thread_safe (0.3.6) 47 | tzinfo (1.2.3) 48 | thread_safe (~> 0.1) 49 | 50 | PLATFORMS 51 | ruby 52 | 53 | DEPENDENCIES 54 | activerecord (= 4.2.9) 55 | activesupport (= 4.2.9) 56 | appraisal 57 | bundler (~> 1.3) 58 | demoji! 59 | mysql2 60 | rake 61 | rspec 62 | 63 | BUNDLED WITH 64 | 1.15.3 65 | -------------------------------------------------------------------------------- /gemfiles/rails_5.gemfile: -------------------------------------------------------------------------------- 1 | # This file was generated by Appraisal 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "mysql2" 6 | gem "rspec" 7 | gem "activesupport", "5.1.2" 8 | gem "activerecord", "5.1.2" 9 | 10 | gemspec path: "../" 11 | -------------------------------------------------------------------------------- /gemfiles/rails_5.gemfile.lock: -------------------------------------------------------------------------------- 1 | PATH 2 | remote: .. 3 | specs: 4 | demoji (0.0.7) 5 | 6 | GEM 7 | remote: https://rubygems.org/ 8 | specs: 9 | activemodel (5.1.2) 10 | activesupport (= 5.1.2) 11 | activerecord (5.1.2) 12 | activemodel (= 5.1.2) 13 | activesupport (= 5.1.2) 14 | arel (~> 8.0) 15 | activesupport (5.1.2) 16 | concurrent-ruby (~> 1.0, >= 1.0.2) 17 | i18n (~> 0.7) 18 | minitest (~> 5.1) 19 | tzinfo (~> 1.1) 20 | appraisal (2.2.0) 21 | bundler 22 | rake 23 | thor (>= 0.14.0) 24 | arel (8.0.0) 25 | concurrent-ruby (1.0.5) 26 | diff-lcs (1.3) 27 | i18n (0.8.6) 28 | minitest (5.10.3) 29 | mysql2 (0.4.8) 30 | rake (12.0.0) 31 | rspec (3.6.0) 32 | rspec-core (~> 3.6.0) 33 | rspec-expectations (~> 3.6.0) 34 | rspec-mocks (~> 3.6.0) 35 | rspec-core (3.6.0) 36 | rspec-support (~> 3.6.0) 37 | rspec-expectations (3.6.0) 38 | diff-lcs (>= 1.2.0, < 2.0) 39 | rspec-support (~> 3.6.0) 40 | rspec-mocks (3.6.0) 41 | diff-lcs (>= 1.2.0, < 2.0) 42 | rspec-support (~> 3.6.0) 43 | rspec-support (3.6.0) 44 | thor (0.19.4) 45 | thread_safe (0.3.6) 46 | tzinfo (1.2.3) 47 | thread_safe (~> 0.1) 48 | 49 | PLATFORMS 50 | ruby 51 | 52 | DEPENDENCIES 53 | activerecord (= 5.1.2) 54 | activesupport (= 5.1.2) 55 | appraisal 56 | bundler (~> 1.3) 57 | demoji! 58 | mysql2 59 | rake 60 | rspec 61 | 62 | BUNDLED WITH 63 | 1.15.3 64 | -------------------------------------------------------------------------------- /lib/demoji.rb: -------------------------------------------------------------------------------- 1 | require "demoji/version" 2 | require 'active_support/concern' 3 | require 'active_support/core_ext/module/aliasing' 4 | require 'active_support/core_ext/object/blank' 5 | 6 | module Demoji 7 | 8 | extend ActiveSupport::Concern 9 | 10 | included do 11 | alias :create_or_update_without_utf8_rescue :create_or_update 12 | alias :create_or_update :create_or_update_with_utf8_rescue 13 | end 14 | 15 | private 16 | 17 | def create_or_update_with_utf8_rescue(*) 18 | _rescued_counter ||= 0 19 | 20 | create_or_update_without_utf8_rescue 21 | rescue ActiveRecord::StatementInvalid => ex 22 | raise ex unless ex.message.match /Mysql2::Error: Incorrect string value:/ 23 | 24 | _rescued_counter += 1 25 | 26 | raise ex if _rescued_counter > 1 27 | 28 | _fix_utf8_attributes 29 | retry 30 | end 31 | 32 | def _fix_utf8_attributes 33 | self.attributes.each do |k, v| 34 | next if v.blank? || !v.is_a?(String) || (self.column_for_attribute(k).type == :binary) 35 | self.send "#{k}=", _fix_chars(v) 36 | end 37 | end 38 | 39 | def _fix_chars(str) 40 | "".tap do |out_str| 41 | 42 | # for instead of split and joins for perf 43 | for i in (0...str.length) 44 | char = str[i] 45 | char = 32.chr if char.ord > 65535 46 | out_str << char 47 | end 48 | 49 | end 50 | end 51 | end 52 | -------------------------------------------------------------------------------- /lib/demoji/version.rb: -------------------------------------------------------------------------------- 1 | module Demoji 2 | VERSION = "0.0.7" 3 | end 4 | -------------------------------------------------------------------------------- /spec/config/database.example.yml: -------------------------------------------------------------------------------- 1 | test: 2 | adapter: mysql2 3 | host: localhost 4 | username: root 5 | password: 6 | database: demoji_test 7 | -------------------------------------------------------------------------------- /spec/models/test_user_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe TestUser do 4 | 5 | def ord_to_str(ord) 6 | ord.chr("UTF-8") 7 | end 8 | 9 | it "doesn't blow up when trying to save emoji" do 10 | u = TestUser.new 11 | u.name = ord_to_str(65554) 12 | expect{ u.save }.to_not raise_error 13 | expect(u).to be_persisted 14 | expect(u.reload.name.strip).to eql "" 15 | end 16 | 17 | it "only tries to fix once" do 18 | allow_any_instance_of(TestUser).to receive(:create).and_raise(ActiveRecord::StatementInvalid.new("Mysql2::Error: Incorrect string value: things!")) 19 | expect_any_instance_of(TestUser).to receive(:_fix_utf8_attributes).once 20 | 21 | u = TestUser.new 22 | u.name = ord_to_str(65554) 23 | expect{ u.save }.to raise_error(ActiveRecord::StatementInvalid) 24 | expect(u).to_not be_persisted 25 | end 26 | 27 | it "leaves other chars alone" do 28 | u = TestUser.new 29 | u.name = "Peter Perez\n#{ord_to_str(65554)}" 30 | expect{ u.save }.to_not raise_error 31 | expect(u).to be_persisted 32 | expect(u.reload.name.strip).to eql "Peter Perez" 33 | end 34 | 35 | it "fixes non-binary columns but leaves binary columns alone" do 36 | u = TestUser.new 37 | u.name = "Peter Perez\n#{ord_to_str(65554)}" 38 | cart = "some binary data #{ord_to_str(65554)}" 39 | u.cart = cart 40 | expect{ u.save }.to_not raise_error 41 | expect(u).to be_persisted 42 | expect(u.reload.name.strip).to eql "Peter Perez" 43 | expect(u.reload.cart).to eql "some binary data #{ord_to_str(65554)}".force_encoding("ASCII-8BIT") 44 | end 45 | 46 | it "doesn't mess up with valid language specific chars" do 47 | u = TestUser.new 48 | u.name = "#{ord_to_str(252)}" 49 | expect { u.save }.to_not raise_error 50 | expect(u).to be_persisted 51 | expect(u.reload.name.strip).to eql "#{ord_to_str(252)}" 52 | end 53 | 54 | it "doesn't remove valid 3-byte utf8 chars" do 55 | u = TestUser.new 56 | u.name = "#{ord_to_str(10004)} #{ord_to_str(10027)} \xE2\x9C\x8C\xEF\xB8\x8F #{ord_to_str(66318)} abc" 57 | expect { u.save }.to_not raise_error 58 | expect(u).to be_persisted 59 | expect(u.reload.name.strip).to eql "#{ord_to_str(10004)} #{ord_to_str(10027)} \xE2\x9C\x8C\xEF\xB8\x8F abc" 60 | end 61 | 62 | it "removes emoji modifier chars" do 63 | u = TestUser.new 64 | u.name = "#{ord_to_str(10004)} #{ord_to_str(10027)} \xE2\x9C\x8C\xF0\x9F\x8F\xBE #{ord_to_str(66318)} abc" 65 | expect { u.save }.to_not raise_error 66 | expect(u).to be_persisted 67 | expect(u.reload.name.strip).to eql "#{ord_to_str(10004)} #{ord_to_str(10027)} #{ord_to_str(9996)} abc" 68 | end 69 | end 70 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require 'yaml' 2 | 3 | ENV['RACK_ENV'] ||= 'test' 4 | 5 | $LOAD_PATH.unshift(File.dirname(__FILE__)) 6 | $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib')) 7 | APP_DIR ||= File.expand_path('../../', __FILE__) 8 | 9 | require 'demoji' 10 | require 'rspec' 11 | require 'active_record' 12 | 13 | ActiveRecord::Base.send :include, Demoji 14 | 15 | Dir["#{APP_DIR}/spec/support/**/*.rb"].each {|f| require f} 16 | 17 | RSpec.configure do |config| 18 | 19 | config.before(:suite) do 20 | db_config = YAML.load_file(File.join(APP_DIR, 'spec', 'config', 'database.yml'))[ENV['RACK_ENV']] 21 | ActiveRecord::Base.establish_connection db_config 22 | 23 | ActiveRecord::Base.connection.execute("DROP TABLE IF EXISTS `test_users`") 24 | ActiveRecord::Base.connection.execute( 25 | "CREATE TABLE IF NOT EXISTS `test_users` (`id` int(11) NOT NULL AUTO_INCREMENT, " \ 26 | "`name` varchar(255) NOT NULL, `cart` blob, PRIMARY KEY (`id`)) DEFAULT CHARSET=utf8;" 27 | ) 28 | end 29 | 30 | config.before(:each) do 31 | ActiveRecord::Base.connection.execute("TRUNCATE TABLE `test_users`") 32 | end 33 | 34 | end 35 | -------------------------------------------------------------------------------- /spec/support/test_user.rb: -------------------------------------------------------------------------------- 1 | require 'demoji' 2 | 3 | class TestUser < ActiveRecord::Base 4 | end 5 | --------------------------------------------------------------------------------