├── .gitignore ├── .travis.yml ├── Gemfile ├── README.md ├── Rakefile ├── data ├── dump └── languages.yml ├── language_list.gemspec ├── lib ├── language_list.rb └── language_list │ └── version.rb └── test ├── benchmark.rb ├── benchmark_lookup.rb └── language_list_test.rb /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | construct 3 | *.gem 4 | .bundle 5 | Gemfile.lock 6 | pkg/* 7 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: ruby 2 | rvm: 3 | - 2.4.0 4 | - 2.3 5 | - 2.2 6 | - 2.1 7 | - 1.9.3 8 | - jruby-18mode 9 | - jruby-19mode 10 | - 1.8.7 11 | 12 | before_install: 13 | - gem update bundler 14 | 15 | script: bundle exec rake test 16 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | # Specify your gem's dependencies in language_list.gemspec 4 | gemspec 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Language List 2 | A list of languages based upon ISO-639-1 and ISO-639-3 with functions to retrieve only common languages. 3 | 4 | [![Build Status](https://travis-ci.org/scsmith/language_list.svg?branch=master)](https://travis-ci.org/scsmith/language_list) 5 | 6 | ## Examples 7 | # Get an array of LanguageList::LanguageInfo classes 8 | all_languages = LanguageList::ALL_LANGUAGES 9 | common_languages = LanguageList::COMMON_LANGUAGES 10 | 11 | # Finding a language based on its ISO-639-1 or ISO-639-3 code or 12 | # name 13 | german = LanguageList::LanguageInfo.find('German') 14 | english = LanguageList::LanguageInfo.find('en') 15 | english.name.inspect #=> "English" 16 | english.iso_639_1.inspect #=> "en" 17 | english.iso_639_3.inspect #=> "eng" 18 | english.common? #=> true 19 | 20 | ## Testing 21 | 22 | rake 23 | 24 | ## Upgrading 25 | 26 | ### 1.1 to 1.2 27 | 28 | * Version 1.2 freezes all of the constants once they're loaded. 29 | 30 | ## Thanks 31 | Thanks goes to Steve Hardie for his work on creating a list of common languages (http://stevehardie.com/2009/10/list-of-common-languages/). 32 | 33 | ## License 34 | I don't actually know the license for this project. The project contains countries from the ISO language list although they were not obtained from the ISO website. The country list has been adapted and placed in data/languages.yml, so that it can be replaced if required. 35 | 36 | All of the code (everything except data/languages.yml) in this project is released under an MIT license. 37 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'bundler/setup' 2 | require 'language_list' 3 | Bundler::GemHelper.install_tasks 4 | 5 | require 'rake/testtask' 6 | Rake::TestTask.new do |t| 7 | t.test_files = FileList['test/language_list_test.rb'] 8 | t.verbose = true 9 | end 10 | 11 | task :dump do 12 | path = File.expand_path(File.join('..', 'data', 'languages.yml'), __FILE__) 13 | data = YAML.load_file(path) 14 | entries = data.map { |entry| LanguageList::LanguageInfo.new(entry) } 15 | output_path = File.expand_path(File.join('..', 'data', 'dump'), __FILE__) 16 | File.write(output_path, Marshal.dump(entries)) 17 | end 18 | 19 | task :default => :test 20 | -------------------------------------------------------------------------------- /language_list.gemspec: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | $:.push File.expand_path("../lib", __FILE__) 3 | require "language_list/version" 4 | 5 | Gem::Specification.new do |s| 6 | s.name = "language_list" 7 | s.version = LanguageList::VERSION 8 | s.platform = Gem::Platform::RUBY 9 | s.authors = ["Steve Smith"] 10 | s.email = ["gems@dynedge.co.uk"] 11 | s.homepage = "https://github.com/scsmith/language_list" 12 | s.summary = %q{A list of languages and methods to find and work with these languages.} 13 | s.description = %q{A list of languages based upon ISO-639-1 and ISO-639-3 with functions to retrieve only common languages.} 14 | 15 | s.rubyforge_project = "language_list" 16 | 17 | s.files = `git ls-files`.split("\n") 18 | s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n") 19 | s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) } 20 | s.require_paths = ["lib"] 21 | s.add_development_dependency 'minitest', '> 5.0.0' 22 | s.add_development_dependency 'rake' 23 | end 24 | -------------------------------------------------------------------------------- /lib/language_list.rb: -------------------------------------------------------------------------------- 1 | require 'yaml' 2 | 3 | module LanguageList 4 | class LanguageInfo 5 | attr_reader :name, :iso_639_3, :iso_639_1, :iso_639_2b, :iso_639_2t, :type 6 | 7 | def initialize(options) 8 | @name = options[:name] 9 | @common_name = options[:common_name] 10 | @iso_639_3 = options[:iso_639_3] 11 | @iso_639_1 = options[:iso_639_1] 12 | @iso_639_2b = options[:iso_639_2b] 13 | @iso_639_2t = options[:iso_639_2t] 14 | @common = options[:common] 15 | @type = options[:type] 16 | end 17 | 18 | def common_name 19 | @common_name || @name 20 | end 21 | 22 | def common? 23 | @common 24 | end 25 | 26 | def <=>(other) 27 | self.name <=> other.name 28 | end 29 | 30 | def iso_639_1? 31 | !@iso_639_1.nil? 32 | end 33 | 34 | [:ancient, :constructed, :extinct, :historical, :living, :special].each do |type| 35 | define_method("#{type.to_s}?") do 36 | @type == type 37 | end 38 | end 39 | 40 | def to_s 41 | "#{@iso_639_3}#{" (#{@iso_639_1})" if @iso_639_1} - #{@name}" 42 | end 43 | 44 | def self.find_by_iso_639_1(code) 45 | LanguageList::BY_ISO_639_1[code] 46 | end 47 | 48 | def self.find_by_iso_639_3(code) 49 | LanguageList::BY_ISO_639_3[code] 50 | end 51 | 52 | def self.find_by_iso_639_2b(code) 53 | LanguageList::BY_ISO_639_2B[code] 54 | end 55 | 56 | def self.find_by_iso_639_2t(code) 57 | LanguageList::BY_ISO_639_2T[code] 58 | end 59 | 60 | def self.find_by_name(name) 61 | return if name.nil? 62 | 63 | LanguageList::BY_NAME[name.downcase] 64 | end 65 | 66 | def self.find(code) 67 | return if code.nil? 68 | 69 | code = code.downcase 70 | find_by_iso_639_1(code) || 71 | find_by_iso_639_3(code) || 72 | find_by_iso_639_2b(code) || 73 | find_by_iso_639_2t(code) || 74 | find_by_name(code) 75 | end 76 | end 77 | 78 | ALL_LANGUAGES = begin 79 | Marshal.load(File.read(File.expand_path('../../data/dump', __FILE__))) 80 | rescue => e 81 | warn "Reverting to hash load: #{e.message}" 82 | yaml_data = YAML.load_file(File.expand_path(File.join(File.dirname(__FILE__),'..', 'data', 'languages.yml'))) 83 | yaml_data.map{|e| LanguageInfo.new(e) } 84 | end.freeze 85 | ISO_639_1 = ALL_LANGUAGES.select(&:iso_639_1?).freeze 86 | LIVING_LANGUAGES = ALL_LANGUAGES.select(&:living?).freeze 87 | COMMON_LANGUAGES = ALL_LANGUAGES.select(&:common?).freeze 88 | 89 | BY_NAME = {} 90 | BY_ISO_639_1 = {} 91 | BY_ISO_639_3 = {} 92 | BY_ISO_639_2B = {} 93 | BY_ISO_639_2T = {} 94 | ALL_LANGUAGES.each do |lang| 95 | BY_NAME[lang.name.downcase] = lang 96 | BY_NAME[lang.common_name.downcase] = lang if lang.common_name 97 | BY_ISO_639_1[lang.iso_639_1] = lang if lang.iso_639_1 98 | BY_ISO_639_3[lang.iso_639_3] = lang if lang.iso_639_3 99 | BY_ISO_639_2B[lang.iso_639_2b] = lang if lang.iso_639_2b 100 | BY_ISO_639_2T[lang.iso_639_2t] = lang if lang.iso_639_2t 101 | end 102 | BY_NAME.freeze 103 | BY_ISO_639_1.freeze 104 | BY_ISO_639_3.freeze 105 | BY_ISO_639_2B.freeze 106 | BY_ISO_639_2T.freeze 107 | end 108 | -------------------------------------------------------------------------------- /lib/language_list/version.rb: -------------------------------------------------------------------------------- 1 | module LanguageList 2 | VERSION = "1.2.1" 3 | end 4 | -------------------------------------------------------------------------------- /test/benchmark.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require "benchmark" 4 | $:.unshift(File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib'))) 5 | 6 | time = Benchmark.measure do 7 | require 'language_list' 8 | end 9 | puts time -------------------------------------------------------------------------------- /test/benchmark_lookup.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require "benchmark" 4 | $:.unshift(File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib'))) 5 | require 'language_list' 6 | 7 | 8 | count = LanguageList::ALL_LANGUAGES.count 9 | 10 | time = Benchmark.measure do 11 | 12 | 13 | 1000.times do 14 | target = LanguageList::ALL_LANGUAGES[rand(count)] 15 | 16 | LanguageList::LanguageInfo.find_by_name(target.name) 17 | LanguageList::LanguageInfo.find(target.iso_639_3) 18 | end 19 | end 20 | puts time 21 | -------------------------------------------------------------------------------- /test/language_list_test.rb: -------------------------------------------------------------------------------- 1 | require 'minitest/autorun' 2 | require 'language_list' 3 | 4 | class LanguageListTest < Minitest::Test 5 | def test_all_languages 6 | assert_equal 7707, LanguageList::ALL_LANGUAGES.length 7 | end 8 | 9 | def test_common_languages 10 | assert_equal 71, LanguageList::COMMON_LANGUAGES.length 11 | end 12 | 13 | def test_living_languages 14 | assert_equal 6986, LanguageList::LIVING_LANGUAGES.length 15 | end 16 | 17 | def test_iso_iso_639_1_languages 18 | assert_equal 184, LanguageList::ISO_639_1.length 19 | end 20 | 21 | def test_find_by_iso_639_1 22 | english = LanguageList::LanguageInfo.find_by_iso_639_1('en') 23 | assert_equal 'en', english.iso_639_1 24 | assert_equal 'eng', english.iso_639_3 25 | assert_equal 'English', english.name 26 | end 27 | 28 | def test_find_by_iso_639_1_with_nil 29 | result = LanguageList::LanguageInfo.find_by_iso_639_1(nil) 30 | assert_equal nil, result 31 | end 32 | 33 | def test_find_by_iso_639_1_with_empty_string 34 | result = LanguageList::LanguageInfo.find_by_iso_639_1('') 35 | assert_equal nil, result 36 | end 37 | 38 | def test_find_by_iso_639_1_with_frozen_string 39 | english = LanguageList::LanguageInfo.find_by_iso_639_1('en'.freeze) 40 | assert_equal 'en', english.iso_639_1 41 | assert_equal 'eng', english.iso_639_3 42 | assert_equal 'English', english.name 43 | end 44 | 45 | def test_find_by_iso_639_3 46 | english = LanguageList::LanguageInfo.find_by_iso_639_3('eng') 47 | assert_equal 'en', english.iso_639_1 48 | assert_equal 'eng', english.iso_639_3 49 | assert_equal 'English', english.name 50 | end 51 | 52 | def test_find_by_iso_639_3_with_nil 53 | result = LanguageList::LanguageInfo.find_by_iso_639_3(nil) 54 | assert_equal nil, result 55 | end 56 | 57 | def test_find_by_iso_639_3_with_empty_string 58 | result = LanguageList::LanguageInfo.find_by_iso_639_3('') 59 | assert_equal nil, result 60 | end 61 | 62 | def test_find_by_iso_639_3_with_frozen_string 63 | english = LanguageList::LanguageInfo.find_by_iso_639_3('eng'.freeze) 64 | assert_equal 'en', english.iso_639_1 65 | assert_equal 'eng', english.iso_639_3 66 | assert_equal 'English', english.name 67 | end 68 | 69 | def test_find_by_name_with_nil 70 | result = LanguageList::LanguageInfo.find_by_name(nil) 71 | assert_equal nil, result 72 | end 73 | 74 | def test_find_by_name_with_empty_string 75 | result = LanguageList::LanguageInfo.find_by_name('') 76 | assert_equal nil, result 77 | end 78 | 79 | def test_find_by_name_with_frozen_string 80 | english = LanguageList::LanguageInfo.find_by_name('English'.freeze) 81 | assert_equal 'en', english.iso_639_1 82 | assert_equal 'eng', english.iso_639_3 83 | assert_equal 'eng', english.iso_639_2b 84 | assert_equal 'eng', english.iso_639_2t 85 | assert_equal 'English', english.name 86 | end 87 | 88 | def test_find_by_name 89 | english = LanguageList::LanguageInfo.find_by_name('English') 90 | assert_equal 'en', english.iso_639_1 91 | assert_equal 'eng', english.iso_639_3 92 | assert_equal 'eng', english.iso_639_2b 93 | assert_equal 'eng', english.iso_639_2t 94 | assert_equal 'English', english.name 95 | end 96 | 97 | def test_find_by_common_name 98 | greek = LanguageList::LanguageInfo.find_by_name('greek') 99 | assert_equal 'el', greek.iso_639_1 100 | assert_equal 'ell', greek.iso_639_3 101 | assert_equal 'gre', greek.iso_639_2b 102 | assert_equal 'ell', greek.iso_639_2t 103 | assert_equal 'Greek', greek.common_name 104 | end 105 | 106 | def test_case_insensitive_find_by_name 107 | english = LanguageList::LanguageInfo.find_by_name('english') 108 | assert_equal 'en', english.iso_639_1 109 | assert_equal 'eng', english.iso_639_3 110 | assert_equal 'English', english.name 111 | end 112 | 113 | def test_find_with_nil 114 | result = LanguageList::LanguageInfo.find(nil) 115 | assert_equal nil, result 116 | end 117 | 118 | def test_find_with_empty_string 119 | result = LanguageList::LanguageInfo.find('') 120 | assert_equal nil, result 121 | end 122 | 123 | def test_find_with_frozen_string 124 | english = LanguageList::LanguageInfo.find('EN'.freeze) 125 | assert_equal 'en', english.iso_639_1 126 | assert_equal 'eng', english.iso_639_3 127 | assert_equal 'English', english.name 128 | end 129 | 130 | def test_find_with_iso_639_1_code 131 | english = LanguageList::LanguageInfo.find('en') 132 | assert_equal 'en', english.iso_639_1 133 | assert_equal 'eng', english.iso_639_3 134 | assert_equal 'English', english.name 135 | end 136 | 137 | def test_case_insensitive_find_with_iso_639_1_code 138 | english = LanguageList::LanguageInfo.find('EN') 139 | assert_equal 'en', english.iso_639_1 140 | assert_equal 'eng', english.iso_639_3 141 | assert_equal 'English', english.name 142 | end 143 | 144 | def test_find_with_iso_639_2b_code 145 | macedonian = LanguageList::LanguageInfo.find('mac') 146 | assert_equal 'mk', macedonian.iso_639_1 147 | assert_equal 'mac', macedonian.iso_639_2b 148 | assert_equal 'mkd', macedonian.iso_639_2t 149 | assert_equal 'mkd', macedonian.iso_639_3 150 | assert_equal 'Macedonian', macedonian.name 151 | end 152 | 153 | def test_case_insensitive_find_with_iso_639_2b_code 154 | macedonian = LanguageList::LanguageInfo.find('MAC') 155 | assert_equal 'mk', macedonian.iso_639_1 156 | assert_equal 'mac', macedonian.iso_639_2b 157 | assert_equal 'mkd', macedonian.iso_639_2t 158 | assert_equal 'mkd', macedonian.iso_639_3 159 | assert_equal 'Macedonian', macedonian.name 160 | end 161 | 162 | def test_find_with_iso_639_2t_code 163 | macedonian = LanguageList::LanguageInfo.find('mkd') 164 | assert_equal 'mk', macedonian.iso_639_1 165 | assert_equal 'mac', macedonian.iso_639_2b 166 | assert_equal 'mkd', macedonian.iso_639_2t 167 | assert_equal 'mkd', macedonian.iso_639_3 168 | assert_equal 'Macedonian', macedonian.name 169 | end 170 | 171 | def test_case_insensitive_find_with_iso_639_2t_code 172 | macedonian = LanguageList::LanguageInfo.find('MKD') 173 | assert_equal 'mk', macedonian.iso_639_1 174 | assert_equal 'mac', macedonian.iso_639_2b 175 | assert_equal 'mkd', macedonian.iso_639_2t 176 | assert_equal 'mkd', macedonian.iso_639_3 177 | assert_equal 'Macedonian', macedonian.name 178 | end 179 | 180 | def test_find_with_iso_639_3_code 181 | english = LanguageList::LanguageInfo.find('eng') 182 | assert_equal 'en', english.iso_639_1 183 | assert_equal 'eng', english.iso_639_3 184 | assert_equal 'English', english.name 185 | end 186 | 187 | def test_case_insensitive_find_with_iso_639_3_code 188 | english = LanguageList::LanguageInfo.find('Eng') 189 | assert_equal 'en', english.iso_639_1 190 | assert_equal 'eng', english.iso_639_3 191 | assert_equal 'English', english.name 192 | end 193 | 194 | def test_find_with_name 195 | english = LanguageList::LanguageInfo.find('English') 196 | assert_equal 'en', english.iso_639_1 197 | assert_equal 'eng', english.iso_639_3 198 | assert_equal 'eng', english.iso_639_2b 199 | assert_equal 'eng', english.iso_639_2t 200 | assert_equal 'English', english.name 201 | end 202 | 203 | def test_sort_should_order_by_name 204 | sorted = LanguageList::COMMON_LANGUAGES.sort 205 | expected = LanguageList::COMMON_LANGUAGES.map(&:name).sort 206 | 207 | assert_equal expected, sorted.map(&:name) 208 | end 209 | 210 | def test_common_name_when_present 211 | greek = LanguageList::LanguageInfo.find('el') 212 | assert_equal 'Greek', greek.common_name 213 | end 214 | 215 | def test_common_name_when_not_present 216 | english = LanguageList::LanguageInfo.find('en') 217 | assert_equal english.name, english.common_name 218 | end 219 | 220 | def test_cannot_change_constants 221 | assert_raises RuntimeError, TypeError do 222 | LanguageList::ALL_LANGUAGES[0] = nil 223 | end 224 | 225 | assert_raises RuntimeError, TypeError do 226 | LanguageList::ISO_639_1[0] = nil 227 | end 228 | 229 | assert_raises RuntimeError, TypeError do 230 | LanguageList::LIVING_LANGUAGES[0] = nil 231 | end 232 | 233 | assert_raises RuntimeError, TypeError do 234 | LanguageList::COMMON_LANGUAGES[0] = nil 235 | end 236 | 237 | assert_raises RuntimeError, TypeError do 238 | LanguageList::BY_NAME[0] = nil 239 | end 240 | 241 | assert_raises RuntimeError, TypeError do 242 | LanguageList::BY_ISO_639_1[0] = nil 243 | end 244 | 245 | assert_raises RuntimeError, TypeError do 246 | LanguageList::BY_ISO_639_3[0] = nil 247 | end 248 | 249 | assert_raises RuntimeError, TypeError do 250 | LanguageList::BY_ISO_639_2T[0] = nil 251 | end 252 | 253 | assert_raises RuntimeError, TypeError do 254 | LanguageList::BY_ISO_639_2B[0] = nil 255 | end 256 | end 257 | end 258 | --------------------------------------------------------------------------------