├── lib ├── kdl │ ├── version.rb │ ├── v1.rb │ ├── v1 │ │ ├── document.rb │ │ ├── node.rb │ │ ├── string_dumper.rb │ │ ├── value.rb │ │ ├── kdl.yy │ │ └── tokenizer.rb │ ├── types │ │ ├── regex.rb │ │ ├── decimal.rb │ │ ├── uuid.rb │ │ ├── base64.rb │ │ ├── ip.rb │ │ ├── url.rb │ │ ├── currency.rb │ │ ├── duration.rb │ │ ├── hostname.rb │ │ ├── date_time.rb │ │ ├── hostname │ │ │ └── validator.rb │ │ ├── email.rb │ │ ├── irl.rb │ │ ├── country.rb │ │ ├── irl │ │ │ └── parser.rb │ │ ├── email │ │ │ └── parser.rb │ │ ├── duration │ │ │ └── iso8601_parser.rb │ │ ├── url_template.rb │ │ └── currency │ │ │ └── iso4217_currencies.rb │ ├── types.rb │ ├── parser_common.rb │ ├── string_dumper.rb │ ├── error.rb │ ├── document.rb │ ├── builder.rb │ ├── node.rb │ ├── kdl.yy │ ├── value.rb │ └── tokenizer.rb └── kdl.rb ├── bin ├── kdl ├── setup ├── console ├── racc └── rake ├── .gitignore ├── .gitmodules ├── Gemfile ├── test ├── test_helper.rb ├── types │ ├── regex_test.rb │ ├── decimal_test.rb │ ├── base64_test.rb │ ├── currency_test.rb │ ├── uuid_test.rb │ ├── ip_test.rb │ ├── url_test.rb │ ├── country_test.rb │ ├── date_time_test.rb │ ├── hostname_test.rb │ ├── duration_test.rb │ ├── irl_test.rb │ ├── email_test.rb │ └── url_template_test.rb ├── kdl_test.rb ├── v1 │ ├── document_test.rb │ ├── node_test.rb │ ├── spec_test.rb │ ├── value_test.rb │ ├── examples_test.rb │ └── tokenizer_test.rb ├── spec_test.rb ├── builder_test.rb ├── examples_test.rb ├── document_test.rb ├── types_test.rb ├── value_test.rb ├── node_test.rb └── tokenizer_test.rb ├── Rakefile ├── LICENSE.txt ├── .github └── workflows │ └── ruby.yml ├── kdl.gemspec ├── README.md └── CODE_OF_CONDUCT.md /lib/kdl/version.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module KDL 4 | VERSION = "2.1.2" 5 | end 6 | -------------------------------------------------------------------------------- /bin/kdl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require "bundler/setup" 4 | require "kdl" 5 | 6 | system 'bin/rake racc' 7 | 8 | puts KDL.parse(ARGF.read).to_s 9 | -------------------------------------------------------------------------------- /bin/setup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | IFS=$'\n\t' 4 | set -vx 5 | 6 | bundle install 7 | 8 | # Do any other automated setup that you need to do here 9 | bin/rake racc 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.bundle/ 2 | /.yardoc 3 | /_yardoc/ 4 | /coverage/ 5 | /doc/ 6 | /pkg/ 7 | /spec/reports/ 8 | /tmp/ 9 | 10 | Gemfile.lock 11 | 12 | lib/kdl/kdl.tab.rb 13 | lib/kdl/v1/kdl.tab.rb 14 | kdl.output 15 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "test/kdl-org"] 2 | path = test/kdl-org 3 | url = git@github.com:kdl-org/kdl 4 | [submodule "test/v1/kdl-org"] 5 | path = test/v1/kdl-org 6 | url = git@github.com:kdl-org/kdl 7 | branch = release/v1 8 | -------------------------------------------------------------------------------- /lib/kdl/v1.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "kdl/v1/tokenizer" 4 | require "kdl/v1/document" 5 | require "kdl/v1/value" 6 | require "kdl/v1/node" 7 | require "kdl/v1/string_dumper" 8 | require "kdl/v1/kdl.tab" 9 | 10 | module KDL 11 | module V1 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | # Specify your gem's dependencies in kdl.gemspec 4 | gemspec 5 | 6 | gem "rake", "~> 13.0" 7 | 8 | group :test do 9 | gem "minitest", "~> 5.0" 10 | gem "simplecov", require: false 11 | gem "coveralls_reborn", require: false 12 | end 13 | -------------------------------------------------------------------------------- /test/test_helper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "simplecov" 4 | require "coveralls" 5 | SimpleCov.formatters = [ 6 | SimpleCov::Formatter::HTMLFormatter, 7 | Coveralls::SimpleCov::Formatter 8 | ] 9 | SimpleCov.start 10 | 11 | $LOAD_PATH.unshift File.expand_path("../lib", __dir__) 12 | require "kdl" 13 | 14 | require "minitest/autorun" 15 | -------------------------------------------------------------------------------- /lib/kdl/v1/document.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module KDL 4 | module V1 5 | class Document < ::KDL::Document 6 | def version 7 | 1 8 | end 9 | 10 | def to_v1 11 | self 12 | end 13 | 14 | def to_v2 15 | ::KDL::Document.new(nodes.map(&:to_v2)) 16 | end 17 | end 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /test/types/regex_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "test_helper" 4 | 5 | class RegexTest < Minitest::Test 6 | def test_regex 7 | value = KDL::Types::Regex.call(::KDL::Value::String.new('asdf')) 8 | assert_equal(/asdf/, value.value) 9 | 10 | assert_raises { KDL::Types::Regex.call(::KDL::Value::String.new('invalid(regex]')) } 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /lib/kdl/types/regex.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module KDL 4 | module Types 5 | class Regex < Value::Custom 6 | def self.call(value, type = 'regex') 7 | return nil unless value.is_a? ::KDL::Value::String 8 | 9 | regex = ::Regexp.new(value.value) 10 | new(regex, type: type) 11 | end 12 | end 13 | MAPPING['regex'] = Regex 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /test/types/decimal_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "test_helper" 4 | 5 | class DecimalTest < Minitest::Test 6 | def test_decimal 7 | value = KDL::Types::Decimal.call(::KDL::Value::String.new('10000000000000')) 8 | assert_equal(BigDecimal('10000000000000'), value.value) 9 | assert_raises { KDL::Types::Decimal.call(::KDL::Value::String.new('not a decimal')) } 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /test/types/base64_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "test_helper" 4 | 5 | class Base64Test < Minitest::Test 6 | def test_base64 7 | value = KDL::Types::Base64.call(::KDL::Value::String.new('U2VuZCByZWluZm9yY2VtZW50cw==')) 8 | assert_equal 'Send reinforcements', value.value 9 | 10 | assert_raises { KDL::Types::Base64.call(::KDL::Value::String.new('not base64')) } 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /lib/kdl/types/decimal.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module KDL 4 | module Types 5 | class Decimal < Value::Custom 6 | def self.call(value, type = 'decimal') 7 | return nil unless value.is_a? ::KDL::Value::String 8 | 9 | big_decimal = BigDecimal(value.value) 10 | new(big_decimal, type: type) 11 | end 12 | end 13 | MAPPING['decimal'] = Decimal 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /bin/console: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require "bundler/setup" 4 | require "kdl" 5 | 6 | # You can add fixtures and/or initialization code here to make experimenting 7 | # with your gem easier. You can also use a different console, if you like. 8 | 9 | # (If you use this, don't forget to add pry to your Gemfile!) 10 | # require "pry" 11 | # Pry.start 12 | 13 | system 'bin/rake racc' 14 | 15 | require "irb" 16 | IRB.start(__FILE__) 17 | -------------------------------------------------------------------------------- /test/types/currency_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'test_helper' 4 | 5 | class CurrencyTest < Minitest::Test 6 | def test_currency 7 | value = KDL::Types::Currency.call(::KDL::Value::String.new('ZAR')) 8 | assert_equal({ numeric_code: 710, 9 | minor_unit: 2, 10 | name: 'South African rand' }, value.value) 11 | 12 | assert_raises { KDL::Types::Currency.call(::KDL::Value::String.new('ZZZ')) } 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /lib/kdl/types/uuid.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module KDL 4 | module Types 5 | class UUID < Value::Custom 6 | RGX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/ 7 | 8 | def self.call(value, type = 'uuid') 9 | return nil unless value.is_a? ::KDL::Value::String 10 | 11 | uuid = value.value.downcase 12 | raise ArgumentError, "`#{value.value}' is not a valid uuid" unless uuid =~ RGX 13 | 14 | new(uuid, type: type) 15 | end 16 | end 17 | MAPPING['uuid'] = UUID 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /lib/kdl/types.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module KDL 4 | module Types 5 | MAPPING = {} 6 | end 7 | end 8 | 9 | require 'kdl/types/date_time' 10 | require 'kdl/types/duration' 11 | require 'kdl/types/currency' 12 | require 'kdl/types/country' 13 | require 'kdl/types/ip' 14 | require 'kdl/types/url' 15 | require 'kdl/types/uuid' 16 | require 'kdl/types/regex' 17 | require 'kdl/types/base64' 18 | require 'kdl/types/decimal' 19 | require 'kdl/types/hostname' 20 | require 'kdl/types/email' 21 | require 'kdl/types/irl' 22 | require 'kdl/types/url_template' 23 | 24 | KDL::Types::MAPPING.freeze 25 | -------------------------------------------------------------------------------- /test/types/uuid_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "test_helper" 4 | 5 | class UUIDTest < Minitest::Test 6 | def test_uuid 7 | value = KDL::Types::UUID.call(::KDL::Value::String.new('f81d4fae-7dec-11d0-a765-00a0c91e6bf6')) 8 | assert_equal 'f81d4fae-7dec-11d0-a765-00a0c91e6bf6', value.value 9 | value = KDL::Types::UUID.call(::KDL::Value::String.new('F81D4FAE-7DEC-11D0-A765-00A0C91E6BF6')) 10 | assert_equal 'f81d4fae-7dec-11d0-a765-00a0c91e6bf6', value.value 11 | 12 | assert_raises { KDL::Types::UUID.call(::KDL::Value::String.new('not a uuid')) } 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /lib/kdl/types/base64.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'base64' 4 | 5 | module KDL 6 | module Types 7 | class Base64 < Value::Custom 8 | RGX = /^[A-Za-z0-9+\/=]+$/.freeze 9 | 10 | def self.call(value, type = 'base64') 11 | return nil unless value.is_a? ::KDL::Value::String 12 | 13 | unless RGX.match?(value.value) 14 | raise ArgumentError, "invalid base64: #{value.value}" 15 | end 16 | 17 | data = ::Base64.decode64(value.value) 18 | new(data, type: type) 19 | end 20 | end 21 | MAPPING['base64'] = Base64 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /test/kdl_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "test_helper" 4 | 5 | class KDLTest < Minitest::Test 6 | def test_that_it_has_a_version_number 7 | refute_nil ::KDL::VERSION 8 | end 9 | 10 | def test_parse_document 11 | assert_equal KDL.parse('node 1 2 3'), KDL.parse_document('node 1 2 3') 12 | end 13 | 14 | def test_unsupported_version 15 | assert_raises(KDL::UnsupportedVersionError) { KDL.parse('node 1 2 3', version: 3) } 16 | assert_raises(KDL::UnsupportedVersionError) do 17 | KDL.parse <<~KDL 18 | /- kdl-version 3 19 | node 1 2 3 20 | KDL 21 | end 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'bundler/gem_tasks' 2 | require 'rake/testtask' 3 | 4 | file 'lib/kdl/kdl.tab.rb' => ['lib/kdl/kdl.yy'] do 5 | raise "racc command failed" unless system 'bin/racc lib/kdl/kdl.yy' 6 | end 7 | task :racc => 'lib/kdl/kdl.tab.rb' 8 | 9 | file 'lib/kdl/v1/kdl.tab.rb' => ['lib/kdl/v1/kdl.yy'] do 10 | raise "racc command failed (v1)" unless system 'bin/racc lib/kdl/v1/kdl.yy' 11 | end 12 | task :racc_v1 => 'lib/kdl/v1/kdl.tab.rb' 13 | 14 | Rake::TestTask.new(:test => [:racc, :racc_v1]) do |t| 15 | t.libs << 'test' 16 | t.libs << 'lib' 17 | t.test_files = FileList['test/**/*_test.rb'] 18 | t.options = '--pride' 19 | end 20 | 21 | task :default => :test 22 | -------------------------------------------------------------------------------- /lib/kdl/v1/node.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module KDL 4 | module V1 5 | class Node < ::KDL::Node 6 | def version 7 | 1 8 | end 9 | 10 | def to_v1 11 | self 12 | end 13 | 14 | def to_v2 15 | ::KDL::Node.new(name, 16 | arguments: arguments.map(&:to_v2), 17 | properties: properties.transform_values(&:to_v2), 18 | children: children.map(&:to_v2), 19 | type: type 20 | ) 21 | end 22 | 23 | private 24 | 25 | def id_to_s(id, m = :to_s) 26 | return id.public_send(m) unless m == :to_s 27 | 28 | StringDumper.stringify_identifier(id) 29 | end 30 | end 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /lib/kdl/v1/string_dumper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module KDL 4 | module V1 5 | module StringDumper 6 | include ::KDL::StringDumper 7 | 8 | def call(string) 9 | %("#{string.each_char.map { |char| escape(char) }.join}") 10 | end 11 | 12 | def stringify_identifier(ident) 13 | if bare_identifier?(ident) 14 | ident 15 | else 16 | call(ident) 17 | end 18 | end 19 | 20 | private 21 | 22 | def bare_identifier?(name) 23 | escape_chars = '\\\/(){}<>;\[\]=,"' 24 | name =~ /^([^0-9\-+\s#{escape_chars}][^\s#{escape_chars}]*|[\-+](?!true|false|null)[^0-9\s#{escape_chars}][^\s#{escape_chars}]*)$/ 25 | end 26 | 27 | extend self 28 | end 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /lib/kdl/types/ip.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module KDL 4 | module Types 5 | class IP < Value::Custom 6 | def self.call(value, type = ip_type) 7 | return nil unless value.is_a? ::KDL::Value::String 8 | 9 | ip = ::IPAddr.new(value.value) 10 | raise ArgumentError, "invalid #{ip_type} address" unless valid_ip?(ip) 11 | 12 | new(ip, type: type) 13 | end 14 | 15 | def self.valid_ip?(ip) 16 | ip.__send__(:"#{ip_type}?") 17 | end 18 | end 19 | 20 | class IPV4 < IP 21 | def self.ip_type 22 | 'ipv4' 23 | end 24 | end 25 | MAPPING['ipv4'] = IPV4 26 | 27 | class IPV6 < IP 28 | def self.ip_type 29 | 'ipv6' 30 | end 31 | end 32 | MAPPING['ipv6'] = IPV6 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /test/v1/document_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "test_helper" 4 | 5 | class KDL::V1::DocumentTest < Minitest::Test 6 | def test_version 7 | assert_equal 1, KDL::V1::Document.new([]).version 8 | end 9 | 10 | def test_to_v2 11 | doc = KDL.parse <<~KDL, version: 1 12 | foo "lorem" 1 true null { 13 | bar " 14 | baz 15 | qux 16 | " 17 | } 18 | KDL 19 | assert_equal 1, doc.version 20 | 21 | doc = doc.to_v2 22 | assert_equal 2, doc.version 23 | 24 | assert_equal <<~KDL, doc.to_s 25 | foo lorem 1 #true #null { 26 | bar "\\n baz\\n qux\\n " 27 | } 28 | KDL 29 | end 30 | 31 | def test_to_v1 32 | doc = KDL::V1::Document.new([]) 33 | assert_same doc, doc.to_v1 34 | end 35 | end 36 | -------------------------------------------------------------------------------- /lib/kdl/types/url.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module KDL 4 | module Types 5 | class URLReference < Value::Custom 6 | def self.call(value, type = 'url-reference') 7 | return nil unless value.is_a? ::KDL::Value::String 8 | 9 | uri = parse_url(value.value) 10 | new(uri, type: type) 11 | end 12 | 13 | def self.parse_url(string) 14 | URI.parse(string) 15 | end 16 | end 17 | MAPPING['url-reference'] = URLReference 18 | 19 | class URL < URLReference 20 | def self.call(value, type = 'url') 21 | super(value, type) 22 | end 23 | 24 | def self.parse_url(string) 25 | super.tap do |uri| 26 | raise 'invalid URL' if uri.scheme.nil? 27 | end 28 | end 29 | end 30 | MAPPING['url'] = URL 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /lib/kdl/types/currency.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'kdl/types/currency/iso4217_currencies' 4 | 5 | module KDL 6 | module Types 7 | class Currency < Value::Custom 8 | attr_reader :numeric_code, :minor_unit, :name 9 | 10 | def initialize(value, format: nil, type: 'currency') 11 | super 12 | @numeric = value.fetch(:numeric, nil) 13 | @minor_unit = value.fetch(:minor_unit, nil) 14 | @name = value.fetch(:name, '') 15 | end 16 | 17 | def self.call(value, type = 'currency') 18 | return nil unless value.is_a? ::KDL::Value::String 19 | 20 | currency = CURRENCIES[value.value.upcase] 21 | raise ArgumentError, 'invalid currency' if currency.nil? 22 | 23 | new(currency, type: type) 24 | end 25 | end 26 | MAPPING['currency'] = Currency 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /bin/racc: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | # 5 | # This file was generated by Bundler. 6 | # 7 | # The application 'racc' is installed as part of a gem, and 8 | # this file is here to facilitate running it. 9 | # 10 | 11 | require "pathname" 12 | ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile", 13 | Pathname.new(__FILE__).realpath) 14 | 15 | bundle_binstub = File.expand_path("../bundle", __FILE__) 16 | 17 | if File.file?(bundle_binstub) 18 | if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/ 19 | load(bundle_binstub) 20 | else 21 | abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run. 22 | Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.") 23 | end 24 | end 25 | 26 | require "rubygems" 27 | require "bundler/setup" 28 | 29 | load Gem.bin_path("racc", "racc") 30 | -------------------------------------------------------------------------------- /bin/rake: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | # 5 | # This file was generated by Bundler. 6 | # 7 | # The application 'rake' is installed as part of a gem, and 8 | # this file is here to facilitate running it. 9 | # 10 | 11 | require "pathname" 12 | ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile", 13 | Pathname.new(__FILE__).realpath) 14 | 15 | bundle_binstub = File.expand_path("../bundle", __FILE__) 16 | 17 | if File.file?(bundle_binstub) 18 | if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/ 19 | load(bundle_binstub) 20 | else 21 | abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run. 22 | Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.") 23 | end 24 | end 25 | 26 | require "rubygems" 27 | require "bundler/setup" 28 | 29 | load Gem.bin_path("rake", "rake") 30 | -------------------------------------------------------------------------------- /test/v1/node_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "test_helper" 4 | 5 | class KDL::V1::NodeTest < Minitest::Test 6 | def test_version 7 | node = KDL::V1::Node.new("foo") 8 | assert_equal 1, node.version 9 | end 10 | 11 | def test_to_v2 12 | node = KDL::V1::Node.new("foo", 13 | arguments: [v(true)], 14 | properties: { bar: v("baz") }, 15 | children: [KDL::V1::Node.new("qux")] 16 | ) 17 | 18 | node = node.to_v2 19 | assert_equal 2, node.version 20 | assert_equal 2, node.arguments[0].version 21 | assert_equal 2, node.properties['bar'].version 22 | assert_equal 2, node.child(0).version 23 | end 24 | 25 | def test_to_v1 26 | node = KDL::V1::Node.new("foo") 27 | assert_same node, node.to_v1 28 | end 29 | 30 | private 31 | 32 | def v(x, t=nil) 33 | val = ::KDL::V1::Value.from(x) 34 | return val.as_type(t) if t 35 | val 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /test/types/ip_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "test_helper" 4 | 5 | class IPTest < Minitest::Test 6 | def test_ipv4 7 | value = KDL::Types::IPV4.call(::KDL::Value::String.new('127.0.0.1')) 8 | assert_equal ::IPAddr.new('127.0.0.1'), value.value 9 | 10 | assert_raises { KDL::Types::IPV4.call(::KDL::Value::String.new('not an ipv4 address')) } 11 | assert_raises { KDL::Types::IPV4.call(::KDL::Value::String.new('3ffe:505:2::1')) } 12 | end 13 | 14 | def test_ipv6 15 | value = KDL::Types::IPV6.call(::KDL::Value::String.new('::')) 16 | assert_equal ::IPAddr.new('::'), value.value 17 | value = KDL::Types::IPV6.call(::KDL::Value::String.new('3ffe:505:2::1')) 18 | assert_equal ::IPAddr.new('3ffe:505:2::1'), value.value 19 | 20 | assert_raises { KDL::Types::IPV6.call(::KDL::Value::String.new('not an ipv6 address')) } 21 | assert_raises { KDL::Types::IPV6.call(::KDL::Value::String.new('127.0.0.1')) } 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /test/types/url_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "test_helper" 4 | 5 | class URLTest < Minitest::Test 6 | def test_url 7 | value = KDL::Types::URL.call(::KDL::Value::String.new('https://www.example.com/foo/bar')) 8 | assert_equal URI('https://www.example.com/foo/bar'), value.value 9 | 10 | assert_raises { KDL::Types::URL.call(::KDL::Value::String.new('not a url')) } 11 | assert_raises { KDL::Types::URL.call(::KDL::Value::String.new('/reference/to/something')) } 12 | end 13 | 14 | def test_url_reference 15 | value = KDL::Types::URLReference.call(::KDL::Value::String.new('https://www.example.com/foo/bar')) 16 | assert_equal URI('https://www.example.com/foo/bar'), value.value 17 | value = KDL::Types::URLReference.call(::KDL::Value::String.new('/foo/bar')) 18 | assert_equal URI('/foo/bar'), value.value 19 | 20 | assert_raises { KDL::Types::URLReference.call(::KDL::Value::String.new('not a url reference')) } 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /lib/kdl/types/duration.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'kdl/types/duration/iso8601_parser' 4 | 5 | module KDL 6 | module Types 7 | class Duration < Value::Custom 8 | attr_reader :years, :months, :weeks, :days, :hours, :minutes, :seconds 9 | 10 | def initialize(parts = {}, format: nil, type: 'duration') 11 | super 12 | @years = parts.fetch(:years, 0) 13 | @months = parts.fetch(:months, 0) 14 | @weeks = parts.fetch(:weeks, 0) 15 | @days = parts.fetch(:days, 0) 16 | @hours = parts.fetch(:hours, 0) 17 | @minutes = parts.fetch(:minutes, 0) 18 | @seconds = parts.fetch(:seconds, 0) 19 | end 20 | 21 | def self.call(value, type = 'duration') 22 | return nil unless value.is_a? ::KDL::Value::String 23 | 24 | parts = ISO8601Parser.new(value.value).parse! 25 | new(parts, type: type) 26 | end 27 | end 28 | MAPPING['duration'] = Duration 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /lib/kdl/parser_common.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module KDL 4 | module ParserCommon 5 | 6 | private 7 | 8 | def init(parse_types: true, type_parsers: {}, output_module: ::KDL) 9 | @output_module = output_module 10 | if parse_types 11 | @type_parsers = ::KDL::Types::MAPPING.merge(type_parsers) 12 | else 13 | @type_parsers = {} 14 | end 15 | end 16 | 17 | def next_token 18 | @tokenizer.next_token 19 | end 20 | 21 | def check_version 22 | return unless doc_version = @tokenizer.version_directive 23 | if doc_version != parser_version 24 | raise VersionMismatchError.new("version mismatch, document specified v#{doc_version}, but this is a v#{parser_version} parser", doc_version, parser_version) 25 | end 26 | end 27 | 28 | def on_error(t, val, vstack) 29 | raise KDL::ParseError.new("unexpected #{token_to_str(t)} #{val&.value.inspect}", @tokenizer.filename, val&.line, val&.column) 30 | end 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /test/spec_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'test_helper' 4 | 5 | class SpecTest < Minitest::Test 6 | TEST_CASES_DIR = File.join(__dir__, 'kdl-org/tests/test_cases') 7 | INPUTS_DIR = File.join(TEST_CASES_DIR, 'input') 8 | EXPECTED_DIR = File.join(TEST_CASES_DIR, 'expected_kdl') 9 | 10 | Dir.glob(File.join(INPUTS_DIR, '*.kdl')).each do |input_path| 11 | input_name = File.basename(input_path, '.kdl') 12 | expected_path = File.join(EXPECTED_DIR, "#{input_name}.kdl") 13 | if File.exist?(expected_path) 14 | define_method "test_#{input_name}_matches_expected_output" do 15 | expected = File.read(expected_path, encoding: Encoding::UTF_8) 16 | assert_equal expected, ::KDL.load_file(input_path, version: 2).to_s 17 | end 18 | else 19 | define_method "test_#{input_name}_does_not_parse" do 20 | err = assert_raises { ::KDL.load_file(input_path, version: 2) } 21 | raise err unless err.is_a?(KDL::Error) || err.is_a?(Racc::ParseError) 22 | end 23 | end 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /lib/kdl/string_dumper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module KDL 4 | module StringDumper 5 | def call(string) 6 | return string if bare_identifier?(string) 7 | 8 | %("#{string.each_char.map { |char| escape(char) }.join}") 9 | end 10 | 11 | private 12 | 13 | def escape(char) 14 | case char 15 | when "\n" then '\n' 16 | when "\r" then '\r' 17 | when "\t" then '\t' 18 | when '\\' then '\\\\' 19 | when '"' then '\"' 20 | when "\b" then '\b' 21 | when "\f" then '\f' 22 | else char 23 | end 24 | end 25 | 26 | FORBIDDEN = 27 | Tokenizer::SYMBOLS.keys + 28 | Tokenizer::WHITESPACE + 29 | Tokenizer::NEWLINES + 30 | "()[]/\\\"#".chars + 31 | ("\x0".."\x20").to_a 32 | 33 | def bare_identifier?(name) 34 | case name 35 | when '', 'true', 'false', 'null', '#true', '#false', '#null', /\A\.?\d/ 36 | false 37 | else 38 | !name.each_char.any? { |c| FORBIDDEN.include?(c) } 39 | end 40 | end 41 | 42 | extend self 43 | end 44 | end 45 | -------------------------------------------------------------------------------- /lib/kdl/error.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module KDL 4 | class Error < StandardError 5 | attr_reader :filename, :line, :column 6 | 7 | def initialize(message, filename = nil, line = nil, column = nil) 8 | message += " (#{line}:#{column})" if line 9 | message = "#{[filename, line, column].compact.join(':')}: #{message}" if filename 10 | super(message) 11 | @filename = filename 12 | @line = line 13 | @column = column 14 | end 15 | end 16 | 17 | class VersionMismatchError < Error 18 | attr_reader :version, :parser_version 19 | 20 | def initialize(message, version = nil, parser_version = nil, filename = nil) 21 | super(message, filename, 1, 1) 22 | @version = version 23 | @parser_version = parser_version 24 | end 25 | end 26 | 27 | class UnsupportedVersionError < Error 28 | attr_reader :version 29 | 30 | def initialize(message, version = nil, filename = nil) 31 | super(message, filename, 1, 1) 32 | @version = version 33 | end 34 | end 35 | 36 | class ParseError < Error; end 37 | end 38 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2020 Daniel Smith 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /test/v1/spec_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'test_helper' 4 | 5 | class KDL::V1::SpecTest < Minitest::Test 6 | TEST_CASES_DIR = File.join(__dir__, 'kdl-org/tests/test_cases') 7 | INPUTS_DIR = File.join(TEST_CASES_DIR, 'input') 8 | EXPECTED_DIR = File.join(TEST_CASES_DIR, 'expected_kdl') 9 | 10 | EXCLUDE = %w[ 11 | escline_comment_node 12 | ] 13 | 14 | Dir.glob(File.join(INPUTS_DIR, '*.kdl')).each do |input_path| 15 | input_name = File.basename(input_path, '.kdl') 16 | next if EXCLUDE.include?(input_name) 17 | expected_path = File.join(EXPECTED_DIR, "#{input_name}.kdl") 18 | if File.exist?(expected_path) 19 | define_method "test_v1_#{input_name}_matches_expected_output" do 20 | expected = File.read(expected_path, encoding: Encoding::UTF_8) 21 | assert_equal expected, ::KDL.load_file(input_path, version: 1).to_s 22 | end 23 | else 24 | define_method "test_v1_#{input_name}_does_not_parse" do 25 | err = assert_raises { ::KDL.load_file(input_path, version: 1) } 26 | raise err unless err.is_a?(KDL::Error) || err.is_a?(Racc::ParseError) 27 | end 28 | end 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /lib/kdl/types/hostname.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative './hostname/validator' 4 | 5 | module KDL 6 | module Types 7 | class Hostname < Value::Custom 8 | def self.call(value, type = 'hostname') 9 | return nil unless value.is_a? ::KDL::Value::String 10 | 11 | validator = Validator.new(value.value) 12 | raise ArgumentError, "invalid hostname #{value}" unless validator.valid? 13 | 14 | new(value.value, type: type) 15 | end 16 | end 17 | MAPPING['hostname'] = Hostname 18 | 19 | class IDNHostname < Hostname 20 | attr_reader :unicode_value 21 | 22 | def initialize(value, unicode_value:, **kwargs) 23 | super(value, **kwargs) 24 | @unicode_value = unicode_value 25 | end 26 | 27 | def self.call(value, type = 'idn-hostname') 28 | return nil unless value.is_a? ::KDL::Value::String 29 | 30 | validator = Validator.new(value.value) 31 | raise ArgumentError, "invalid hostname #{value}" unless validator.valid? 32 | 33 | new(validator.ascii, type: type, unicode_value: validator.unicode) 34 | end 35 | end 36 | MAPPING['idn-hostname'] = IDNHostname 37 | end 38 | end 39 | -------------------------------------------------------------------------------- /.github/workflows/ruby.yml: -------------------------------------------------------------------------------- 1 | # This workflow uses actions that are not certified by GitHub. 2 | # They are provided by a third-party and are governed by 3 | # separate terms of service, privacy policy, and support 4 | # documentation. 5 | # This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake 6 | # For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby 7 | 8 | name: Ruby 9 | 10 | on: 11 | push: 12 | branches: [ main ] 13 | pull_request: 14 | branches: [ main ] 15 | 16 | jobs: 17 | test: 18 | continue-on-error: ${{ matrix.ruby == 'head' }} 19 | strategy: 20 | matrix: 21 | ruby: [3.1, 3.2, 3.3, 3.4, head] 22 | 23 | runs-on: ubuntu-latest 24 | 25 | steps: 26 | - uses: actions/checkout@v2 27 | with: 28 | submodules: true 29 | 30 | - name: Set up Ruby 31 | uses: ruby/setup-ruby@v1 32 | with: 33 | ruby-version: ${{ matrix.ruby }} 34 | bundler-cache: true # install and cache dependencies 35 | 36 | - name: Build parser 37 | run: bundle exec racc lib/kdl/kdl.yy 38 | 39 | - name: Run tests 40 | run: bundle exec rake test 41 | 42 | - name: Report Coveralls 43 | uses: coverallsapp/github-action@v2 44 | -------------------------------------------------------------------------------- /lib/kdl/types/date_time.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'time' 4 | 5 | module KDL 6 | module Types 7 | class DateTime < Value::Custom 8 | def self.call(value, type = 'date-time') 9 | return nil unless value.is_a? ::KDL::Value::String 10 | 11 | time = ::Time.iso8601(value.value) 12 | new(time, type: type) 13 | end 14 | end 15 | MAPPING['date-time'] = DateTime 16 | 17 | class Time < Value::Custom 18 | # TODO: this is not a perfect ISO8601 time string 19 | REGEX = /^T?((?:2[0-3]|[01][0-9]):[0-5][0-9]:[0-5][0-9](?:\.[0-9]+)?(?:Z|[+-]\d\d:\d\d)?)$/ 20 | 21 | def self.call(value, type = 'time') 22 | return nil unless value.is_a? ::KDL::Value::String 23 | 24 | match = REGEX.match(value.value) 25 | raise ArgumentError, 'invalid time' if match.nil? 26 | 27 | time = ::Time.iso8601("#{::Date.today.iso8601}T#{match[1]}") 28 | new(time, type: type) 29 | end 30 | end 31 | MAPPING['time'] = Time 32 | 33 | class Date < Value::Custom 34 | def self.call(value, type = 'date') 35 | return nil unless value.is_a? ::KDL::Value::String 36 | 37 | date = ::Date.iso8601(value.value) 38 | new(date, type: type) 39 | end 40 | end 41 | MAPPING['date'] = Date 42 | end 43 | end 44 | -------------------------------------------------------------------------------- /lib/kdl/types/hostname/validator.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'simpleidn' 4 | 5 | module KDL 6 | module Types 7 | class Hostname < Value::Custom 8 | class Validator 9 | PART_RGX = /^[a-z0-9_][a-z0-9_\-]{0,62}$/i 10 | 11 | attr_reader :string 12 | alias ascii string 13 | alias unicode string 14 | 15 | def initialize(string) 16 | @string = string 17 | end 18 | 19 | def valid? 20 | return false if @string.length > 253 21 | 22 | @string.split('.').all? { |x| valid_part?(x) } 23 | end 24 | 25 | private 26 | 27 | def valid_part?(part) 28 | return false if part.empty? 29 | return false if part.start_with?('-') || part.end_with?('-') 30 | 31 | part =~ PART_RGX 32 | end 33 | end 34 | end 35 | 36 | class IDNHostname < Hostname 37 | class Validator < Hostname::Validator 38 | attr_reader :unicode 39 | 40 | def initialize(string) 41 | is_ascii = string.split('.').any? { |x| x.start_with?('xn--') } 42 | if is_ascii 43 | super(string) 44 | @unicode = SimpleIDN.to_unicode(string) 45 | else 46 | super(SimpleIDN.to_ascii(string)) 47 | @unicode = string 48 | end 49 | end 50 | end 51 | end 52 | end 53 | end 54 | -------------------------------------------------------------------------------- /lib/kdl/types/email.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative './email/parser' 4 | 5 | module KDL 6 | module Types 7 | class Email < Value::Custom 8 | attr_reader :local, :domain 9 | 10 | def initialize(value, local:, domain:, **kwargs) 11 | super(value, **kwargs) 12 | @local = local 13 | @domain = domain 14 | end 15 | 16 | def self.call(value, type = 'email') 17 | return nil unless value.is_a? ::KDL::Value::String 18 | 19 | local, domain = Parser.new(value.value).parse 20 | 21 | new(value.value, type: type, local: local, domain: domain) 22 | end 23 | 24 | end 25 | MAPPING['email'] = Email 26 | 27 | class IDNEmail < Email 28 | attr_reader :unicode_domain 29 | 30 | def initialize(value, unicode_domain:, **kwargs) 31 | super(value, **kwargs) 32 | @unicode_domain = unicode_domain 33 | end 34 | 35 | def self.call(value, type = 'email') 36 | return nil unless value.is_a? ::KDL::Value::String 37 | 38 | local, domain, unicode_domain = Email::Parser.new(value.value, idn: true).parse 39 | 40 | new("#{local}@#{domain}", type: type, local: local, domain: domain, unicode_domain: unicode_domain) 41 | end 42 | 43 | def unicode_value 44 | "#{local}@#{unicode_domain}" 45 | end 46 | end 47 | MAPPING['idn-email'] = IDNEmail 48 | end 49 | end 50 | -------------------------------------------------------------------------------- /kdl.gemspec: -------------------------------------------------------------------------------- 1 | require_relative 'lib/kdl/version' 2 | 3 | Gem::Specification.new do |spec| 4 | spec.name = "kdl" 5 | spec.version = KDL::VERSION 6 | spec.authors = ["Danielle Smith"] 7 | spec.email = ["danini@hey.com"] 8 | 9 | spec.summary = %q{KDL Document Language} 10 | spec.description = %q{Ruby implementation of the KDL Document Language Spec} 11 | spec.homepage = "https://kdl.dev" 12 | spec.license = "MIT" 13 | spec.required_ruby_version = Gem::Requirement.new(">= 3.1.0") 14 | 15 | spec.metadata["homepage_uri"] = spec.homepage 16 | spec.metadata["source_code_uri"] = "https://github.com/danini-the-panini/kdl-rb" 17 | spec.metadata["changelog_uri"] = "https://github.com/danini-the-panini/kdl-rb/releases" 18 | 19 | # Specify which files should be added to the gem when it is released. 20 | # The `git ls-files -z` loads the files in the RubyGem that have been added into git. 21 | spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do 22 | `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } 23 | end + ['lib/kdl/kdl.tab.rb', 'lib/kdl/v1/kdl.tab.rb'] 24 | spec.bindir = "exe" 25 | spec.require_paths = ["lib"] 26 | 27 | spec.add_dependency 'racc', '~> 1.5' 28 | spec.add_dependency 'simpleidn', '~> 0.2.1' 29 | spec.add_dependency 'bigdecimal', '~> 3.1.6' 30 | spec.add_dependency 'base64', '~> 0.2.0' 31 | end 32 | -------------------------------------------------------------------------------- /lib/kdl/types/irl.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative './irl/parser' 4 | 5 | module KDL 6 | module Types 7 | class IRLReference < Value::Custom 8 | attr_reader :unicode_value, 9 | :unicode_domain, 10 | :unicode_path, 11 | :unicode_search, 12 | :unicode_hash 13 | 14 | def initialize(value, unicode_value:, unicode_domain:, unicode_path:, unicode_search:, unicode_hash:, **kwargs) 15 | super(value, **kwargs) 16 | @unicode_value = unicode_value 17 | @unicode_domain = unicode_domain 18 | @unicode_path = unicode_path 19 | @unicode_search = unicode_search 20 | @unicode_hash = unicode_hash 21 | end 22 | 23 | def self.call(value, type = 'irl-reference') 24 | return nil unless value.is_a? ::KDL::Value::String 25 | 26 | ascii_value, params = parser(value.value).parse 27 | 28 | new(URI.parse(ascii_value), type: type, **params) 29 | end 30 | 31 | def self.parser(string) 32 | IRLReference::Parser.new(string) 33 | end 34 | end 35 | MAPPING['irl-reference'] = IRLReference 36 | 37 | class IRL < IRLReference 38 | def self.call(value, type = 'irl') 39 | super(value, type) 40 | end 41 | 42 | def self.parser(string) 43 | IRL::Parser.new(string) 44 | end 45 | end 46 | MAPPING['irl'] = IRL 47 | end 48 | end 49 | -------------------------------------------------------------------------------- /test/types/country_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'test_helper' 4 | 5 | class CountryTest < Minitest::Test 6 | def test_country3 7 | value = KDL::Types::Country3.call(::KDL::Value::String.new('ZAF')) 8 | assert_equal({ alpha3: 'ZAF', 9 | alpha2: 'ZA', 10 | numeric_code: 710, 11 | name: 'South Africa' }, value.value) 12 | 13 | assert_raises { KDL::Types::Country3.call(::KDL::Value::String.new('ZZZ')) } 14 | end 15 | 16 | def test_country2 17 | value = KDL::Types::Country2.call(::KDL::Value::String.new('ZA')) 18 | assert_equal({ alpha3: 'ZAF', 19 | alpha2: 'ZA', 20 | numeric_code: 710, 21 | name: 'South Africa' }, value.value) 22 | 23 | assert_raises { KDL::Types::Country2.call(::KDL::Value::String.new('ZZ')) } 24 | end 25 | 26 | def test_country_subdivision 27 | value = KDL::Types::CountrySubdivision.call(::KDL::Value::String.new('ZA-GP')) 28 | assert_equal('ZA-GP', value.value) 29 | assert_equal('Gauteng', value.name) 30 | assert_equal({ alpha3: 'ZAF', 31 | alpha2: 'ZA', 32 | numeric_code: 710, 33 | name: 'South Africa' }, value.country) 34 | 35 | assert_raises { KDL::Types::Country2.call(::KDL::Value::String.new('ZA-ZZ')) } 36 | assert_raises { KDL::Types::Country2.call(::KDL::Value::String.new('ZZ-GP')) } 37 | end 38 | end 39 | -------------------------------------------------------------------------------- /test/types/date_time_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "test_helper" 4 | 5 | class DateTimeTest < Minitest::Test 6 | def test_date_time 7 | value = KDL::Types::DateTime.call(::KDL::Value::String.new('2011-10-05T22:26:12-04:00')) 8 | assert_equal ::Time.iso8601('2011-10-05T22:26:12-04:00'), value.value 9 | 10 | assert_raises { KDL::Types::DateTime.call(::KDL::Value::String.new('not a date-time')) } 11 | end 12 | 13 | def test_time 14 | today = ::Date.today.iso8601 15 | value = KDL::Types::Time.call(::KDL::Value::String.new('22:26:12')) 16 | assert_equal ::Time.parse("#{today}T22:26:12"), value.value 17 | value = KDL::Types::Time.call(::KDL::Value::String.new('T22:26:12Z')) 18 | assert_equal ::Time.parse("#{today}T22:26:12Z"), value.value 19 | value = KDL::Types::Time.call(::KDL::Value::String.new('22:26:12.000Z')) 20 | assert_equal ::Time.parse("#{today}T22:26:12Z"), value.value 21 | value = KDL::Types::Time.call(::KDL::Value::String.new('22:26:12-04:00')) 22 | assert_equal ::Time.parse("#{today}T22:26:12-04:00"), value.value 23 | 24 | assert_raises { KDL::Types::DateTime.call(::KDL::Value::String.new('not a time')) } 25 | end 26 | 27 | def test_date 28 | value = KDL::Types::Date.call(::KDL::Value::String.new('2011-10-05')) 29 | assert_equal ::Date.iso8601('2011-10-05'), value.value 30 | 31 | assert_raises { KDL::Types::DateTime.call(::KDL::Value::String.new('not a date')) } 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /test/types/hostname_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "test_helper" 4 | 5 | class HostnameTest < Minitest::Test 6 | def test_hostname 7 | value = KDL::Types::Hostname.call(::KDL::Value::String.new('www.example.com')) 8 | assert_equal 'www.example.com', value.value 9 | refute_nil KDL::Types::Hostname.call(::KDL::Value::String.new('a'*63 + '.com')) 10 | refute_nil KDL::Types::Hostname.call(::KDL::Value::String.new([63, 63, 63, 61].map { |x| 'a' * x }.join('.'))) 11 | 12 | assert_raises { KDL::Types::Hostname.call(::KDL::Value::String.new('not a hostname')) } 13 | assert_raises { KDL::Types::Hostname.call(::KDL::Value::String.new('-starts-with-a-dash.com')) } 14 | assert_raises { KDL::Types::Hostname.call(::KDL::Value::String.new('a'*64 + '.com')) } 15 | assert_raises { KDL::Types::Hostname.call(::KDL::Value::String.new((['a' * 63] * 4).join('.'))) } 16 | end 17 | 18 | def test_idn_hostname 19 | value = KDL::Types::IDNHostname.call(::KDL::Value::String.new('xn--bcher-kva.example')) 20 | assert_equal 'xn--bcher-kva.example', value.value 21 | assert_equal 'bücher.example', value.unicode_value 22 | 23 | value = KDL::Types::IDNHostname.call(::KDL::Value::String.new('bücher.example')) 24 | assert_equal 'xn--bcher-kva.example', value.value 25 | assert_equal 'bücher.example', value.unicode_value 26 | 27 | assert_raises { KDL::Types::IDNHostname.call(::KDL::Value::String.new('not an idn hostname')) } 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /test/types/duration_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "test_helper" 4 | 5 | class DurationTest < Minitest::Test 6 | def test_duration 7 | value = KDL::Types::Duration.call(::KDL::Value::String.new('P3Y6M4DT12H30M5S')) 8 | assert_equal({ years: 3, months: 6, days: 4, hours: 12, minutes: 30, seconds: 5 }, value.value) 9 | value = KDL::Types::Duration.call(::KDL::Value::String.new('P23DT23H')) 10 | assert_equal({ days: 23, hours: 23 }, value.value) 11 | value = KDL::Types::Duration.call(::KDL::Value::String.new('P4Y')) 12 | assert_equal({ years: 4 }, value.value) 13 | value = KDL::Types::Duration.call(::KDL::Value::String.new('PT0S')) 14 | assert_equal({ seconds: 0 }, value.value) 15 | value = KDL::Types::Duration.call(::KDL::Value::String.new('P0D')) 16 | assert_equal({ days: 0 }, value.value) 17 | value = KDL::Types::Duration.call(::KDL::Value::String.new('P0.5Y')) 18 | assert_equal({ years: 0.5 }, value.value) 19 | value = KDL::Types::Duration.call(::KDL::Value::String.new('P0,5Y')) 20 | assert_equal({ years: 0.5 }, value.value) 21 | value = KDL::Types::Duration.call(::KDL::Value::String.new('P1M')) 22 | assert_equal({ months: 1 }, value.value) 23 | value = KDL::Types::Duration.call(::KDL::Value::String.new('PT1M')) 24 | assert_equal({ minutes: 1 }, value.value) 25 | value = KDL::Types::Duration.call(::KDL::Value::String.new('P7W')) 26 | assert_equal({ weeks: 7 }, value.value) 27 | 28 | assert_raises { KDL::Types::Duration.call(::KDL::Value::String.new('not a duration')) } 29 | assert_raises { KDL::Types::Duration.call(::KDL::Value::String.new('P')) } 30 | end 31 | end 32 | -------------------------------------------------------------------------------- /lib/kdl/document.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module KDL 4 | class Document 5 | include Enumerable 6 | 7 | attr_accessor :nodes 8 | 9 | def initialize(nodes = []) 10 | @nodes = nodes 11 | end 12 | 13 | def [](key) 14 | case key 15 | when Integer 16 | nodes[key] 17 | when String, Symbol 18 | nodes.find { _1.name == key.to_s } 19 | else 20 | raise ArgumentError, "document can only be indexed by Integer, String, or Symbol" 21 | end 22 | end 23 | 24 | def <<(node) 25 | nodes << node 26 | end 27 | 28 | def arg(key) 29 | self[key]&.arguments&.first&.value 30 | end 31 | 32 | def args(key) 33 | self[key]&.arguments&.map(&:value) 34 | end 35 | 36 | def each_arg(key, &block) 37 | args(key)&.each(&block) 38 | end 39 | 40 | def dash_vals(key) 41 | self[key] 42 | &.children 43 | &.select { _1.name == "-" } 44 | &.map { _1.arguments.first&.value } 45 | end 46 | 47 | def each_dash_val(key, &block) 48 | dash_vals(key)&.each(&block) 49 | end 50 | 51 | def each(&block) 52 | nodes.each(&block) 53 | end 54 | 55 | def to_s 56 | nodes.map(&:to_s).join("\n") + "\n" 57 | end 58 | 59 | def inspect 60 | nodes.map(&:inspect).join("\n") + "\n" 61 | end 62 | 63 | def ==(other) 64 | return false unless other.is_a?(Document) 65 | 66 | nodes == other.nodes 67 | end 68 | 69 | def version 70 | 2 71 | end 72 | 73 | def to_v2 74 | self 75 | end 76 | 77 | def to_v1 78 | KDL::V1::Document.new(nodes.map(&:to_v1)) 79 | end 80 | end 81 | end 82 | -------------------------------------------------------------------------------- /test/v1/value_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "test_helper" 4 | 5 | class KDL::V1::ValueTest < Minitest::Test 6 | def test_to_s 7 | assert_equal "1", ::KDL::V1::Value::Int.new(1).to_s 8 | assert_equal "1.5", ::KDL::V1::Value::Float.new(1.5).to_s 9 | assert_equal "null", ::KDL::V1::Value::Float.new(Float::INFINITY).to_s 10 | assert_equal "null", ::KDL::V1::Value::Float.new(-Float::INFINITY).to_s 11 | assert_equal "null", ::KDL::V1::Value::Float.new(Float::NAN).to_s 12 | assert_equal "true", ::KDL::V1::Value::Boolean.new(true).to_s 13 | assert_equal "false", ::KDL::V1::Value::Boolean.new(false).to_s 14 | assert_equal "null", ::KDL::V1::Value::Null.to_s 15 | assert_equal '"foo"', ::KDL::V1::Value::String.new("foo").to_s 16 | assert_equal '"foo \"bar\" baz"', ::KDL::V1::Value::String.new('foo "bar" baz').to_s 17 | assert_equal '(ty)"foo"', ::KDL::V1::Value::String.new("foo", type: 'ty').to_s 18 | end 19 | 20 | def test_from 21 | assert_equal(::KDL::V1::Value::Int.new(1), ::KDL::V1::Value::from(1)) 22 | assert_equal(::KDL::V1::Value::Float.new(1.5), ::KDL::V1::Value::from(1.5)) 23 | assert_equal( 24 | ::KDL::V1::Value::String.new("foo"), 25 | ::KDL::V1::Value::from("foo") 26 | ) 27 | assert_equal(::KDL::V1::Value::String.new("bar"), ::KDL::V1::Value::from("bar")) 28 | assert_equal(::KDL::V1::Value::Boolean.new(true), ::KDL::V1::Value::from(true)) 29 | assert_equal(::KDL::V1::Value::Null, ::KDL::V1::Value::from(nil)) 30 | assert_raises { ::KDL::V1::Value.from(Object.new) } 31 | end 32 | 33 | def test_equal 34 | assert(::KDL::V1::Value::String.new("foo") == ::KDL::Value::String.new("foo")) 35 | assert(::KDL::Value::String.new("foo") == ::KDL::V1::Value::String.new("foo")) 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /lib/kdl/builder.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module KDL 4 | class Builder < BasicObject 5 | class Error < ::KDL::Error; end 6 | 7 | def initialize 8 | @nesting = [] 9 | @document = Document.new 10 | end 11 | 12 | def document(&block) 13 | yield if block 14 | @document 15 | end 16 | 17 | def node(name = nil, *args, type: nil, **props, &block) 18 | n = Node.new(name&.to_s || "node", type:) 19 | @nesting << n 20 | args.each do |value| 21 | case value 22 | when ::Hash 23 | value.each { |k, v| prop k, v } 24 | else arg value 25 | end 26 | end 27 | props.each do |key, value| 28 | prop key, value 29 | end 30 | yield if block 31 | @nesting.pop 32 | if parent = current_node 33 | parent.children << n 34 | else 35 | @document << n 36 | end 37 | n 38 | end 39 | alias _ node 40 | 41 | def arg(value, type: nil) 42 | if n = current_node 43 | val = Value.from(value) 44 | val = val.as_type(type) if type 45 | n.arguments << val 46 | val 47 | else 48 | raise Error, "Can't do argument, not inside Node" 49 | end 50 | end 51 | 52 | def prop(key, value, type: nil) 53 | key = key.to_s 54 | if n = current_node 55 | val = Value.from(value) 56 | val = val.as_type(type) if type 57 | n.properties[key] = val 58 | val 59 | else 60 | raise Error, "Can't do property, not inside Node" 61 | end 62 | end 63 | 64 | def method_missing(name, *args, **props, &block) 65 | node name, *args, **props, &block 66 | end 67 | 68 | private 69 | 70 | def current_node 71 | return nil if @nesting.empty? 72 | 73 | @nesting.last 74 | end 75 | end 76 | end 77 | -------------------------------------------------------------------------------- /test/builder_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "test_helper" 4 | 5 | class BuilderTest < Minitest::Test 6 | def test_build 7 | doc = KDL.build do |kdl| 8 | kdl.node "foo" 9 | kdl.node "bar", type: "baz" 10 | kdl.node "qux" do 11 | kdl.arg 123 12 | kdl.prop "norf", "wat" 13 | kdl.prop "when", "2025-01-30", type: "date" 14 | kdl.node "child" 15 | end 16 | end 17 | 18 | assert_equal <<~KDL, doc.to_s 19 | foo 20 | (baz)bar 21 | qux 123 norf=wat when=(date)"2025-01-30" { 22 | child 23 | } 24 | KDL 25 | end 26 | 27 | def test_shorthand 28 | doc = KDL.build do |kdl| 29 | kdl.node "pokemon", "snorlax", { "Pokemon type" => "normal" }, "jigglypuff", level: 10, trainer: "Sylphrena" 30 | end 31 | 32 | assert_equal <<~KDL, doc.to_s 33 | pokemon snorlax jigglypuff "Pokemon type"=normal level=10 trainer=Sylphrena 34 | KDL 35 | end 36 | 37 | def test_implicit_block 38 | doc = KDL.build do 39 | node "foo" 40 | node "bar", type: "baz" 41 | node "qux" do 42 | arg 123 43 | prop "norf", "wat" 44 | prop "when", "2025-01-30", type: "date" 45 | node "child" 46 | end 47 | end 48 | 49 | assert_equal <<~KDL, doc.to_s 50 | foo 51 | (baz)bar 52 | qux 123 norf=wat when=(date)"2025-01-30" { 53 | child 54 | } 55 | KDL 56 | end 57 | 58 | def test_magic_nodes 59 | doc = KDL.build do 60 | foo 61 | bar type: "baz" 62 | qux do 63 | arg 123 64 | prop "norf", "wat" 65 | prop "when", "2025-01-30", type: "date" 66 | _ "child" 67 | end 68 | end 69 | 70 | assert_equal <<~KDL, doc.to_s 71 | foo 72 | (baz)bar 73 | qux 123 norf=wat when=(date)"2025-01-30" { 74 | child 75 | } 76 | KDL 77 | end 78 | 79 | def test_failures 80 | assert_raises do 81 | KDL.build do |kdl| 82 | kdl.prop foo: "bar" 83 | end 84 | end 85 | 86 | assert_raises do 87 | KDL.build do |kdl| 88 | kdl.arg "asdf" 89 | end 90 | end 91 | end 92 | 93 | end 94 | -------------------------------------------------------------------------------- /lib/kdl/v1/value.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module KDL 4 | module V1 5 | class Value < ::KDL::Value 6 | module Methods 7 | def to_s 8 | return stringify_value unless type 9 | 10 | "(#{StringDumper.stringify_identifier type})#{stringify_value}" 11 | end 12 | 13 | def ==(other) 14 | return self == other.value if other.is_a?(self.class.superclass) 15 | 16 | value == other 17 | end 18 | 19 | def version 20 | 1 21 | end 22 | 23 | def to_v1 24 | self 25 | end 26 | 27 | def to_v2 28 | self.class.superclass.new(value, format:, type:) 29 | end 30 | end 31 | 32 | include Methods 33 | 34 | class Int < ::KDL::Value::Int 35 | include Methods 36 | end 37 | 38 | class Float < ::KDL::Value::Float 39 | include Methods 40 | 41 | def stringify_value 42 | if value.nan? || value.infinite? 43 | warn "[WARNING] Attempting to serialize non-finite Float using KDL v1" 44 | return Null.stringify_value 45 | end 46 | super 47 | end 48 | end 49 | 50 | class Boolean < ::KDL::Value::Boolean 51 | include Methods 52 | 53 | def stringify_value 54 | value.to_s 55 | end 56 | end 57 | 58 | class String < ::KDL::Value::String 59 | include Methods 60 | 61 | def stringify_value 62 | StringDumper.call(value) 63 | end 64 | end 65 | 66 | class NullImpl < ::KDL::Value::NullImpl 67 | include Methods 68 | 69 | def stringify_value 70 | "null" 71 | end 72 | 73 | def to_v2 74 | type ? ::KDL::Value::NullImpl.new(type:) : ::KDL::Value::Null 75 | end 76 | end 77 | Null = NullImpl.new 78 | 79 | def self.from(value) 80 | case value 81 | when ::String then String.new(value) 82 | when Integer then Int.new(value) 83 | when ::Float then Float.new(value) 84 | when TrueClass, FalseClass then Boolean.new(value) 85 | when NilClass then Null 86 | else raise Error("unsupported value type: #{value.class}") 87 | end 88 | end 89 | end 90 | end 91 | end 92 | -------------------------------------------------------------------------------- /lib/kdl.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "kdl/version" 4 | require "kdl/error" 5 | require "kdl/tokenizer" 6 | require "kdl/document" 7 | require "kdl/value" 8 | require "kdl/node" 9 | require "kdl/string_dumper" 10 | require "kdl/types" 11 | require "kdl/parser_common" 12 | require "kdl/kdl.tab" 13 | require "kdl/builder" 14 | require "kdl/v1" 15 | 16 | module KDL 17 | class << self 18 | attr_accessor :default_version 19 | attr_accessor :default_output_version 20 | end 21 | 22 | def self.parse_document(input, options = {}) 23 | warn "[DEPRECATION] `KDL.parse_document' is deprecated. Please use `KDL.parse' instead." 24 | parse(input, **options) 25 | end 26 | 27 | def self.parse(input, version: default_version, output_version: default_output_version, filename: nil, **options) 28 | case version 29 | when 2 30 | Parser.new(output_module: output_module(output_version || 2), **options).parse(input, filename:) 31 | when 1 32 | V1::Parser.new.parse(input, output_module: output_module(output_version || 1), filename:, **options) 33 | when nil 34 | auto_parse(input, output_version:, **options) 35 | else 36 | raise UnsupportedVersionError.new("unsupported version '#{version}'", version) 37 | end 38 | end 39 | 40 | def self.load_file(filespec, **options) 41 | File.open(filespec, 'r:BOM|UTF-8') do |file| 42 | parse(file.read, **options, filename: file.to_path) 43 | end 44 | end 45 | 46 | def self.auto_parse(input, output_version: default_output_version, **options) 47 | parse(input, version: 2, output_version: output_version || 2, **options) 48 | rescue VersionMismatchError => e 49 | parse(input, version: e.version, output_version: output_version || e.version, **options) 50 | rescue ParseError => e 51 | parse(input, version: 1, output_version: output_version || 1, **options) rescue raise e 52 | end 53 | 54 | def self.output_module(version) 55 | case version 56 | when 1 then KDL::V1 57 | when 2 then KDL 58 | else 59 | warn "[WARNING] Unknown output_version '#{version}', defaulting to v2" 60 | KDL 61 | end 62 | end 63 | 64 | def self.build(&block) 65 | builder = Builder.new 66 | if block.arity >= 1 67 | builder.document do 68 | yield builder 69 | end 70 | else 71 | builder.instance_exec(&block) 72 | builder.document 73 | end 74 | end 75 | end 76 | -------------------------------------------------------------------------------- /lib/kdl/types/country.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'kdl/types/country/iso3166_countries' 4 | require 'kdl/types/country/iso3166_subdivisions' 5 | 6 | module KDL 7 | module Types 8 | class Country < Value::Custom 9 | attr_reader :name, :alpha2, :alpha3, :numeric_code 10 | 11 | def initialize(value, format: nil, type: 'country-3') 12 | super 13 | @name = value.fetch(:name, '') 14 | @alpha3 = value.fetch(:alpha3, nil) 15 | @alpha2 = value.fetch(:alpha2, nil) 16 | @numeric_code = value.fetch(:numeric_code, nil) 17 | end 18 | 19 | def self.call(value, type = 'country-3') 20 | return nil unless value.is_a? ::KDL::Value::String 21 | 22 | country = COUNTRIES3[value.value.upcase] 23 | raise ArgumentError, 'invalid country-3' if country.nil? 24 | 25 | new(country, type: type) 26 | end 27 | end 28 | Country3 = Country 29 | MAPPING['country-3'] = Country3 30 | 31 | class Country2 < Country 32 | def initialize(value, format: nil, type: 'country-2') 33 | super 34 | end 35 | 36 | def self.call(value, type = 'country-2') 37 | return nil unless value.is_a? ::KDL::Value::String 38 | 39 | country = COUNTRIES2[value.value.upcase] 40 | raise ArgumentError, 'invalid country-3' if country.nil? 41 | 42 | new(country, type: type) 43 | end 44 | end 45 | MAPPING['country-2'] = Country2 46 | 47 | class CountrySubdivision < Value::Custom 48 | attr_reader :country, :name 49 | 50 | def initialize(value, type: 'country-subdivision', country:, name:, **kwargs) 51 | super(value, type: type, **kwargs) 52 | @country = country 53 | @name = name 54 | end 55 | 56 | def self.call(value, type = 'country-subdivision') 57 | return nil unless value.is_a? ::KDL::Value::String 58 | 59 | country2 = value.value.split('-').first 60 | country = Country::COUNTRIES2[country2.upcase] 61 | raise ArgumentError, 'invalid country' unless country 62 | 63 | subdivision = COUNTRY_SUBDIVISIONS.dig(country2.upcase, value.value) 64 | raise ArgumentError, 'invalid country subdivision' unless subdivision 65 | 66 | new(value.value, type: type, name: subdivision, country: country) 67 | end 68 | end 69 | MAPPING['country-subdivision'] = CountrySubdivision 70 | end 71 | end 72 | -------------------------------------------------------------------------------- /test/types/irl_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "test_helper" 4 | 5 | class IRLTest < Minitest::Test 6 | def test_irl 7 | value = KDL::Types::IRL.call(::KDL::Value::String.new('https://bücher.example/foo/Ῥόδος')) 8 | assert_equal URI('https://xn--bcher-kva.example/foo/%E1%BF%AC%CF%8C%CE%B4%CE%BF%CF%82'), value.value 9 | assert_equal 'https://bücher.example/foo/Ῥόδος', value.unicode_value 10 | value = KDL::Types::IRL.call(::KDL::Value::String.new('https://xn--bcher-kva.example/foo/%E1%BF%AC%CF%8C%CE%B4%CE%BF%CF%82')) 11 | assert_equal URI('https://xn--bcher-kva.example/foo/%E1%BF%AC%CF%8C%CE%B4%CE%BF%CF%82'), value.value 12 | assert_equal 'https://bücher.example/foo/Ῥόδος', value.unicode_value 13 | value = KDL::Types::IRL.call(::KDL::Value::String.new('https://bücher.example/foo/Ῥόδος?🌈=✔️#🦄')) 14 | assert_equal URI('https://xn--bcher-kva.example/foo/%E1%BF%AC%CF%8C%CE%B4%CE%BF%CF%82?%F0%9F%8C%88=%E2%9C%94%EF%B8%8F#%F0%9F%A6%84'), value.value 15 | assert_equal 'https://bücher.example/foo/Ῥόδος?🌈=✔️#🦄', value.unicode_value 16 | 17 | assert_raises { KDL::Types::IRL.call(::KDL::Value::String.new('not a url')) } 18 | assert_raises { KDL::Types::IRL.call(::KDL::Value::String.new('/reference/to/something')) } 19 | end 20 | 21 | def test_irl_reference 22 | value = KDL::Types::IRLReference.call(::KDL::Value::String.new('https://bücher.example/foo/Ῥόδος')) 23 | assert_equal URI('https://xn--bcher-kva.example/foo/%E1%BF%AC%CF%8C%CE%B4%CE%BF%CF%82'), value.value 24 | assert_equal 'https://bücher.example/foo/Ῥόδος', value.unicode_value 25 | value = KDL::Types::IRLReference.call(::KDL::Value::String.new('https://xn--bcher-kva.example/foo/%E1%BF%AC%CF%8C%CE%B4%CE%BF%CF%82')) 26 | assert_equal URI('https://xn--bcher-kva.example/foo/%E1%BF%AC%CF%8C%CE%B4%CE%BF%CF%82'), value.value 27 | assert_equal 'https://bücher.example/foo/Ῥόδος', value.unicode_value 28 | value = KDL::Types::IRLReference.call(::KDL::Value::String.new('/foo/Ῥόδος')) 29 | assert_equal URI('/foo/%E1%BF%AC%CF%8C%CE%B4%CE%BF%CF%82'), value.value 30 | assert_equal '/foo/Ῥόδος', value.unicode_value 31 | value = KDL::Types::IRLReference.call(::KDL::Value::String.new('/foo/%E1%BF%AC%CF%8C%CE%B4%CE%BF%CF%82')) 32 | assert_equal URI('/foo/%E1%BF%AC%CF%8C%CE%B4%CE%BF%CF%82'), value.value 33 | assert_equal '/foo/Ῥόδος', value.unicode_value 34 | 35 | assert_raises { KDL::Types::IRLReference.call(::KDL::Value::String.new('not a url reference')) } 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /test/v1/examples_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "test_helper" 4 | 5 | class KDL::V1::ExamplesTest < Minitest::Test 6 | def example_path(name) 7 | File.join(__dir__, "kdl-org/examples/#{name}.kdl") 8 | end 9 | 10 | def test_ci 11 | doc = ::KDL.load_file(example_path('ci')) 12 | nodes = KDL.build { 13 | name "CI" 14 | on "push", "pull_request" 15 | env { 16 | RUSTFLAGS "-Dwarnings" 17 | } 18 | jobs { 19 | fmt_and_docs("Check fmt & build docs") { 20 | _ "runs-on", "ubuntu-latest" 21 | steps { 22 | step uses: "actions/checkout@v1" 23 | step("Install Rust", uses: "actions-rs/toolchain@v1") { 24 | profile "minimal" 25 | toolchain "stable" 26 | components "rustfmt" 27 | override true 28 | } 29 | step "rustfmt", run: "cargo fmt --all -- --check" 30 | step "docs", run: "cargo doc --no-deps" 31 | } 32 | } 33 | build_and_test("Build & Test") { 34 | _ "runs-on", "${{ matrix.os }}" 35 | strategy { 36 | matrix { 37 | rust "1.46.0", "stable" 38 | os "ubuntu-latest", "macOS-latest", "windows-latest" 39 | } 40 | } 41 | 42 | steps { 43 | step uses: "actions/checkout@v1" 44 | step("Install Rust", uses: "actions-rs/toolchain@v1") { 45 | profile "minimal" 46 | toolchain "${{ matrix.rust }}" 47 | components "clippy" 48 | override true 49 | } 50 | step "Clippy", run: "cargo clippy --all -- -D warnings" 51 | step "Run tests", run: "cargo test --all --verbose" 52 | } 53 | } 54 | } 55 | } 56 | assert_equal nodes, doc 57 | end 58 | 59 | def test_cargo 60 | doc = ::KDL.load_file(example_path('Cargo')) 61 | nodes = KDL.build { 62 | package { 63 | name "kdl" 64 | version "0.0.0" 65 | description "kat's document language" 66 | authors "Kat Marchán " 67 | _ "license-file", "LICENSE.md" 68 | edition "2018" 69 | } 70 | dependencies { 71 | nom "6.0.1" 72 | thiserror "1.0.22" 73 | } 74 | } 75 | assert_equal nodes, doc 76 | end 77 | 78 | def test_nuget 79 | doc = ::KDL.load_file(example_path('nuget')) 80 | # This file is particularly large. It would be nice to validate it, but for now 81 | # I'm just going to settle for making sure it parses. 82 | refute_nil doc 83 | end 84 | 85 | def test_kdl_schema 86 | doc = ::KDL.load_file(example_path('kdl-schema')) 87 | # This file is particularly large. It would be nice to validate it, but for now 88 | # I'm just going to settle for making sure it parses. 89 | refute_nil doc 90 | end 91 | 92 | def test_website 93 | doc = ::KDL.load_file(example_path('website')) 94 | # This file is particularly large. It would be nice to validate it, but for now 95 | # I'm just going to settle for making sure it parses. 96 | refute_nil doc 97 | end 98 | end 99 | -------------------------------------------------------------------------------- /test/types/email_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "test_helper" 4 | 5 | class EmailTest < Minitest::Test 6 | def test_email 7 | value = KDL::Types::Email.call(::KDL::Value::String.new('danielle@example.com')) 8 | assert_equal 'danielle@example.com', value.value 9 | assert_equal 'danielle', value.local 10 | assert_equal 'example.com', value.domain 11 | 12 | assert_raises { KDL::Types::Email.call(::KDL::Value::String.new('not an email')) } 13 | end 14 | 15 | VALID_EMAILS = [ 16 | ['simple@example.com', 'simple', 'example.com'], 17 | ['very.common@example.com', 'very.common', 'example.com'], 18 | ['disposable.style.email.with+symbol@example.com', 'disposable.style.email.with+symbol', 'example.com'], 19 | ['other.email-with-hyphen@example.com', 'other.email-with-hyphen', 'example.com'], 20 | ['fully-qualified-domain@example.com', 'fully-qualified-domain', 'example.com'], 21 | ['user.name+tag+sorting@example.com', 'user.name+tag+sorting', 'example.com'], 22 | ['x@example.com', 'x', 'example.com'], 23 | ['example-indeed@strange-example.com', 'example-indeed', 'strange-example.com'], 24 | ['test/test@test.com', 'test/test', 'test.com'], 25 | ['admin@mailserver1', 'admin', 'mailserver1'], 26 | ['example@s.example', 'example', 's.example'], 27 | ['" "@example.org', ' ', 'example.org'], 28 | ['"john..doe"@example.org', 'john..doe', 'example.org'], 29 | ['mailhost!username@example.org', 'mailhost!username', 'example.org'], 30 | ['user%example.com@example.org', 'user%example.com', 'example.org'], 31 | ['user-@example.org', 'user-', 'example.org'] 32 | ] 33 | 34 | def test_valid_emails 35 | VALID_EMAILS.each do |email, local, domain| 36 | value = KDL::Types::Email.call(::KDL::Value::String.new(email)) 37 | assert_equal email, value.value 38 | assert_equal local, value.local 39 | assert_equal domain, value.domain 40 | end 41 | end 42 | 43 | INVALID_EMAILS = [ 44 | 'Abc.example.com', 45 | 'A@b@c@example.com', 46 | 'a"b(c)d,e:f;gi[j\k]l@example.com', 47 | 'just"not"right@example.com', 48 | 'this is"not\allowed@example.com', 49 | 'this\ still\"not\\allowed@example.com', 50 | '1234567890123456789012345678901234567890123456789012345678901234+x@example.com', 51 | '-some-user-@-example-.com', 52 | 'QA🦄CHOCOLATE🌈@test.com' 53 | ] 54 | 55 | def test_invalid_emails 56 | INVALID_EMAILS.each do |email| 57 | assert_raises { KDL::Types::Email.call(::KDL::Value::String.new(email)) } 58 | end 59 | end 60 | 61 | def test_idn_email 62 | value = KDL::Types::IDNEmail.call(::KDL::Value::String.new('🌈@xn--9ckb.com')) 63 | assert_equal '🌈@xn--9ckb.com', value.value 64 | assert_equal '🌈@ツッ.com', value.unicode_value 65 | assert_equal '🌈', value.local 66 | assert_equal 'ツッ.com', value.unicode_domain 67 | assert_equal 'xn--9ckb.com', value.domain 68 | value = KDL::Types::IDNEmail.call(::KDL::Value::String.new('🌈@ツッ.com')) 69 | assert_equal '🌈@xn--9ckb.com', value.value 70 | assert_equal '🌈@ツッ.com', value.unicode_value 71 | assert_equal '🌈', value.local 72 | assert_equal 'ツッ.com', value.unicode_domain 73 | assert_equal 'xn--9ckb.com', value.domain 74 | 75 | assert_raises { KDL::Types::IDNEmail.call(::KDL::Value::String.new('not an email')) } 76 | end 77 | end 78 | -------------------------------------------------------------------------------- /test/examples_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "test_helper" 4 | 5 | class ExamplesTest < Minitest::Test 6 | def example_path(name) 7 | File.join(__dir__, "kdl-org/examples/#{name}.kdl") 8 | end 9 | 10 | def test_ci 11 | doc = ::KDL.load_file(example_path('ci')) 12 | nodes = KDL.build { 13 | name "CI" 14 | on "push", "pull_request" 15 | env { 16 | RUSTFLAGS("-Dwarnings") 17 | } 18 | jobs { 19 | fmt_and_docs("Check fmt & build docs") { 20 | _ "runs-on", "ubuntu-latest" 21 | steps { 22 | step uses: "actions/checkout@v1" 23 | step("Install Rust", uses: "actions-rs/toolchain@v1") { 24 | profile "minimal" 25 | toolchain "stable" 26 | components "rustfmt" 27 | override true 28 | } 29 | step("rustfmt") { run "cargo", "fmt", "--all", "--", "--check" } 30 | step("docs") { run "cargo", "doc", "--no-deps" } 31 | } 32 | } 33 | build_and_test("Build & Test") { 34 | _ "runs-on", "${{ matrix.os }}" 35 | strategy { 36 | matrix { 37 | rust "1.46.0", "stable" 38 | os "ubuntu-latest", "macOS-latest", "windows-latest" 39 | } 40 | } 41 | 42 | steps { 43 | step uses: "actions/checkout@v1" 44 | step("Install Rust", uses: "actions-rs/toolchain@v1") { 45 | profile "minimal" 46 | toolchain "${{ matrix.rust }}" 47 | components "clippy" 48 | override true 49 | } 50 | step("Clippy") { run "cargo", "clippy", "--all", "--", "-D", "warnings" } 51 | step("Run tests") { run "cargo", "test", "--all", "--verbose" } 52 | step "Other Stuff", run: "echo foo\necho bar\necho baz" 53 | } 54 | } 55 | } 56 | } 57 | assert_equal nodes, doc 58 | end 59 | 60 | def test_cargo 61 | doc = ::KDL.load_file(example_path('Cargo')) 62 | nodes = KDL.build { 63 | package { 64 | name "kdl" 65 | version "0.0.0" 66 | description "The kdl document language" 67 | authors "Kat Marchán " 68 | _ "license-file", "LICENSE.md" 69 | edition "2018" 70 | } 71 | dependencies { 72 | nom "6.0.1" 73 | thiserror "1.0.22" 74 | } 75 | } 76 | assert_equal nodes, doc 77 | end 78 | 79 | def test_nuget 80 | doc = ::KDL.load_file(example_path('nuget')) 81 | # This file is particularly large. It would be nice to validate it, but for now 82 | # I'm just going to settle for making sure it parses. 83 | refute_nil doc 84 | end 85 | 86 | def test_kdl_schema 87 | doc = ::KDL.load_file(example_path('kdl-schema')) 88 | # This file is particularly large. It would be nice to validate it, but for now 89 | # I'm just going to settle for making sure it parses. 90 | refute_nil doc 91 | end 92 | 93 | def test_website 94 | doc = ::KDL.load_file(example_path('website')) 95 | # This file is particularly large. It would be nice to validate it, but for now 96 | # I'm just going to settle for making sure it parses. 97 | refute_nil doc 98 | end 99 | end 100 | -------------------------------------------------------------------------------- /test/document_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "test_helper" 4 | 5 | class DocumentTest < Minitest::Test 6 | 7 | def test_ref 8 | doc = KDL::Document.new([ 9 | KDL::Node.new("foo"), 10 | KDL::Node.new("bar") 11 | ]) 12 | 13 | assert_equal doc.nodes[0], doc[0] 14 | assert_equal doc.nodes[1], doc[1] 15 | 16 | assert_equal doc.nodes[0], doc["foo"] 17 | assert_equal doc.nodes[0], doc[:foo] 18 | assert_equal doc.nodes[1], doc[:bar] 19 | 20 | assert_raises { doc[nil] } 21 | end 22 | 23 | def test_arg 24 | doc = KDL::Document.new([ 25 | KDL::Node.new("foo", arguments: [KDL::Value::String.new("bar")]), 26 | KDL::Node.new("baz", arguments: [KDL::Value::String.new("qux")]) 27 | ]) 28 | 29 | assert_equal "bar", doc.arg(0) 30 | assert_equal "bar", doc.arg("foo") 31 | assert_equal "bar", doc.arg(:foo) 32 | assert_equal "qux", doc.arg(1) 33 | assert_equal "qux", doc.arg(:baz) 34 | assert_nil doc.arg(:norf) 35 | 36 | assert_raises { doc.arg(nil) } 37 | end 38 | 39 | def test_args 40 | doc = KDL::Document.new([ 41 | KDL::Node.new("foo", arguments: [KDL::Value::String.new("bar"), KDL::Value::String.new("baz")]), 42 | KDL::Node.new("qux", arguments: [KDL::Value::String.new("norf")]) 43 | ]) 44 | 45 | assert_equal ["bar", "baz"], doc.args(0) 46 | assert_equal ["bar", "baz"], doc.args("foo") 47 | assert_equal ["bar", "baz"], doc.args(:foo) 48 | assert_equal ["norf"], doc.args(1) 49 | assert_equal ["norf"], doc.args(:qux) 50 | assert_nil doc.args(:wat) 51 | 52 | a = []; doc.each_arg("foo") { a << _1 } 53 | assert_equal ["bar", "baz"], a 54 | 55 | a = []; doc.each_arg(:wat) { a << _1 } 56 | assert_equal [], a 57 | 58 | assert_raises { doc.arg(nil) } 59 | end 60 | 61 | def test_dash_vals 62 | doc = KDL::Document.new([ 63 | KDL::Node.new("node", children: [ 64 | KDL::Node.new("-", arguments: [KDL::Value::String.new("foo")]), 65 | KDL::Node.new("-", arguments: [KDL::Value::String.new("bar")]), 66 | KDL::Node.new("-", arguments: [KDL::Value::String.new("baz")]) 67 | ]) 68 | ]) 69 | 70 | assert_equal ["foo", "bar", "baz"], doc.dash_vals(0) 71 | assert_equal ["foo", "bar", "baz"], doc.dash_vals("node") 72 | assert_equal ["foo", "bar", "baz"], doc.dash_vals(:node) 73 | assert_nil doc.dash_vals(:nope) 74 | 75 | a = []; doc.each_dash_val("node") { a << _1 } 76 | assert_equal ["foo", "bar", "baz"], a 77 | 78 | a = []; doc.each_dash_val(:nope) { a << _1 } 79 | assert_equal [], a 80 | 81 | assert_raises { doc.dash_vals(nil) } 82 | end 83 | 84 | def test_each 85 | doc = KDL::Document.new([ 86 | KDL::Node.new("foo"), 87 | KDL::Node.new("bar") 88 | ]) 89 | 90 | a = []; doc.each { a << _1.name } 91 | assert_equal ["foo", "bar"], a 92 | end 93 | 94 | def test_inspect 95 | doc = KDL::Document.new([]) 96 | 97 | assert_kind_of String, doc.inspect 98 | end 99 | 100 | def test_version 101 | assert_equal 2, KDL::Document.new([]).version 102 | end 103 | 104 | def test_to_v1 105 | doc = KDL.parse <<~KDL, version: 2 106 | foo lorem 1 #true #null { 107 | bar """ 108 | baz 109 | qux 110 | """ 111 | } 112 | KDL 113 | assert_equal 2, doc.version 114 | 115 | doc = doc.to_v1 116 | assert_equal 1, doc.version 117 | 118 | assert_equal <<~KDL, doc.to_s 119 | foo "lorem" 1 true null { 120 | bar " baz\\n qux" 121 | } 122 | KDL 123 | end 124 | 125 | def test_to_v2 126 | doc = KDL::Document.new([]) 127 | assert_same doc, doc.to_v2 128 | end 129 | 130 | end 131 | -------------------------------------------------------------------------------- /test/types_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'test_helper' 4 | 5 | class TypesTest < Minitest::Test 6 | def test_types 7 | doc = KDL.parse <<-KDL 8 | node (date-time)"2021-01-01T12:12:12" \\ 9 | (date)"2021-01-01" \\ 10 | (time)"22:23:12" \\ 11 | (duration)"P3Y6M4DT12H30M5S" \\ 12 | (currency)"ZAR" \\ 13 | (country-3)"ZAF" \\ 14 | (country-2)"ZA" \\ 15 | (country-subdivision)"ZA-GP" \\ 16 | (ipv4)"127.0.0.1" \\ 17 | (ipv6)"3ffe:505:2::1" \\ 18 | (url)"https://kdl.dev" \\ 19 | (url-reference)"/foo/bar" \\ 20 | (uuid)"f81d4fae-7dec-11d0-a765-00a0c91e6bf6" \\ 21 | (regex)"asdf" \\ 22 | (base64)"U2VuZCByZWluZm9yY2VtZW50cw==" \\ 23 | (decimal)"10000000000000" \\ 24 | (hostname)"www.example.com" \\ 25 | (idn-hostname)"xn--bcher-kva.example" \\ 26 | (email)"simple@example.com" \\ 27 | (idn-email)"🌈@xn--9ckb.com" \\ 28 | (irl)"https://kdl.dev/🦄" \\ 29 | (irl-reference)"/🌈/🦄" \\ 30 | (url-template)"https://kdl.dev/{foo}" 31 | KDL 32 | 33 | refute_nil doc 34 | i = -1 35 | assert_kind_of ::KDL::Types::DateTime, doc.nodes.first.arguments[i += 1] 36 | assert_kind_of ::KDL::Types::Date, doc.nodes.first.arguments[i += 1] 37 | assert_kind_of ::KDL::Types::Time, doc.nodes.first.arguments[i += 1] 38 | assert_kind_of ::KDL::Types::Duration, doc.nodes.first.arguments[i += 1] 39 | assert_kind_of ::KDL::Types::Currency, doc.nodes.first.arguments[i += 1] 40 | assert_kind_of ::KDL::Types::Country3, doc.nodes.first.arguments[i += 1] 41 | assert_kind_of ::KDL::Types::Country2, doc.nodes.first.arguments[i += 1] 42 | assert_kind_of ::KDL::Types::CountrySubdivision, doc.nodes.first.arguments[i += 1] 43 | assert_kind_of ::KDL::Types::IPV4, doc.nodes.first.arguments[i += 1] 44 | assert_kind_of ::KDL::Types::IPV6, doc.nodes.first.arguments[i += 1] 45 | assert_kind_of ::KDL::Types::URL, doc.nodes.first.arguments[i += 1] 46 | assert_kind_of ::KDL::Types::URLReference, doc.nodes.first.arguments[i += 1] 47 | assert_kind_of ::KDL::Types::UUID, doc.nodes.first.arguments[i += 1] 48 | assert_kind_of ::KDL::Types::Regex, doc.nodes.first.arguments[i += 1] 49 | assert_kind_of ::KDL::Types::Base64, doc.nodes.first.arguments[i += 1] 50 | assert_kind_of ::KDL::Types::Decimal, doc.nodes.first.arguments[i += 1] 51 | assert_kind_of ::KDL::Types::Hostname, doc.nodes.first.arguments[i += 1] 52 | assert_kind_of ::KDL::Types::IDNHostname, doc.nodes.first.arguments[i += 1] 53 | assert_kind_of ::KDL::Types::Email, doc.nodes.first.arguments[i += 1] 54 | assert_kind_of ::KDL::Types::IDNEmail, doc.nodes.first.arguments[i += 1] 55 | assert_kind_of ::KDL::Types::IRL, doc.nodes.first.arguments[i += 1] 56 | assert_kind_of ::KDL::Types::IRLReference, doc.nodes.first.arguments[i += 1] 57 | assert_kind_of ::KDL::Types::URLTemplate, doc.nodes.first.arguments[i += 1] 58 | end 59 | 60 | def test_custom_types 61 | parsers = { 62 | 'foo' => lambda { |value, type| 63 | Foo.new(value.value, type: type) if value.is_a?(KDL::Value) 64 | }, 65 | 'bar' => lambda { |node, type| 66 | Bar.new(node, type: type) if node.is_a?(KDL::Node) 67 | } 68 | } 69 | doc = KDL.parse <<-KDL, type_parsers: parsers 70 | (bar)barnode (foo)"foovalue" 71 | (foo)foonode (bar)"barvalue" 72 | KDL 73 | refute_nil doc 74 | assert_kind_of Bar, doc.nodes.first 75 | assert_kind_of Foo, doc.nodes.first.arguments.first 76 | assert_kind_of KDL::Node, doc.nodes[1] 77 | assert_kind_of KDL::Value, doc.nodes[1].arguments.first 78 | end 79 | 80 | def test_parse_false 81 | doc = KDL.parse <<-KDL, parse_types: false 82 | node (date-time)"2021-01-01T12:12:12" 83 | KDL 84 | 85 | refute_nil doc 86 | assert_kind_of ::KDL::Value::String, doc.nodes.first.arguments.first 87 | end 88 | 89 | class Foo < KDL::Value::Custom 90 | end 91 | 92 | class Bar < KDL::Node::Custom 93 | def initialize(node, type: nil) 94 | super(node.name, arguments: node.arguments, properties: node.properties, children: node.children, type: type) 95 | end 96 | end 97 | end 98 | -------------------------------------------------------------------------------- /lib/kdl/v1/kdl.yy: -------------------------------------------------------------------------------- 1 | class KDL::V1::Parser 2 | options no_result_var 3 | token IDENT STRING RAWSTRING 4 | INTEGER FLOAT TRUE FALSE NULL 5 | WS NEWLINE 6 | LBRACE RBRACE 7 | LPAREN RPAREN 8 | EQUALS 9 | SEMICOLON 10 | EOF 11 | SLASHDASH 12 | rule 13 | document : nodes { @output_module::Document.new(val[0]) } 14 | | linespaces { @output_module::Document.new([]) } 15 | 16 | nodes : none { [] } 17 | | linespaces node { [val[1]] } 18 | | linespaces empty_node { [] } 19 | | nodes node { [*val[0], val[1]] } 20 | | nodes empty_node { val[0] } 21 | node : unterm_node node_term { val[0] } 22 | unterm_node : untyped_node { val[0] } 23 | | type untyped_node { val[1].as_type(val[0], @type_parsers.fetch(val[0], nil)) } 24 | untyped_node : node_decl { val[0].tap { |x| x.children = [] } } 25 | | node_decl node_children { val[0].tap { |x| x.children = val[1] } } 26 | | node_decl empty_childrens { val[0].tap { |x| x.children = [] } } 27 | | node_decl empty_childrens node_children { val[0].tap { |x| x.children = val[2] } } 28 | | node_decl node_children empty_childrens { val[0].tap { |x| x.children = val[1] } } 29 | | node_decl empty_childrens node_children empty_childrens { val[0].tap { |x| x.children = val[2] } } 30 | node_decl : identifier { @output_module::Node.new(val[0]) } 31 | | node_decl ws_plus value { val[0].tap { |x| x.arguments << val[2] } } 32 | | node_decl ws_plus slashdash value { val[0] } 33 | | node_decl ws_plus property { val[0].tap { |x| x.properties[val[2][0]] = val[2][1] } } 34 | | node_decl ws_plus slashdash property { val[0] } 35 | | node_decl ws_plus { val[0] } 36 | empty_node : slashdash node 37 | node_children : ws_star LBRACE nodes RBRACE { val[2] } 38 | | ws_star LBRACE linespaces RBRACE { [] } 39 | | ws_star LBRACE nodes unterm_node ws_star RBRACE { [*val[2], val[3]] } 40 | | ws_star LBRACE linespaces unterm_node ws_star RBRACE { [val[3]] } 41 | empty_children : slashdash node_children 42 | | ws_plus empty_children 43 | empty_childrens: empty_children | empty_childrens empty_children 44 | node_term: linespaces | semicolon_term 45 | semicolon_term: SEMICOLON | SEMICOLON linespaces 46 | slashdash: SLASHDASH | slashdash ws_plus | slashdash linespaces 47 | 48 | type : LPAREN ws_star identifier ws_star RPAREN { val[2] } 49 | 50 | identifier : IDENT { val[0].value } 51 | | STRING { val[0].value } 52 | | RAWSTRING { val[0].value } 53 | 54 | property : identifier EQUALS value { [val[0], val[2]] } 55 | 56 | value : untyped_value 57 | | type untyped_value { val[1].as_type(val[0], @type_parsers.fetch(val[0], nil)) } 58 | 59 | untyped_value : STRING { @output_module::Value::String.new(val[0].value) } 60 | | RAWSTRING { @output_module::Value::String.new(val[0].value) } 61 | | INTEGER { @output_module::Value::Int.new(val[0].value) } 62 | | FLOAT { @output_module::Value::Float.new(val[0].value, format: val[0].meta[:format]) } 63 | | boolean { @output_module::Value::Boolean.new(val[0]) } 64 | | NULL { @output_module::Value::Null } 65 | 66 | boolean : TRUE { true } 67 | | FALSE { false } 68 | 69 | ws_plus: WS | WS ws_plus 70 | ws_star: none | ws_plus 71 | linespace: WS | NEWLINE | EOF 72 | linespaces: linespace | linespaces linespace 73 | 74 | none: { nil } 75 | 76 | ---- inner 77 | 78 | include KDL::ParserCommon 79 | 80 | def parser_version 81 | 1 82 | end 83 | 84 | def parse(str, filename: nil, **options) 85 | init(**options) 86 | @tokenizer = ::KDL::V1::Tokenizer.new(str, filename:) 87 | check_version 88 | do_parse 89 | end 90 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # KDL 2 | 3 | [![Gem Version](https://badge.fury.io/rb/kdl.svg)](https://badge.fury.io/rb/kdl) 4 | [![Actions Status](https://github.com/danini-the-panini/kdl-rb/workflows/Ruby/badge.svg)](https://github.com/danini-the-panini/kdl-rb/actions) 5 | [![Coverage Status](https://coveralls.io/repos/github/danini-the-panini/kdl-rb/badge.svg?branch=main)](https://coveralls.io/github/danini-the-panini/kdl-rb?branch=main) 6 | 7 | This is a Ruby implementation of the [KDL Document Language](https://kdl.dev) 8 | 9 | ## Installation 10 | 11 | Add this line to your application's Gemfile: 12 | 13 | ```ruby 14 | gem 'kdl' 15 | ``` 16 | 17 | And then execute: 18 | 19 | $ bundle install 20 | 21 | Or install it yourself as: 22 | 23 | $ gem install kdl 24 | 25 | ## Usage 26 | 27 | ```ruby 28 | require 'kdl' 29 | 30 | KDL.parse(a_string) #=> KDL::Document 31 | KDL.load_file('path/to/file') #=> KDL::Document 32 | ``` 33 | 34 | You can optionally provide your own type annotation handlers: 35 | 36 | ```ruby 37 | class Foo < KDL::Value::Custom 38 | end 39 | 40 | KDL.parse(a_string, type_parsers: { 41 | 'foo' => Foo 42 | }) 43 | ``` 44 | 45 | The `foo` custom type will be called with instances of Value or Node with the type annotation `(foo)`. 46 | 47 | Custom types are expected to have a `call` method that takes the Value or Node, and the type annotation itself, as arguments, and is expected to return either an instance of `KDL::Value::Custom` or `KDL::Node::Custom` (depending on the input type) or `nil` to return the original value as is. Take a look at [the built in custom types](lib/kdl/types) as a reference. 48 | 49 | You can also disable type annotation parsing entirely (including the built in ones): 50 | 51 | ```ruby 52 | KDL.parse(a_string, parse_types: false) 53 | ``` 54 | 55 | ## KDL v1 56 | 57 | kdl-rb maintains backwards compatibility with the KDL v1 spec. By default, KDL will attempt to parse a file with the v1 parser if it fails to parse with v2. This behaviour can be changed by specifying the `version` option: 58 | 59 | ```ruby 60 | KDL.parse(a_string, version: 2) 61 | ``` 62 | 63 | The resulting document will also serialize back to the same version it was parsed as. For example, if you parse a v2 document and call `to_s` on it, it will output a v2 document, and similarly with v1. This behaviour can be changed by specifying the `output_version` option: 64 | 65 | ```ruby 66 | KDL.parse(a_string, output_version: 2) 67 | ``` 68 | 69 | This allows you to to convert documents between versions: 70 | 71 | ```ruby 72 | KDL.parse('foo "bar" true', version: 1, output_version: 2).to_s #=> 'foo bar #true' 73 | ``` 74 | 75 | You can also convert an already parsed document between versions with `to_v1` and `to_v2`: 76 | 77 | ```ruby 78 | doc = KDL.parse('foo "bar" true', version: 1) 79 | doc.version #=> 1 80 | doc.to_v2.to_s #=> 'foo bar #true' 81 | ``` 82 | 83 | You can also set the default version globally: 84 | 85 | ```ruby 86 | KDL.default_version = 2 87 | KDL.default_output_version = 2 88 | ``` 89 | 90 | You can still force automatic version detection with `auto_parse`: 91 | 92 | ```ruby 93 | KDL.default_version = 2 94 | KDL.parse('foo "bar" true') #=> Error 95 | KDL.auto_parse('foo "bar" true') #=> KDL::V1::Document 96 | ``` 97 | 98 | Version directives are also respected: 99 | 100 | ```ruby 101 | KDL.parse("/- kdl-version 2\nfoo bar", version: 1) 102 | #=> Version mismatch, document specified v2, but this is a v1 parser (Racc::ParseError) 103 | ``` 104 | 105 | ## Development 106 | 107 | After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment. 108 | 109 | To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org). 110 | 111 | ## Contributing 112 | 113 | Bug reports and pull requests are welcome on GitHub at https://github.com/danini-the-panini/kdl-rb. 114 | 115 | 116 | ## License 117 | 118 | The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT). 119 | -------------------------------------------------------------------------------- /lib/kdl/node.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module KDL 4 | class Node 5 | class Custom < Node 6 | def self.call(node, type) 7 | new(node.name, arguments: node.arguments, properties: node.properties, children: node.children, type:) 8 | end 9 | 10 | def version 11 | nil 12 | end 13 | 14 | def to_v1 15 | self 16 | end 17 | 18 | def to_v2 19 | self 20 | end 21 | end 22 | 23 | include Enumerable 24 | 25 | attr_accessor :name, :arguments, :properties, :children, :type 26 | 27 | def initialize(name, _args = [], _props = {}, _children = [], 28 | arguments: _args, 29 | properties: _props, 30 | children: _children, 31 | type: nil 32 | ) 33 | @name = name 34 | @arguments = arguments 35 | @properties = properties.transform_keys(&:to_s) 36 | @children = children 37 | @type = type 38 | end 39 | 40 | def [](key) 41 | case key 42 | when Integer 43 | arguments[key]&.value 44 | when String, Symbol 45 | properties[key.to_s]&.value 46 | else 47 | raise ArgumentError, "node can only be indexed by Integer, String, or Symbol" 48 | end 49 | end 50 | 51 | def <<(node) 52 | children << node 53 | end 54 | 55 | def child(key) 56 | case key 57 | when Integer 58 | children[key] 59 | when String, Symbol 60 | children.find { _1.name == key.to_s } 61 | else 62 | raise ArgumentError, "node can only be indexed by Integer, String, or Symbol" 63 | end 64 | end 65 | 66 | def arg(key) 67 | child(key)&.arguments&.first&.value 68 | end 69 | 70 | def args(key) 71 | child(key)&.arguments&.map(&:value) 72 | end 73 | 74 | def each_arg(key, &block) 75 | args(key)&.each(&block) 76 | end 77 | 78 | def dash_vals(key) 79 | child(key) 80 | &.children 81 | &.select { _1.name == "-" } 82 | &.map { _1.arguments.first&.value } 83 | end 84 | 85 | def each_dash_val(key, &block) 86 | dash_vals(key)&.each(&block) 87 | end 88 | 89 | def each(&block) 90 | children.each(&block) 91 | end 92 | 93 | def <=>(other) 94 | name <=> other.name 95 | end 96 | 97 | def to_s(level = 0, m = :to_s) 98 | indent = ' ' * level 99 | s = "#{indent}#{type ? "(#{id_to_s type, m })" : ''}#{id_to_s name, m}" 100 | unless arguments.empty? 101 | s << " #{arguments.map(&m).join(' ')}" 102 | end 103 | unless properties.empty? 104 | s << " #{properties.map { |k, v| "#{id_to_s k, m}=#{v.public_send(m)}" }.join(' ')}" 105 | end 106 | unless children.empty? 107 | s << " {\n" 108 | s << children.map { |c| "#{c.public_send(m, level + 1)}" }.join("\n") 109 | s << "\n#{indent}}" 110 | end 111 | s 112 | end 113 | 114 | def inspect(level = 0) 115 | to_s(level, :inspect) 116 | end 117 | 118 | def ==(other) 119 | return false unless other.is_a?(Node) 120 | 121 | name == other.name && 122 | arguments == other.arguments && 123 | properties == other.properties && 124 | children == other.children 125 | end 126 | 127 | def as_type(type, parser = nil) 128 | if parser.nil? 129 | @type = type 130 | self 131 | else 132 | result = parser.call(self, type) 133 | 134 | return self.as_type(type) if result.nil? 135 | 136 | unless result.is_a?(::KDL::Node::Custom) 137 | raise ArgumentError, "expected parser to return an instance of ::KDL::Node::Custom, got `#{result.class}'" 138 | end 139 | 140 | result 141 | end 142 | end 143 | 144 | def version 145 | 2 146 | end 147 | 148 | def to_v2 149 | self 150 | end 151 | 152 | def to_v1 153 | ::KDL::V1::Node.new(name, 154 | arguments: arguments.map(&:to_v1), 155 | properties: properties.transform_values(&:to_v1), 156 | children: children.map(&:to_v1), 157 | type: type 158 | ) 159 | end 160 | 161 | private 162 | 163 | def id_to_s(id, m = :to_s) 164 | return id.public_send(m) unless m == :to_s 165 | 166 | StringDumper.call(id.to_s) 167 | end 168 | end 169 | end 170 | -------------------------------------------------------------------------------- /lib/kdl/kdl.yy: -------------------------------------------------------------------------------- 1 | class KDL::Parser 2 | options no_result_var 3 | token IDENT STRING RAWSTRING 4 | INTEGER FLOAT TRUE FALSE NULL 5 | WS NEWLINE 6 | LBRACE RBRACE 7 | LPAREN RPAREN 8 | EQUALS 9 | SEMICOLON 10 | EOF 11 | SLASHDASH 12 | rule 13 | document : nodes { @output_module::Document.new(val[0]) } 14 | | linespaces { @output_module::Document.new([]) } 15 | 16 | nodes : none { [] } 17 | | linespaces node { [val[1]] } 18 | | linespaces empty_node { [] } 19 | | nodes node { [*val[0], val[1]] } 20 | | nodes empty_node { val[0] } 21 | node : unterm_node node_term { val[0] } 22 | unterm_node : untyped_node { val[0] } 23 | | type untyped_node { val[1].as_type(val[0], @type_parsers.fetch(val[0], nil)) } 24 | untyped_node : node_decl { val[0].tap { |x| x.children = [] } } 25 | | node_decl ws_star node_children { val[0].tap { |x| x.children = val[2] } } 26 | | node_decl ws_star empty_childrens { val[0].tap { |x| x.children = [] } } 27 | | node_decl ws_star empty_childrens node_children { val[0].tap { |x| x.children = val[3] } } 28 | | node_decl ws_star node_children empty_childrens { val[0].tap { |x| x.children = val[2] } } 29 | | node_decl ws_star empty_childrens node_children empty_childrens { val[0].tap { |x| x.children = val[3] } } 30 | node_decl : identifier { @output_module::Node.new(val[0]) } 31 | | node_decl ws_plus value { val[0].tap { |x| x.arguments << val[2] } } 32 | | node_decl ws_star slashdash value { val[0] } 33 | | node_decl ws_plus property { val[0].tap { |x| x.properties[val[2][0]] = val[2][1] } } 34 | | node_decl ws_star slashdash property { val[0] } 35 | | node_decl ws_plus { val[0] } 36 | empty_node : slashdash node 37 | node_children : ws_star LBRACE nodes RBRACE { val[2] } 38 | | ws_star LBRACE linespaces RBRACE { [] } 39 | | ws_star LBRACE nodes unterm_node ws_star RBRACE { [*val[2], val[3]] } 40 | | ws_star LBRACE linespaces unterm_node ws_star RBRACE { [val[3]] } 41 | empty_children : slashdash node_children 42 | | ws_plus empty_children 43 | empty_childrens: empty_children | empty_childrens empty_children 44 | node_term: linespaces | semicolon_term 45 | semicolon_term: SEMICOLON | SEMICOLON linespaces 46 | slashdash: SLASHDASH | slashdash linespaces 47 | 48 | type : LPAREN ws_star identifier ws_star RPAREN ws_star { val[2] } 49 | 50 | identifier : IDENT { val[0].value } 51 | | STRING { val[0].value } 52 | | RAWSTRING { val[0].value } 53 | 54 | property : identifier EQUALS value { [val[0], val[2]] } 55 | 56 | value : untyped_value 57 | | type untyped_value { val[1].as_type(val[0], @type_parsers.fetch(val[0], nil)) } 58 | 59 | untyped_value : IDENT { @output_module::Value::String.new(val[0].value) } 60 | | STRING { @output_module::Value::String.new(val[0].value) } 61 | | RAWSTRING { @output_module::Value::String.new(val[0].value) } 62 | | INTEGER { @output_module::Value::Int.new(val[0].value) } 63 | | FLOAT { @output_module::Value::Float.new(val[0].value, format: val[0].meta[:format]) } 64 | | boolean { @output_module::Value::Boolean.new(val[0]) } 65 | | NULL { @output_module::Value::Null } 66 | 67 | boolean : TRUE { true } 68 | | FALSE { false } 69 | 70 | ws_plus: WS | WS ws_plus 71 | ws_star: none | ws_plus 72 | linespace: WS | NEWLINE | EOF 73 | linespaces: linespace | linespaces linespace 74 | 75 | none: { nil } 76 | 77 | ---- inner 78 | 79 | include KDL::ParserCommon 80 | 81 | def initialize(**options) 82 | init(**options) 83 | end 84 | 85 | def parser_version 86 | 2 87 | end 88 | 89 | def parse(str, filename: nil) 90 | @tokenizer = ::KDL::Tokenizer.new(str, filename:) 91 | check_version 92 | do_parse 93 | end 94 | -------------------------------------------------------------------------------- /lib/kdl/value.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module KDL 4 | class Value 5 | attr_reader :value, :format, :type 6 | 7 | def initialize(value, format: nil, type: nil) 8 | @value = value 9 | @format = format 10 | @type = type 11 | end 12 | 13 | def as_type(type, parser = nil) 14 | if parser.nil? 15 | self.class.new(value, format: format, type: type) 16 | else 17 | result = parser.call(self, type) 18 | return self.as_type(type) if result.nil? 19 | 20 | unless result.is_a?(::KDL::Value::Custom) 21 | raise ArgumentError, "expected parser to return an instance of ::KDL::Value::Custom, got `#{result.class}'" 22 | end 23 | 24 | result 25 | end 26 | end 27 | 28 | def ==(other) 29 | return self == other.value if other.is_a?(self.class) 30 | 31 | value == other 32 | end 33 | 34 | def to_s 35 | return stringify_value unless type 36 | 37 | "(#{StringDumper.call type})#{stringify_value}" 38 | end 39 | 40 | def inspect 41 | return value.inspect unless type 42 | 43 | "(#{type.inspect})#{value.inspect}" 44 | end 45 | 46 | def stringify_value 47 | return format % value if format 48 | 49 | value.to_s 50 | end 51 | 52 | def version 53 | 2 54 | end 55 | 56 | def to_v2 57 | self 58 | end 59 | 60 | def method_missing(name, *args, **kwargs, &block) 61 | value.public_send(name, *args, **kwargs, &block) 62 | end 63 | 64 | def respond_to_missing?(name, include_all = false) 65 | value.respond_to?(name, include_all) 66 | end 67 | 68 | class Int < Value 69 | def to_v1 70 | V1::Value::Int.new(value, format:, type:) 71 | end 72 | end 73 | 74 | class Float < Value 75 | def ==(other) 76 | return self == other.value if other.is_a?(Float) 77 | return other.nan? if value.nan? 78 | 79 | value == other 80 | end 81 | 82 | def stringify_value 83 | return '#nan' if value.nan? 84 | return '#inf' if value == ::Float::INFINITY 85 | return '#-inf' if value == -::Float::INFINITY 86 | return super.upcase unless value.is_a?(BigDecimal) 87 | 88 | sign, digits, _, exponent = value.split 89 | s = +'' 90 | s << '-' if sign.negative? 91 | s << "#{digits[0]}.#{digits[1..-1]}" 92 | s << "E#{exponent.negative? ? '' : '+'}#{exponent - 1}" 93 | s 94 | end 95 | 96 | def to_v1 97 | if value.nan? || value.infinite? 98 | warn "[WARNING] Converting non-finite Float to KDL v1" 99 | end 100 | V1::Value::Float.new(value, format:, type:) 101 | end 102 | end 103 | 104 | class Boolean < Value 105 | def stringify_value 106 | "##{value}" 107 | end 108 | 109 | def to_v1 110 | V1::Value::Boolean.new(value, format:, type:) 111 | end 112 | end 113 | 114 | class String < Value 115 | def stringify_value 116 | StringDumper.call(value) 117 | end 118 | 119 | def to_v1 120 | V1::Value::String.new(value, format:, type:) 121 | end 122 | end 123 | 124 | class NullImpl < Value 125 | def initialize(_=nil, format: nil, type: nil) 126 | super(nil, type: type) 127 | end 128 | 129 | def stringify_value 130 | "#null" 131 | end 132 | 133 | def ==(other) 134 | other.is_a?(NullImpl) || other.nil? 135 | end 136 | 137 | def to_v1 138 | type ? V1::Value::NullImpl.new(type:) : V1::Value::Null 139 | end 140 | end 141 | Null = NullImpl.new 142 | 143 | class Custom < Value 144 | attr_reader :oriinal_value 145 | 146 | def self.call(value, type) 147 | new(value, type:) 148 | end 149 | 150 | def version 151 | nil 152 | end 153 | 154 | def to_v1 155 | self 156 | end 157 | 158 | def to_v2 159 | self 160 | end 161 | end 162 | 163 | def self.from(value) 164 | case value 165 | when ::String then String.new(value) 166 | when Integer then Int.new(value) 167 | when ::Float then Float.new(value) 168 | when TrueClass, FalseClass then Boolean.new(value) 169 | when NilClass then Null 170 | else raise Error, "unsupported value type: #{value.class}" 171 | end 172 | end 173 | end 174 | end 175 | -------------------------------------------------------------------------------- /lib/kdl/types/irl/parser.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module KDL 4 | module Types 5 | class IRLReference < Value::Custom 6 | class Parser 7 | RGX = /^(?:(?:([a-z][a-z0-9+.\-]+)):\/\/([^@]+@)?([^\/?#]+)?)?(\/?[^?#]*)?(?:\?([^#]*))?(?:#(.*))?$/i.freeze 8 | PERCENT_RGX = /%[a-f0-9]{2}/i.freeze 9 | 10 | RESERVED_URL_CHARS = %w[! # $ & ' ( ) * + , / : ; = ? @ \[ \] %] 11 | UNRESERVED_URL_CHARS = %w[A B C D E F G H I J K L M N O P Q R S T U V W X Y Z 12 | a b c d e f g h i j k l m n o p q r s t u v w x y z 13 | 0 1 2 3 4 5 6 7 8 9 - _ . ~].freeze 14 | URL_CHARS = RESERVED_URL_CHARS + UNRESERVED_URL_CHARS 15 | 16 | def initialize(string) 17 | @string = string 18 | end 19 | 20 | def parse 21 | scheme, auth, domain, path, search, hash = *parse_url 22 | 23 | if @string.ascii_only? 24 | unicode_path = Parser.decode(path) 25 | unicode_search = Parser.decode(search) 26 | unicode_hash = Parser.decode(hash) 27 | else 28 | unicode_path = path 29 | path = Parser.encode(unicode_path) 30 | unicode_search = search 31 | search_params = unicode_search ? unicode_search.split('&').map { |x| x.split('=') } : nil 32 | search = search_params ? search_params.map { |k, v| "#{Parser.encode(k)}=#{Parser.encode(v)}" }.join('&') : nil 33 | unicode_hash = hash 34 | hash = Parser.encode(hash) 35 | end 36 | 37 | if domain 38 | validator = IDNHostname::Validator.new(domain) 39 | domain = validator.ascii 40 | unicode_domain = validator.unicode 41 | else 42 | unicode_domain = domain 43 | end 44 | 45 | unicode_value = Parser.build_uri_string(scheme, auth, unicode_domain, unicode_path, unicode_search, unicode_hash) 46 | ascii_value = Parser.build_uri_string(scheme, auth, domain, path, search, hash) 47 | 48 | [ascii_value, 49 | { unicode_value: unicode_value, 50 | unicode_domain: unicode_domain, 51 | unicode_path: unicode_path, 52 | unicode_search: unicode_search, 53 | unicode_hash: unicode_hash }] 54 | end 55 | 56 | def parse_url 57 | match = RGX.match(@string) 58 | raise ArgumentError, "invalid IRL `#{@string}'" if match.nil? 59 | 60 | _, *parts = *match 61 | raise ArgumentError, "invalid IRL `#{@string}'" unless parts.all? { |part| Parser.valid_url_part?(part) } 62 | 63 | parts 64 | end 65 | 66 | def self.valid_url_part?(string) 67 | return true unless string 68 | 69 | string.chars.all? do |char| 70 | !char.ascii_only? || URL_CHARS.include?(char) 71 | end 72 | end 73 | 74 | def self.encode(string) 75 | return string unless string 76 | 77 | string.chars 78 | .map { |c| c.ascii_only? ? c : percent_encode(c) } 79 | .join 80 | .force_encoding('utf-8') 81 | end 82 | 83 | def self.decode(string) 84 | return string unless string 85 | 86 | string.gsub(PERCENT_RGX) do |match| 87 | char = match[1, 2].to_i(16).chr 88 | if RESERVED_URL_CHARS.include?(char) 89 | match 90 | else 91 | char 92 | end 93 | end.force_encoding('utf-8') 94 | end 95 | 96 | def self.percent_encode(c) 97 | c.bytes.map { |b| "%#{b.to_s(16)}" }.join.upcase 98 | end 99 | 100 | def self.build_uri_string(scheme, auth, domain, path, search, hash) 101 | string = +'' 102 | string << "#{scheme}://" if scheme 103 | string << auth if auth 104 | string << domain if domain 105 | string << path if path 106 | string << "?#{search}" if search 107 | string << "##{hash}" if hash 108 | string 109 | end 110 | end 111 | end 112 | 113 | class IRL < IRLReference 114 | class Parser < IRLReference::Parser 115 | def parse_url 116 | parts = super 117 | scheme, * = parts 118 | raise ArgumentError, "invalid IRL `#{@string}'" if scheme.nil? || scheme.empty? 119 | 120 | parts 121 | end 122 | end 123 | end 124 | end 125 | end 126 | -------------------------------------------------------------------------------- /lib/kdl/types/email/parser.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative '../hostname/validator' 4 | 5 | module KDL 6 | module Types 7 | class Email < Value::Custom 8 | class Parser 9 | def initialize(string, idn: false) 10 | @string = string 11 | @idn = idn 12 | @tokenizer = Tokenizer.new(string, idn: idn) 13 | end 14 | 15 | def parse 16 | local = +'' 17 | unicode_domain = nil 18 | domain = nil 19 | context = :start 20 | 21 | loop do 22 | type, value = @tokenizer.next_token 23 | 24 | case type 25 | when :part 26 | case context 27 | when :start, :after_dot 28 | local << value 29 | context = :after_part 30 | else 31 | raise ArgumentError, "invalid email #{@string} (unexpected part #{value} at #{context})" 32 | end 33 | when :dot 34 | case context 35 | when :after_part 36 | local << value 37 | context = :after_dot 38 | else 39 | raise ArgumentError, "invalid email #{@string} (unexpected dot at #{context})" 40 | end 41 | when :at 42 | case context 43 | when :after_part 44 | context = :after_at 45 | end 46 | when :domain 47 | case context 48 | when :after_at 49 | validator = (@idn ? IDNHostname : Hostname)::Validator.new(value) 50 | raise ArgumentError, "invalid hostname #{value}" unless validator.valid? 51 | 52 | unicode_domain = validator.unicode 53 | domain = validator.ascii 54 | context = :after_domain 55 | else 56 | raise ArgumentError, "invalid email #{@string} (unexpected domain at #{context})" 57 | end 58 | when :end 59 | case context 60 | when :after_domain 61 | if local.size > 64 62 | raise ArgumentError, "invalid email #{@string} (local part length #{local.size} exceeds maximaum of 64)" 63 | end 64 | 65 | return [local, domain, unicode_domain] 66 | else 67 | raise ArgumentError, "invalid email #{@string} (unexpected end at #{context})" 68 | end 69 | end 70 | end 71 | end 72 | end 73 | 74 | class Tokenizer 75 | LOCAL_PART_ASCII = %r{[a-zA-Z0-9!#$%&'*+\-/=?^_`{|}~]}.freeze 76 | LOCAL_PART_IDN = /[^\x00-\x1f ".@]/.freeze 77 | 78 | def initialize(string, idn: false) 79 | @string = string 80 | @idn = idn 81 | @index = 0 82 | @after_at = false 83 | end 84 | 85 | def next_token 86 | if @after_at 87 | if @index < @string.size 88 | domain_start = @index 89 | @index = @string.size 90 | return [:domain, @string[domain_start..-1]] 91 | else 92 | return [:end, nil] 93 | end 94 | end 95 | @context = nil 96 | @buffer = +'' 97 | loop do 98 | c = @string[@index] 99 | return [:end, nil] if c.nil? 100 | 101 | case @context 102 | when nil 103 | case c 104 | when '.' 105 | @index += 1 106 | return [:dot, '.'] 107 | when '@' 108 | @after_at = true 109 | @index += 1 110 | return [:at, '@'] 111 | when '"' 112 | @context = :quote 113 | @index += 1 114 | when local_part_chars 115 | @context = :part 116 | @buffer << c 117 | @index += 1 118 | else 119 | raise ArgumentError, "invalid email #{@string} (unexpected #{c})" 120 | end 121 | when :part 122 | case c 123 | when local_part_chars 124 | @buffer << c 125 | @index += 1 126 | when '.', '@' 127 | return [:part, @buffer] 128 | else 129 | raise ArgumentError, "invalid email #{@string} (unexpected #{c})" 130 | end 131 | when :quote 132 | case c 133 | when '"' 134 | n = @string[@index + 1] 135 | raise ArgumentError, "invalid email #{@string} (unexpected #{c})" unless n == '.' || n == '@' 136 | 137 | @index += 1 138 | return [:part, @buffer] 139 | else 140 | @buffer << c 141 | @index += 1 142 | end 143 | end 144 | end 145 | end 146 | 147 | def local_part_chars 148 | @idn ? LOCAL_PART_IDN : LOCAL_PART_ASCII 149 | end 150 | end 151 | end 152 | end 153 | end 154 | -------------------------------------------------------------------------------- /test/value_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "test_helper" 4 | 5 | class ValueTest < Minitest::Test 6 | def test_to_s 7 | assert_equal "1", ::KDL::Value::Int.new(1).to_s 8 | assert_equal "1.5", ::KDL::Value::Float.new(1.5).to_s 9 | assert_equal "#inf", ::KDL::Value::Float.new(Float::INFINITY).to_s 10 | assert_equal "#-inf", ::KDL::Value::Float.new(-Float::INFINITY).to_s 11 | assert_equal "#nan", ::KDL::Value::Float.new(Float::NAN).to_s 12 | assert_equal "#true", ::KDL::Value::Boolean.new(true).to_s 13 | assert_equal "#false", ::KDL::Value::Boolean.new(false).to_s 14 | assert_equal "#null", ::KDL::Value::Null.to_s 15 | assert_equal 'foo', ::KDL::Value::String.new("foo").to_s 16 | assert_equal '"foo \"bar\" baz"', ::KDL::Value::String.new('foo "bar" baz').to_s 17 | assert_equal '"false"', ::KDL::Value::String.new("false").to_s 18 | assert_equal '(ty)foo', ::KDL::Value::String.new("foo", type: 'ty').to_s 19 | end 20 | 21 | def test_from 22 | assert_equal(KDL::Value::Int.new(1), KDL::Value::from(1)) 23 | assert_equal(KDL::Value::Float.new(1.5), KDL::Value::from(1.5)) 24 | assert_equal( 25 | KDL::Value::String.new("foo"), 26 | KDL::Value::from("foo") 27 | ) 28 | assert_equal(KDL::Value::String.new("bar"), KDL::Value::from("bar")) 29 | assert_equal(KDL::Value::Boolean.new(true), KDL::Value::from(true)) 30 | assert_equal(KDL::Value::Null, KDL::Value::from(nil)) 31 | assert_raises { ::KDL::Value.from(Object.new) } 32 | end 33 | 34 | def test_equal 35 | assert_equal ::KDL::Value::Int.new(42), ::KDL::Value::Int.new(42) 36 | assert_equal ::KDL::Value::Float.new(3.14), ::KDL::Value::Float.new(3.14) 37 | assert_equal ::KDL::Value::Float.new(::Float::NAN), ::KDL::Value::Float.new(::Float::NAN) 38 | assert_equal ::KDL::Value::Boolean.new(true), ::KDL::Value::Boolean.new(true) 39 | assert_equal ::KDL::Value::NullImpl.new, ::KDL::Value::NullImpl.new 40 | assert_equal ::KDL::Value::String.new("lorem"), ::KDL::Value::String.new("lorem") 41 | 42 | assert_equal ::KDL::Value::Int.new(42), 42 43 | assert_equal ::KDL::Value::Float.new(3.14), 3.14 44 | assert_equal ::KDL::Value::Boolean.new(true), true 45 | assert_equal ::KDL::Value::NullImpl.new, nil 46 | assert_equal ::KDL::Value::String.new("lorem"), "lorem" 47 | 48 | refute_equal ::KDL::Value::Int.new(69), ::KDL::Value::Int.new(42) 49 | refute_equal ::KDL::Value::Float.new(6.28), ::KDL::Value::Float.new(3.14) 50 | refute_equal ::KDL::Value::Boolean.new(false), ::KDL::Value::Boolean.new(true) 51 | refute_equal ::KDL::Value::String.new("ipsum"), ::KDL::Value::String.new("lorem") 52 | 53 | refute_equal ::KDL::Value::Int.new(42), 69 54 | refute_equal ::KDL::Value::Float.new(3.14), 6.28 55 | refute_equal ::KDL::Value::Boolean.new(true), false 56 | refute_equal ::KDL::Value::NullImpl.new, 7 57 | refute_equal ::KDL::Value::String.new("lorem"), "ipsum" 58 | end 59 | 60 | class Something < KDL::Value::Custom 61 | end 62 | 63 | def test_as_type 64 | value = ::KDL::Value::String.new("foo") 65 | assert_equal "bar", value.as_type("bar").type 66 | assert_kind_of Something, value.as_type("bar", lambda { |v, type| Something.new(v) }) 67 | nil_parse = value.as_type("bar", lambda { |v, type| nil }) 68 | assert_equal value, nil_parse 69 | assert_equal "bar", nil_parse.type 70 | 71 | assert_raises { value.as_type("bar", lambda { |v, type| Object.new }) } 72 | end 73 | 74 | def test_inspect 75 | assert_equal "1", ::KDL::Value::Int.new(1).inspect 76 | assert_equal "1.5", ::KDL::Value::Float.new(1.5).inspect 77 | assert_equal "true", ::KDL::Value::Boolean.new(true).inspect 78 | assert_equal "false", ::KDL::Value::Boolean.new(false).inspect 79 | assert_equal "nil", ::KDL::Value::Null.inspect 80 | assert_equal '"foo"', ::KDL::Value::String.new("foo").inspect 81 | assert_equal '"foo \"bar\" baz"', ::KDL::Value::String.new('foo "bar" baz').inspect 82 | assert_equal '("ty")"foo"', ::KDL::Value::String.new("foo", type: 'ty').inspect 83 | end 84 | 85 | def test_version 86 | assert_equal 2, ::KDL::Value::Int.new(1).version 87 | assert_equal 2, ::KDL::Value::Float.new(1.5).version 88 | assert_equal 2, ::KDL::Value::Boolean.new(true).version 89 | assert_equal 2, ::KDL::Value::Boolean.new(false).version 90 | assert_equal 2, ::KDL::Value::Null.version 91 | assert_equal 2, ::KDL::Value::String.new("foo").version 92 | end 93 | 94 | def test_to_v1 95 | [ 96 | ::KDL::Value::Int.new(1), 97 | ::KDL::Value::Float.new(1.5), 98 | ::KDL::Value::Boolean.new(true), 99 | ::KDL::Value::Boolean.new(false), 100 | ::KDL::Value::Null, 101 | ::KDL::Value::String.new("foo") 102 | ].each do |v| 103 | v1 = v.to_v1 104 | assert_equal 1, v1.version 105 | assert_equal v, v1 106 | assert_equal v1, v 107 | end 108 | end 109 | 110 | def test_to_v2 111 | [ 112 | ::KDL::Value::Int.new(1), 113 | ::KDL::Value::Float.new(1.5), 114 | ::KDL::Value::Boolean.new(true), 115 | ::KDL::Value::Boolean.new(false), 116 | ::KDL::Value::Null, 117 | ::KDL::Value::String.new("foo") 118 | ].each do |v| 119 | assert_same v, v.to_v2 120 | end 121 | end 122 | 123 | def test_method_missing 124 | v = ::KDL::Value::String.new("foo") 125 | 126 | assert v.respond_to?(:upcase) 127 | assert_equal "FOO", v.upcase 128 | end 129 | end 130 | -------------------------------------------------------------------------------- /lib/kdl/types/duration/iso8601_parser.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # Shamelessly borrowed from https://github.com/rails/rails/tree/main/activesupport 4 | # 5 | # Copyright (c) 2005-2021 David Heinemeier Hansson 6 | 7 | # Permission is hereby granted, free of charge, to any person obtaining 8 | # a copy of this software and associated documentation files (the 9 | # "Software"), to deal in the Software without restriction, including 10 | # without limitation the rights to use, copy, modify, merge, publish, 11 | # distribute, sublicense, and/or sell copies of the Software, and to 12 | # permit persons to whom the Software is furnished to do so, subject to 13 | # the following conditions: 14 | 15 | # The above copyright notice and this permission notice shall be 16 | # included in all copies or substantial portions of the Software. 17 | 18 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 19 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 20 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 21 | # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 22 | # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 23 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 24 | # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 25 | 26 | require 'strscan' 27 | 28 | module KDL 29 | module Types 30 | class Duration < Value::Custom 31 | # Parses a string formatted according to ISO 8601 Duration into the hash. 32 | # 33 | # See {ISO 8601}[https://en.wikipedia.org/wiki/ISO_8601#Durations] for more information. 34 | # 35 | # This parser allows negative parts to be present in pattern. 36 | class ISO8601Parser # :nodoc: 37 | class ParsingError < ::ArgumentError; end 38 | 39 | PERIOD_OR_COMMA = /\.|,/ 40 | PERIOD = '.' 41 | COMMA = ',' 42 | 43 | SIGN_MARKER = /\A-|\+|/ 44 | DATE_MARKER = /P/ 45 | TIME_MARKER = /T/ 46 | DATE_COMPONENT = /(-?\d+(?:[.,]\d+)?)(Y|M|D|W)/ 47 | TIME_COMPONENT = /(-?\d+(?:[.,]\d+)?)(H|M|S)/ 48 | 49 | DATE_TO_PART = { 'Y' => :years, 'M' => :months, 'W' => :weeks, 'D' => :days } 50 | TIME_TO_PART = { 'H' => :hours, 'M' => :minutes, 'S' => :seconds } 51 | 52 | DATE_COMPONENTS = %i[years months days] 53 | TIME_COMPONENTS = %i[hours minutes seconds] 54 | 55 | attr_reader :parts, :scanner 56 | attr_accessor :mode, :sign 57 | 58 | def initialize(string) 59 | @scanner = StringScanner.new(string) 60 | @parts = {} 61 | @mode = :start 62 | @sign = 1 63 | end 64 | 65 | def parse! 66 | until finished? 67 | case mode 68 | when :start 69 | if scan(SIGN_MARKER) 70 | self.sign = scanner.matched == '-' ? -1 : 1 71 | self.mode = :sign 72 | else 73 | raise_parsing_error 74 | end 75 | 76 | when :sign 77 | if scan(DATE_MARKER) 78 | self.mode = :date 79 | else 80 | raise_parsing_error 81 | end 82 | 83 | when :date 84 | if scan(TIME_MARKER) 85 | self.mode = :time 86 | elsif scan(DATE_COMPONENT) 87 | parts[DATE_TO_PART[scanner[2]]] = number * sign 88 | else 89 | raise_parsing_error 90 | end 91 | 92 | when :time 93 | if scan(TIME_COMPONENT) 94 | parts[TIME_TO_PART[scanner[2]]] = number * sign 95 | else 96 | raise_parsing_error 97 | end 98 | 99 | end 100 | end 101 | 102 | validate! 103 | parts 104 | end 105 | 106 | private 107 | 108 | def finished? 109 | scanner.eos? 110 | end 111 | 112 | # Parses number which can be a float with either comma or period. 113 | def number 114 | (PERIOD_OR_COMMA =~ scanner[1]) ? scanner[1].tr(COMMA, PERIOD).to_f : scanner[1].to_i 115 | end 116 | 117 | def scan(pattern) 118 | scanner.scan(pattern) 119 | end 120 | 121 | def raise_parsing_error(reason = nil) 122 | raise ParsingError, "Invalid ISO 8601 duration: #{scanner.string.inspect} #{reason}".strip 123 | end 124 | 125 | # Checks for various semantic errors as stated in ISO 8601 standard. 126 | def validate! 127 | raise_parsing_error('is empty duration') if parts.empty? 128 | 129 | # Mixing any of Y, M, D with W is invalid. 130 | if parts.key?(:weeks) && (parts.keys & DATE_COMPONENTS).any? 131 | raise_parsing_error('mixing weeks with other date parts not allowed') 132 | end 133 | 134 | # Specifying an empty T part is invalid. 135 | if mode == :time && (parts.keys & TIME_COMPONENTS).empty? 136 | raise_parsing_error('time part marker is present but time part is empty') 137 | end 138 | 139 | fractions = parts.values.reject(&:zero?).select { |a| (a % 1) != 0 } 140 | unless fractions.empty? || (fractions.size == 1 && fractions.last == @parts.values.reject(&:zero?).last) 141 | raise_parsing_error '(only last part can be fractional)' 142 | end 143 | 144 | true 145 | end 146 | end 147 | end 148 | end 149 | end 150 | -------------------------------------------------------------------------------- /test/node_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "test_helper" 4 | 5 | class NodeTest < Minitest::Test 6 | def test_ref 7 | node = KDL::Node.new("node", arguments: [v(1), v("two")], properties: { "three" => v(3), "four" => v(4) }) 8 | 9 | assert_equal 1, node[0] 10 | assert_equal "two", node[1] 11 | assert_nil node[2] 12 | 13 | assert_equal 3, node["three"] 14 | assert_equal 3, node[:three] 15 | assert_equal 4, node[:four] 16 | 17 | assert_raises { node[nil] } 18 | end 19 | 20 | def test_child 21 | node = KDL::Node.new("node", children: [ 22 | KDL::Node.new("foo"), 23 | KDL::Node.new("bar") 24 | ]) 25 | 26 | assert_equal node.children[0], node.child(0) 27 | assert_equal node.children[1], node.child(1) 28 | 29 | assert_equal node.children[0], node.child("foo") 30 | assert_equal node.children[0], node.child(:foo) 31 | assert_equal node.children[1], node.child(:bar) 32 | 33 | a = []; node.each { a << _1.name } 34 | assert_equal ["foo", "bar"], a 35 | 36 | assert_raises { node.child(nil) } 37 | end 38 | 39 | def test_arg 40 | node = KDL::Node.new("node", children: [ 41 | KDL::Node.new("foo", arguments: [KDL::Value::String.new("bar")]), 42 | KDL::Node.new("baz", arguments: [KDL::Value::String.new("qux")]) 43 | ]) 44 | 45 | assert_equal "bar", node.arg(0) 46 | assert_equal "bar", node.arg("foo") 47 | assert_equal "bar", node.arg(:foo) 48 | assert_equal "qux", node.arg(1) 49 | assert_equal "qux", node.arg(:baz) 50 | assert_nil node.arg(:norf) 51 | 52 | assert_raises { node.arg(nil) } 53 | end 54 | 55 | def test_args 56 | node = KDL::Node.new("node", children: [ 57 | KDL::Node.new("foo", arguments: [KDL::Value::String.new("bar"), KDL::Value::String.new("baz")]), 58 | KDL::Node.new("qux", arguments: [KDL::Value::String.new("norf")]) 59 | ]) 60 | 61 | assert_equal ["bar", "baz"], node.args(0) 62 | assert_equal ["bar", "baz"], node.args("foo") 63 | assert_equal ["bar", "baz"], node.args(:foo) 64 | assert_equal ["norf"], node.args(1) 65 | assert_equal ["norf"], node.args(:qux) 66 | assert_nil node.args(:wat) 67 | 68 | a = []; node.each_arg(:foo) { a << _1 } 69 | assert_equal ["bar", "baz"], a 70 | 71 | a = []; node.each_arg(:wat) { a << _1 } 72 | assert_equal [], a 73 | 74 | assert_raises { node.args(nil) } 75 | end 76 | 77 | def test_dash_vals 78 | node = KDL::Node.new("node", children: [ 79 | KDL::Node.new("node", children: [ 80 | KDL::Node.new("-", arguments: [KDL::Value::String.new("foo")]), 81 | KDL::Node.new("-", arguments: [KDL::Value::String.new("bar")]), 82 | KDL::Node.new("-", arguments: [KDL::Value::String.new("baz")]) 83 | ]) 84 | ]) 85 | 86 | assert_equal ["foo", "bar", "baz"], node.dash_vals(0) 87 | assert_equal ["foo", "bar", "baz"], node.dash_vals("node") 88 | assert_equal ["foo", "bar", "baz"], node.dash_vals(:node) 89 | assert_nil node.dash_vals(:nope) 90 | 91 | a = []; node.each_dash_val(:node) { a << _1 } 92 | assert_equal ["foo", "bar", "baz"], a 93 | 94 | a = []; node.each_dash_val(:nope) { a << _1 } 95 | assert_equal [], a 96 | 97 | assert_raises { node.dash_vals(nil) } 98 | end 99 | 100 | def test_to_s 101 | node = ::KDL::Node.new("foo", arguments: [v(1), v("two")], properties: { "three" => v(3) }) 102 | 103 | assert_equal 'foo 1 two three=3', node.to_s 104 | end 105 | 106 | def test_nested_to_s 107 | node = ::KDL::Node.new("a1", arguments: [v("a"), v(1)], properties: { a: v(1) }, children: [ 108 | ::KDL::Node.new("b1", arguments: [v("b"), v(1, "foo")], children: [ 109 | ::KDL::Node.new("c1", arguments: [v("c"), v(1)]) 110 | ]), 111 | ::KDL::Node.new("b2", arguments: [v("b")], properties: { c: v(2, "bar") }, children: [ 112 | ::KDL::Node.new("c2", arguments: [v("c"), v(2)]) 113 | ]), 114 | ::KDL::Node.new("b3", type: "baz"), 115 | ]) 116 | 117 | assert_equal <<~KDL.strip, node.to_s 118 | a1 a 1 a=1 { 119 | b1 b (foo)1 { 120 | c1 c 1 121 | } 122 | b2 b c=(bar)2 { 123 | c2 c 2 124 | } 125 | (baz)b3 126 | } 127 | KDL 128 | 129 | assert_equal <<~KDL.strip, node.inspect 130 | "a1" "a" 1 "a"=1 { 131 | "b1" "b" ("foo")1 { 132 | "c1" "c" 1 133 | } 134 | "b2" "b" "c"=("bar")2 { 135 | "c2" "c" 2 136 | } 137 | ("baz")"b3" 138 | } 139 | KDL 140 | end 141 | 142 | def test_compare 143 | a = KDL::Node.new("a") 144 | b = KDL::Node.new("b") 145 | 146 | assert_equal -1, a <=> b 147 | assert_equal 1, b <=> a 148 | assert_equal 0, a <=> a 149 | end 150 | 151 | class Something < KDL::Node::Custom 152 | end 153 | 154 | def test_as_type 155 | node = KDL::Node.new("foo") 156 | assert_equal "bar", node.as_type("bar").type 157 | assert_kind_of Something, node.as_type("bar", lambda { |n, type| Something.new(n) }) 158 | nil_parse = node.as_type("bar", lambda { |n, type| nil }) 159 | assert_equal node, nil_parse 160 | assert_equal "bar", nil_parse.type 161 | 162 | assert_raises { node.as_type("bar", lambda { |n, type| Object.new }) } 163 | end 164 | 165 | def test_version 166 | node = KDL::Node.new("foo") 167 | assert_equal 2, node.version 168 | end 169 | 170 | def test_to_v1 171 | node = KDL::Node.new("foo", 172 | arguments: [v(true)], 173 | properties: { bar: v("baz") }, 174 | children: [KDL::Node.new("qux")] 175 | ) 176 | 177 | node = node.to_v1 178 | assert_equal 1, node.version 179 | assert_equal 1, node.arguments[0].version 180 | assert_equal 1, node.properties['bar'].version 181 | assert_equal 1, node.child(0).version 182 | end 183 | 184 | def test_to_v2 185 | node = KDL::Node.new("foo") 186 | assert_same node, node.to_v2 187 | end 188 | 189 | private 190 | 191 | def v(x, t=nil) 192 | val = ::KDL::Value.from(x) 193 | return val.as_type(t) if t 194 | val 195 | end 196 | end 197 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | 3 | ## When Something Happens 4 | 5 | If you see behavoir that makes you feel unsafe or unwelcome or otherwise uncomfortable, follow these steps: 6 | 7 | 1. Let the person know that what they did is not appropriate and ask them to stop and/or edit their message(s) or commits. 8 | 2. That person should immediately stop the behavior and correct the issue. 9 | 3. If this doesn’t happen, or if you're uncomfortable speaking up, [contact the maintainers](#contacting-maintainers). 10 | 4. As soon as available, a maintainer will look into the issue, and take [further action (see below)](#further-enforcement), starting with a warning, then temporary block, then long-term repo or organization ban. 11 | 12 | **The maintainer team will prioritize the well-being and comfort of those affected over the comfort of the offending party.** See [some examples below](#enforcement-examples). 13 | 14 | ## Our Pledge 15 | 16 | In the interest of fostering an open and welcoming environment, we as contributors and maintainers of this project pledge to making participation in our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, technical preferences, nationality, personal appearance, race, religion, or sexual identity and orientation. 17 | 18 | This commitment means that inappropriate behavior can lead to intervention. This includes our intention to address issues with [missing stairs](https://en.wikipedia.org/wiki/Missing_stair) who may not have explicitly violated any written-down rules but might still be disrupting the community. 19 | 20 | ## Scope 21 | 22 | This Code of Conduct applies both within spaces involving this project and in other spaces involving community members. This includes the repository, its Pull Requests and Issue tracker, its Twitter community, private email communications in the context of the project, and any events where members of the project are participating, as well as adjacent communities and venues affecting the project's members. 23 | 24 | Depending on the violation, the maintainers may decide that violations of this code of conduct that have happened outside of the scope of the community may deem an individual unwelcome, and take appropriate action to maintain the comfort and safety of its members. 25 | 26 | ## Contacting Maintainers 27 | 28 | - [Dan Smith ](mailto:danini@hey.com) 29 | 30 | ## Further Enforcement 31 | 32 | If you've already followed the [initial enforcement steps](#enforcement), these are the steps maintainers will take for further enforcement, as needed: 33 | 34 | 1. Repeat the request to stop. 35 | 2. If the person doubles down, they will have offending messages removed or edited by a maintainers given an official warning. The PR or Issue may be locked. 36 | 3. If the behavior continues or is repeated later, the person will be blocked from participating for 24 hours. 37 | 4. If the behavior continues or is repeated after the temporary block, a long-term (6-12mo) ban will be used. 38 | 39 | On top of this, maintainers may remove any offending messages, images, contributions, etc, as they deem necessary. 40 | 41 | Maintainers reserve full rights to skip any of these steps, at their discretion, if the violation is considered to be a serious and/or immediate threat to the health and well-being of members of the community. These include any threats, serious physical or verbal attacks, and other such behavior that would be completely unacceptable in any social setting that puts our members at risk. 42 | 43 | Members expelled from events or venues with any sort of paid attendance will not be refunded. 44 | 45 | ## Who Watches the Watchers? 46 | 47 | Maintainers and other leaders who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. These may include anything from removal from the maintainer team to a permanent ban from the community. 48 | 49 | Additionally, as a project hosted on both GitHub, [their Community Guidielines may be applied to maintainers of this project](https://help.github.com/articles/github-community-guidelines/), externally of this project's procedures. 50 | 51 | ## Enforcement Examples 52 | 53 | ### The Best Case 54 | 55 | The vast majority of situations work out like this. This interaction is common, and generally positive. 56 | 57 | > Alex: "Yeah I used X and it was really crazy!" 58 | 59 | > Patt (not a maintainer): "Hey, could you not use that word? What about 'ridiculous' instead?" 60 | 61 | > Alex: "oh sorry, sure." -> edits old comment to say "it was really confusing!" 62 | 63 | ### The Maintainer Case 64 | 65 | Sometimes, though, you need to get maintainers involved. Maintainers will do their best to resolve conflicts, but people who were harmed by something **will take priority**. 66 | 67 | > Patt: "Honestly, sometimes I just really hate using $library and anyone who uses it probably sucks at their job." 68 | 69 | > Alex: "Whoa there, could you dial it back a bit? There's a CoC thing about attacking folks' tech use like that." 70 | 71 | > Patt: "I'm not attacking anyone, what's your problem?" 72 | 73 | > Alex: "@maintainers hey uh. Can someone look at this issue? Patt is getting a bit aggro. I tried to nudge them about it, but nope." 74 | 75 | > KeeperOfCommitBits: (on issue) "Hey Patt, maintainer here. Could you tone it down? This sort of attack is really not okay in this space." 76 | 77 | > Patt: "Leave me alone I haven't said anything bad wtf is wrong with you." 78 | 79 | > KeeperOfCommitBits: (deletes user's comment), "@patt I mean it. Please refer to the CoC over at (URL to this CoC) if you have questions, but you can consider this an actual warning. I'd appreciate it if you reworded your messages in this thread, since they made folks there uncomfortable. Let's try and be kind, yeah?" 80 | 81 | > Patt: "@keeperofbits Okay sorry. I'm just frustrated and I'm kinda burnt out and I guess I got carried away. I'll DM Alex a note apologizing and edit my messages. Sorry for the trouble." 82 | 83 | > KeeperOfCommitBits: "@patt Thanks for that. I hear you on the stress. Burnout sucks :/. Have a good one!" 84 | 85 | ### The Nope Case 86 | 87 | > PepeTheFrog🐸: "Hi, I am a literal actual nazi and I think white supremacists are quite fashionable." 88 | 89 | > Patt: "NOOOOPE. OH NOPE NOPE." 90 | 91 | > Alex: "JFC NO. NOPE. @keeperofbits NOPE NOPE LOOK HERE" 92 | 93 | > KeeperOfCommitBits: "👀 Nope. NOPE NOPE NOPE. 🔥" 94 | 95 | > PepeTheFrog🐸 has been banned from all organization or user repositories belonging to KeeperOfCommitBits. 96 | 97 | ## Attribution 98 | 99 | This Code of Conduct was generated using [WeAllJS Code of Conduct Generator](https://npm.im/weallbehave), which is based on the [WeAllJS Code of 100 | Conduct](https://wealljs.org/code-of-conduct), which is itself based on 101 | [Contributor Covenant](http://contributor-covenant.org), version 1.4, available 102 | at 103 | [http://contributor-covenant.org/version/1/4](http://contributor-covenant.org/version/1/4), 104 | and the LGBTQ in Technology Slack [Code of 105 | Conduct](http://lgbtq.technology/coc.html). 106 | -------------------------------------------------------------------------------- /test/types/url_template_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "test_helper" 4 | 5 | class URLTemplateTest < Minitest::Test 6 | def setup 7 | @variables = { 8 | count: ['one', 'two', 'three'], 9 | dom: ['example', 'com'], 10 | dub: 'me/too', 11 | hello: 'Hello World!', 12 | half: '50%', 13 | var: 'value', 14 | who: 'fred', 15 | base: 'http://example.com/home/', 16 | path: '/foo/bar', 17 | list: ['red', 'green', 'blue'], 18 | keys: { semi: ';', dot: '.', comma: ',' }, 19 | v: '6', 20 | x: '1024', 21 | y: '768', 22 | empty: '', 23 | empty_keys: {}, 24 | undef: nil, 25 | } 26 | end 27 | 28 | def test_no_variables 29 | value = KDL::Types::URLTemplate.call(::KDL::Value::String.new('https://www.example.com/foo/bar')) 30 | assert_equal URI('https://www.example.com/foo/bar'), value.expand({}) 31 | end 32 | 33 | def test_variable 34 | value = KDL::Types::URLTemplate.call(::KDL::Value::String.new('https://www.example.com/{foo}/bar')) 35 | assert_equal URI('https://www.example.com/lorem/bar'), value.expand({ foo: 'lorem' }) 36 | end 37 | 38 | def test_multiple_variables 39 | value = KDL::Types::URLTemplate.call(::KDL::Value::String.new('https://www.example.com/{foo}/{bar}')) 40 | assert_equal URI('https://www.example.com/lorem/ipsum'), value.expand({ foo: 'lorem', bar: 'ipsum' }) 41 | end 42 | 43 | def test_list_expansion 44 | assert_expansion_equal '{count}', 'one,two,three' 45 | assert_expansion_equal '{count*}', 'one,two,three' 46 | assert_expansion_equal '{/count}', '/one,two,three' 47 | assert_expansion_equal '{/count*}', '/one/two/three' 48 | assert_expansion_equal '{;count}', ';count=one,two,three' 49 | assert_expansion_equal '{;count*}', ';count=one;count=two;count=three' 50 | assert_expansion_equal '{?count}', '?count=one,two,three' 51 | assert_expansion_equal '{?count*}', '?count=one&count=two&count=three' 52 | assert_expansion_equal '{&count*}', '&count=one&count=two&count=three' 53 | end 54 | 55 | def test_simple_string 56 | assert_expansion_equal '{var}', 'value' 57 | assert_expansion_equal '{hello}', 'Hello%20World%21' 58 | assert_expansion_equal '{half}', '50%25' 59 | assert_expansion_equal 'O{empty}X', 'OX' 60 | assert_expansion_equal 'O{undef}X', 'OX' 61 | assert_expansion_equal '{x,y}', '1024,768' 62 | assert_expansion_equal '{x,hello,y}', '1024,Hello%20World%21,768' 63 | assert_expansion_equal '?{x,empty}', '?1024,' 64 | assert_expansion_equal '?{x,undef}', '?1024' 65 | assert_expansion_equal '?{undef,y}', '?768' 66 | assert_expansion_equal '{var:3}', 'val' 67 | assert_expansion_equal '{var:30}', 'value' 68 | assert_expansion_equal '{list}', 'red,green,blue' 69 | assert_expansion_equal '{list*}', 'red,green,blue' 70 | assert_expansion_equal '{keys}', 'semi,%3B,dot,.,comma,%2C' 71 | assert_expansion_equal '{keys*}', 'semi=%3B,dot=.,comma=%2C' 72 | end 73 | 74 | def test_reserved_expansion 75 | assert_expansion_equal '{+var}', 'value' 76 | assert_expansion_equal '{+hello}', 'Hello%20World!' 77 | assert_expansion_equal '{+half}', '50%25' 78 | 79 | assert_expansion_equal '{base}index', 'http%3A%2F%2Fexample.com%2Fhome%2Findex' 80 | assert_expansion_equal '{+base}index', 'http://example.com/home/index' 81 | assert_expansion_equal 'O{+empty}X', 'OX' 82 | assert_expansion_equal 'O{+undef}X', 'OX' 83 | 84 | assert_expansion_equal '{+path}/here', '/foo/bar/here' 85 | assert_expansion_equal 'here?ref={+path}', 'here?ref=/foo/bar' 86 | assert_expansion_equal 'up{+path}{var}/here', 'up/foo/barvalue/here' 87 | assert_expansion_equal '{+x,hello,y}', '1024,Hello%20World!,768' 88 | assert_expansion_equal '{+path,x}/here', '/foo/bar,1024/here' 89 | 90 | assert_expansion_equal '{+path:6}/here', '/foo/b/here' 91 | assert_expansion_equal '{+list}', 'red,green,blue' 92 | assert_expansion_equal '{+list*}', 'red,green,blue' 93 | assert_expansion_equal '{+keys}', 'semi,;,dot,.,comma,,' 94 | assert_expansion_equal '{+keys*}', 'semi=;,dot=.,comma=,' 95 | end 96 | 97 | def test_fragment_expansion 98 | assert_expansion_equal '{#var}', '#value' 99 | assert_expansion_equal '{#hello}', '#Hello%20World!' 100 | assert_expansion_equal '{#half}', '#50%25' 101 | assert_expansion_equal 'foo{#empty}', 'foo#' 102 | assert_expansion_equal 'foo{#undef}', 'foo' 103 | assert_expansion_equal '{#x,hello,y}', '#1024,Hello%20World!,768' 104 | assert_expansion_equal '{#path,x}/here', '#/foo/bar,1024/here' 105 | assert_expansion_equal '{#path:6}/here', '#/foo/b/here' 106 | assert_expansion_equal '{#list}', '#red,green,blue' 107 | assert_expansion_equal '{#list*}', '#red,green,blue' 108 | assert_expansion_equal '{#keys}', '#semi,;,dot,.,comma,,' 109 | assert_expansion_equal '{#keys*}', '#semi=;,dot=.,comma=,' 110 | end 111 | 112 | def test_label_expansion 113 | assert_expansion_equal '{.who}', '.fred' 114 | assert_expansion_equal '{.who,who}', '.fred.fred' 115 | assert_expansion_equal '{.half,who}', '.50%25.fred' 116 | assert_expansion_equal 'www{.dom*}', 'www.example.com' 117 | assert_expansion_equal 'X{.var}', 'X.value' 118 | assert_expansion_equal 'X{.empty}', 'X.' 119 | assert_expansion_equal 'X{.undef}', 'X' 120 | assert_expansion_equal 'X{.var:3}', 'X.val' 121 | assert_expansion_equal 'X{.list}', 'X.red,green,blue' 122 | assert_expansion_equal 'X{.list*}', 'X.red.green.blue' 123 | assert_expansion_equal 'X{.keys}', 'X.semi,%3B,dot,.,comma,%2C' 124 | assert_expansion_equal 'X{.keys*}', 'X.semi=%3B.dot=..comma=%2C' 125 | assert_expansion_equal 'X{.empty_keys}', 'X' 126 | assert_expansion_equal 'X{.empty_keys*}', 'X' 127 | end 128 | 129 | def test_path_expansion 130 | assert_expansion_equal '{/who}', '/fred' 131 | assert_expansion_equal '{/who,who}', '/fred/fred' 132 | assert_expansion_equal '{/half,who}', '/50%25/fred' 133 | assert_expansion_equal '{/who,dub}', '/fred/me%2Ftoo' 134 | assert_expansion_equal '{/var}', '/value' 135 | assert_expansion_equal '{/var,empty}', '/value/' 136 | assert_expansion_equal '{/var,undef}', '/value' 137 | assert_expansion_equal '{/var,x}/here', '/value/1024/here' 138 | assert_expansion_equal '{/var:1,var}', '/v/value' 139 | assert_expansion_equal '{/list}', '/red,green,blue' 140 | assert_expansion_equal '{/list*}', '/red/green/blue' 141 | assert_expansion_equal '{/list*,path:4}', '/red/green/blue/%2Ffoo' 142 | assert_expansion_equal '{/keys}', '/semi,%3B,dot,.,comma,%2C' 143 | assert_expansion_equal '{/keys*}', '/semi=%3B/dot=./comma=%2C' 144 | end 145 | 146 | def test_parameter_expansion 147 | assert_expansion_equal '{;who}', ';who=fred' 148 | assert_expansion_equal '{;half}', ';half=50%25' 149 | assert_expansion_equal '{;empty}', ';empty' 150 | assert_expansion_equal '{;v,empty,who}', ';v=6;empty;who=fred' 151 | assert_expansion_equal '{;v,bar,who}', ';v=6;who=fred' 152 | assert_expansion_equal '{;x,y}', ';x=1024;y=768' 153 | assert_expansion_equal '{;x,y,empty}', ';x=1024;y=768;empty' 154 | assert_expansion_equal '{;x,y,undef}', ';x=1024;y=768' 155 | assert_expansion_equal '{;hello:5}', ';hello=Hello' 156 | assert_expansion_equal '{;list}', ';list=red,green,blue' 157 | assert_expansion_equal '{;list*}', ';list=red;list=green;list=blue' 158 | assert_expansion_equal '{;keys}', ';keys=semi,%3B,dot,.,comma,%2C' 159 | assert_expansion_equal '{;keys*}', ';semi=%3B;dot=.;comma=%2C' 160 | end 161 | 162 | def test_query_expansion 163 | assert_expansion_equal '{?who}', '?who=fred' 164 | assert_expansion_equal '{?half}', '?half=50%25' 165 | assert_expansion_equal '{?x,y}', '?x=1024&y=768' 166 | assert_expansion_equal '{?x,y,empty}', '?x=1024&y=768&empty=' 167 | assert_expansion_equal '{?x,y,undef}', '?x=1024&y=768' 168 | assert_expansion_equal '{?var:3}', '?var=val' 169 | assert_expansion_equal '{?list}', '?list=red,green,blue' 170 | assert_expansion_equal '{?list*}', '?list=red&list=green&list=blue' 171 | assert_expansion_equal '{?keys}', '?keys=semi,%3B,dot,.,comma,%2C' 172 | assert_expansion_equal '{?keys*}', '?semi=%3B&dot=.&comma=%2C' 173 | end 174 | 175 | def test_query_continuation 176 | assert_expansion_equal '{&who}', '&who=fred' 177 | assert_expansion_equal '{&half}', '&half=50%25' 178 | assert_expansion_equal '?fixed=yes{&x}', '?fixed=yes&x=1024' 179 | assert_expansion_equal '{&x,y,empty}', '&x=1024&y=768&empty=' 180 | assert_expansion_equal '{&x,y,undef}', '&x=1024&y=768' 181 | assert_expansion_equal '{&var:3}', '&var=val' 182 | assert_expansion_equal '{&list}', '&list=red,green,blue' 183 | assert_expansion_equal '{&list*}', '&list=red&list=green&list=blue' 184 | assert_expansion_equal '{&keys}', '&keys=semi,%3B,dot,.,comma,%2C' 185 | assert_expansion_equal '{&keys*}', '&semi=%3B&dot=.&comma=%2C' 186 | end 187 | 188 | private 189 | 190 | def assert_expansion_equal(template, expected) 191 | value = KDL::Types::URLTemplate.call(::KDL::Value::String.new(template)) 192 | assert_equal(URI(expected), value.expand(@variables)) 193 | end 194 | end 195 | -------------------------------------------------------------------------------- /lib/kdl/types/url_template.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'uri' 4 | 5 | module KDL 6 | module Types 7 | class URLTemplate < Value::Custom 8 | UNRESERVED = /[a-zA-Z0-9\-._~]/.freeze 9 | RESERVED = %r{[:/?#\[\]@!$&'()*+,;=]}.freeze 10 | 11 | def self.call(value, type = 'url-template') 12 | return nil unless value.is_a? ::KDL::Value::String 13 | 14 | parts = Parser.parse(value.value) 15 | new(parts, type: type) 16 | end 17 | 18 | def expand(variables) 19 | result = value.map { |v| v.expand(variables) }.join 20 | parser = IRLReference::Parser.new(result) 21 | uri, * = parser.parse 22 | URI(uri) 23 | end 24 | 25 | class Parser 26 | def self.parse(string) 27 | new(string).parse 28 | end 29 | 30 | def initialize(string) 31 | @string = string 32 | @index = 0 33 | end 34 | 35 | def parse 36 | result = [] 37 | until (token = next_token).nil? 38 | result << token 39 | end 40 | result 41 | end 42 | 43 | def next_token 44 | buffer = +'' 45 | context = nil 46 | expansion_type = nil 47 | loop do 48 | c = @string[@index] 49 | case context 50 | when nil 51 | case c 52 | when '{' 53 | context = :expansion 54 | buffer = +'' 55 | n = @string[@index + 1] 56 | expansion_type = case n 57 | when '+' then ReservedExpansion 58 | when '#' then FragmentExpansion 59 | when '.' then LabelExpansion 60 | when '/' then PathExpansion 61 | when ';' then ParameterExpansion 62 | when '?' then QueryExpansion 63 | when '&' then QueryContinuation 64 | else StringExpansion 65 | end 66 | @index += (expansion_type == StringExpansion ? 1 : 2) 67 | when nil then return nil 68 | else 69 | buffer = +c 70 | @index += 1 71 | context = :literal 72 | end 73 | when :literal 74 | case c 75 | when '{', nil then return StringLiteral.new(buffer) 76 | else 77 | buffer << c 78 | @index += 1 79 | end 80 | when :expansion 81 | case c 82 | when '}' 83 | @index += 1 84 | return parse_expansion(buffer, expansion_type) 85 | when nil 86 | raise ArgumentError, 'unterminated expansion' 87 | else 88 | buffer << c 89 | @index += 1 90 | end 91 | end 92 | end 93 | end 94 | 95 | def parse_expansion(string, type) 96 | variables = string.split(',').map do |str| 97 | case str 98 | when /(.*)\*$/ 99 | Variable.new(Regexp.last_match(1), 100 | explode: true, 101 | allow_reserved: type.allow_reserved?, 102 | with_name: type.with_name?, 103 | keep_empties: type.keep_empties?) 104 | when /(.*):(\d+)/ 105 | Variable.new(Regexp.last_match(1), 106 | limit: Regexp.last_match(2).to_i, 107 | allow_reserved: type.allow_reserved?, 108 | with_name: type.with_name?, 109 | keep_empties: type.keep_empties?) 110 | else 111 | Variable.new(str, 112 | allow_reserved: type.allow_reserved?, 113 | with_name: type.with_name?, 114 | keep_empties: type.keep_empties?) 115 | end 116 | end 117 | type.new(variables) 118 | end 119 | end 120 | 121 | class Variable 122 | attr_reader :name 123 | 124 | def initialize(name, limit: nil, explode: false, allow_reserved: false, with_name: false, keep_empties: false) 125 | @name = name.to_sym 126 | @limit = limit 127 | @explode = explode 128 | @allow_reserved = allow_reserved 129 | @with_name = with_name 130 | @keep_empties = keep_empties 131 | end 132 | 133 | def expand(value) 134 | if @explode 135 | case value 136 | when Array 137 | value.map { |v| prefix(encode(v)) } 138 | when Hash 139 | value.map { |k, v| prefix(encode(v), k) } 140 | else 141 | [prefix(encode(value))] 142 | end 143 | elsif @limit 144 | [prefix(limit(value))].compact 145 | else 146 | [prefix(flatten(value))].compact 147 | end 148 | end 149 | 150 | def limit(string) 151 | return nil unless string 152 | 153 | encode(string[0, @limit]) 154 | end 155 | 156 | def flatten(value) 157 | case value 158 | when String 159 | encode(value) 160 | when Array, Hash 161 | result = value.to_a 162 | .flatten 163 | .compact 164 | .map { |v| encode(v) } 165 | result.empty? ? nil : result.join(',') 166 | end 167 | end 168 | 169 | def encode(string) 170 | return nil unless string 171 | 172 | string.to_s 173 | .chars 174 | .map do |c| 175 | if UNRESERVED.match?(c) || (@allow_reserved && RESERVED.match?(c)) 176 | c 177 | else 178 | IRLReference::Parser.percent_encode(c) 179 | end 180 | end 181 | .join 182 | .force_encoding('utf-8') 183 | end 184 | 185 | def prefix(string, override = nil) 186 | return nil unless string 187 | 188 | key = override || @name 189 | 190 | if @with_name || override 191 | if string.empty? && !@keep_empties 192 | encode(key) 193 | else 194 | "#{encode(key)}=#{string}" 195 | end 196 | else 197 | string 198 | end 199 | end 200 | end 201 | 202 | class Part 203 | def expand_variables(values) 204 | @variables.reduce([]) do |list, variable| 205 | expanded = variable.expand(values[variable.name]) 206 | expanded ? list + expanded : list 207 | end 208 | end 209 | 210 | def separator 211 | ',' 212 | end 213 | 214 | def prefix 215 | '' 216 | end 217 | 218 | def self.allow_reserved? 219 | false 220 | end 221 | 222 | def self.with_name? 223 | false 224 | end 225 | 226 | def self.keep_empties? 227 | false 228 | end 229 | end 230 | 231 | class StringLiteral < Part 232 | def initialize(value) 233 | super() 234 | @value = value 235 | end 236 | 237 | def expand(*) 238 | @value 239 | end 240 | end 241 | 242 | class StringExpansion < Part 243 | def initialize(variables) 244 | super() 245 | @variables = variables 246 | end 247 | 248 | def expand(values) 249 | expanded = expand_variables(values) 250 | return '' if expanded.empty? 251 | 252 | prefix + expanded.join(separator) 253 | end 254 | end 255 | 256 | class ReservedExpansion < StringExpansion 257 | def self.allow_reserved? 258 | true 259 | end 260 | end 261 | 262 | class FragmentExpansion < StringExpansion 263 | def prefix 264 | '#' 265 | end 266 | 267 | def self.allow_reserved? 268 | true 269 | end 270 | end 271 | 272 | class LabelExpansion < StringExpansion 273 | def prefix 274 | '.' 275 | end 276 | 277 | def separator 278 | '.' 279 | end 280 | end 281 | 282 | class PathExpansion < StringExpansion 283 | def prefix 284 | '/' 285 | end 286 | 287 | def separator 288 | '/' 289 | end 290 | end 291 | 292 | class ParameterExpansion < StringExpansion 293 | def prefix 294 | ';' 295 | end 296 | 297 | def separator 298 | ';' 299 | end 300 | 301 | def self.with_name? 302 | true 303 | end 304 | end 305 | 306 | class QueryExpansion < StringExpansion 307 | def prefix 308 | '?' 309 | end 310 | 311 | def separator 312 | '&' 313 | end 314 | 315 | def self.with_name? 316 | true 317 | end 318 | 319 | def self.keep_empties? 320 | true 321 | end 322 | end 323 | 324 | class QueryContinuation < QueryExpansion 325 | def prefix 326 | '&' 327 | end 328 | end 329 | end 330 | MAPPING['url-template'] = URLTemplate 331 | end 332 | end 333 | -------------------------------------------------------------------------------- /lib/kdl/v1/tokenizer.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module KDL 4 | module V1 5 | class Tokenizer < KDL::Tokenizer 6 | NON_IDENTIFIER_CHARS = Regexp.escape "#{SYMBOLS.keys.join}()/\\<>[]\",#{WHITESPACE.join}#{OTHER_NON_IDENTIFIER_CHARS.join}" 7 | IDENTIFIER_CHARS = /[^#{NON_IDENTIFIER_CHARS}]/ 8 | INITIAL_IDENTIFIER_CHARS = /[^#{NON_IDENTIFIER_CHARS}0-9]/ 9 | 10 | def next_token 11 | @context = nil 12 | @previous_context = nil 13 | @line_at_start = @line 14 | @column_at_start = @column 15 | loop do 16 | c = self[@index] 17 | case @context 18 | when nil 19 | case c 20 | when '"' 21 | self.context = :string 22 | @buffer = +'' 23 | traverse(1) 24 | when 'r' 25 | if @str[@index + 1] == '"' 26 | self.context = :rawstring 27 | traverse(2) 28 | @rawstring_hashes = 0 29 | @buffer = +'' 30 | next 31 | elsif @str[@index + 1] == '#' 32 | i = @index + 1 33 | @rawstring_hashes = 0 34 | while @str[i] == '#' 35 | @rawstring_hashes += 1 36 | i += 1 37 | end 38 | if @str[i] == '"' 39 | self.context = :rawstring 40 | @index = i + 1 41 | @buffer = +'' 42 | next 43 | end 44 | end 45 | self.context = :ident 46 | @buffer = +c 47 | traverse(1) 48 | when '-' 49 | n = self[@index + 1] 50 | if n =~ /[0-9]/ 51 | n2 = self[@index + 2] 52 | if n == '0' && n2 =~ /[box]/ 53 | self.context = integer_context(n2) 54 | traverse(3) 55 | else 56 | self.context = :decimal 57 | traverse(1) 58 | end 59 | else 60 | self.context = :ident 61 | traverse(1) 62 | end 63 | @buffer = +c 64 | when /[0-9+]/ 65 | n = self[@index + 1] 66 | if c == '0' && n =~ /[box]/ 67 | traverse(2) 68 | @buffer = +'' 69 | self.context = integer_context(n) 70 | else 71 | self.context = :decimal 72 | @buffer = +c 73 | traverse(1) 74 | end 75 | when '\\' 76 | t = Tokenizer.new(@str, @index + 1) 77 | la = t.next_token 78 | if la[0] == :NEWLINE || la[0] == :EOF || (la[0] == :WS && (lan = t.next_token[0]) == :NEWLINE || lan == :EOF) 79 | traverse_to(t.index) 80 | @buffer = "#{c}#{la[1].value}" 81 | @buffer << "\n" if lan == :NEWLINE 82 | self.context = :whitespace 83 | else 84 | raise_error "Unexpected '\\' (#{la[0]})" 85 | end 86 | when *SYMBOLS.keys 87 | return token(SYMBOLS[c], -c).tap { traverse(1) } 88 | when *NEWLINES, "\r" 89 | nl = expect_newline 90 | return token(:NEWLINE, -nl).tap do 91 | traverse(nl.length) 92 | end 93 | when "/" 94 | if self[@index + 1] == '/' 95 | self.context = :single_line_comment 96 | traverse(2) 97 | elsif self[@index + 1] == '*' 98 | self.context = :multi_line_comment 99 | @comment_nesting = 1 100 | traverse(2) 101 | elsif self[@index + 1] == '-' 102 | return token(:SLASHDASH, '/-').tap { traverse(2) } 103 | else 104 | self.context = :ident 105 | @buffer = +c 106 | traverse(1) 107 | end 108 | when *WHITESPACE 109 | self.context = :whitespace 110 | @buffer = +c 111 | traverse(1) 112 | when nil 113 | return [false, token(:EOF, :EOF)[1]] if @done 114 | 115 | @done = true 116 | return token(:EOF, :EOF) 117 | when INITIAL_IDENTIFIER_CHARS 118 | self.context = :ident 119 | @buffer = +c 120 | traverse(1) 121 | when '(' 122 | @type_context = true 123 | return token(:LPAREN, -c).tap { traverse(1) } 124 | when ')' 125 | @type_context = false 126 | return token(:RPAREN, -c).tap { traverse(1) } 127 | else 128 | raise_error "Unexpected character #{c.inspect}" 129 | end 130 | when :ident 131 | case c 132 | when IDENTIFIER_CHARS 133 | traverse(1) 134 | @buffer << c 135 | else 136 | case @buffer 137 | when 'true' then return token(:TRUE, true) 138 | when 'false' then return token(:FALSE, false) 139 | when 'null' then return token(:NULL, nil) 140 | else return token(:IDENT, -@buffer) 141 | end 142 | end 143 | when :string 144 | case c 145 | when '\\' 146 | c2 = self[@index + 1] 147 | if c2.match?(NEWLINES_PATTERN) 148 | i = 2 149 | while self[@index + i].match?(NEWLINES_PATTERN) 150 | i+=1 151 | end 152 | traverse(i) 153 | else 154 | @buffer << c 155 | @buffer << c2 156 | traverse(2) 157 | end 158 | when '"' 159 | return token(:STRING, -unescape(@buffer)).tap { traverse(1) } 160 | when nil 161 | raise_error "Unterminated string literal" 162 | else 163 | @buffer << c 164 | traverse(1) 165 | end 166 | when :rawstring 167 | raise_error "Unterminated rawstring literal" if c.nil? 168 | 169 | if c == '"' 170 | h = 0 171 | h += 1 while self[@index + 1 + h] == '#' && h < @rawstring_hashes 172 | if h == @rawstring_hashes 173 | return token(:RAWSTRING, -@buffer).tap { traverse(1 + h) } 174 | end 175 | end 176 | 177 | @buffer << c 178 | traverse(1) 179 | when :decimal 180 | case c 181 | when /[0-9.\-+_eE]/ 182 | traverse(1) 183 | @buffer << c 184 | else 185 | return parse_decimal(@buffer) 186 | end 187 | when :hexadecimal 188 | case c 189 | when /[0-9a-fA-F_]/ 190 | traverse(1) 191 | @buffer << c 192 | else 193 | return parse_hexadecimal(@buffer) 194 | end 195 | when :octal 196 | case c 197 | when /[0-7_]/ 198 | traverse(1) 199 | @buffer << c 200 | else 201 | return parse_octal(@buffer) 202 | end 203 | when :binary 204 | case c 205 | when /[01_]/ 206 | traverse(1) 207 | @buffer << c 208 | else 209 | return parse_binary(@buffer) 210 | end 211 | when :single_line_comment 212 | case c 213 | when *NEWLINES, "\r" 214 | self.context = nil 215 | @column_at_start = @column 216 | next 217 | when nil 218 | @done = true 219 | return token(:EOF, :EOF) 220 | else 221 | traverse(1) 222 | end 223 | when :multi_line_comment 224 | if c == '/' && self[@index + 1] == '*' 225 | @comment_nesting += 1 226 | traverse(2) 227 | elsif c == '*' && self[@index + 1] == '/' 228 | @comment_nesting -= 1 229 | traverse(2) 230 | if @comment_nesting == 0 231 | revert_context 232 | end 233 | else 234 | traverse(1) 235 | end 236 | when :whitespace 237 | if WHITESPACE.include?(c) 238 | traverse(1) 239 | @buffer << c 240 | elsif c == "/" && self[@index + 1] == '*' 241 | self.context = :multi_line_comment 242 | @comment_nesting = 1 243 | traverse(2) 244 | elsif c == "\\" 245 | t = Tokenizer.new(@str, @index + 1) 246 | la = t.next_token 247 | if la[0] == :NEWLINE || la[0] == :EOF || (la[0] == :WS && (lan = t.next_token[0]) == :NEWLINE || lan == :EOF) 248 | traverse_to(t.index) 249 | @buffer << "#{c}#{la[1].value}" 250 | @buffer << "\n" if lan == :NEWLINE 251 | else 252 | raise_error "Unexpected '\\' (#{la[0]})" 253 | end 254 | else 255 | return token(:WS, -@buffer) 256 | end 257 | else 258 | # :nocov: 259 | raise_error "Unknown context `#{@context}'" 260 | # :nocov: 261 | end 262 | end 263 | end 264 | 265 | private 266 | 267 | def allowed_in_type?(val) 268 | %i[ident string rawstring].include?(val) 269 | end 270 | 271 | def allowed_after_type?(val) 272 | !%i[single_line_comment multi_line_comment].include?(val) 273 | end 274 | 275 | def unescape(string) 276 | string.gsub(/\\[^u]/) do |m| 277 | case m 278 | when '\n' then "\n" 279 | when '\r' then "\r" 280 | when '\t' then "\t" 281 | when '\\\\' then "\\" 282 | when '\"' then "\"" 283 | when '\b' then "\b" 284 | when '\f' then "\f" 285 | when '\/' then "/" 286 | else raise_error "Unexpected escape #{m.inspect}" 287 | end 288 | end.gsub(/\\u\{[0-9a-fA-F]{0,6}\}/) do |m| 289 | i = Integer(m[3..-2], 16) 290 | if i < 0 || i > 0x10FFFF 291 | raise_error "Invalid code point #{u}" 292 | end 293 | i.chr(Encoding::UTF_8) 294 | end 295 | end 296 | end 297 | end 298 | end 299 | -------------------------------------------------------------------------------- /test/v1/tokenizer_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "test_helper" 4 | 5 | class KDL::V1::TokenizerTest < Minitest::Test 6 | def test_identifier 7 | assert_equal t(:IDENT, "foo"), ::KDL::V1::Tokenizer.new("foo").next_token 8 | assert_equal t(:IDENT, "foo-bar123"), ::KDL::V1::Tokenizer.new("foo-bar123").next_token 9 | assert_equal t(:IDENT, "-"), ::KDL::V1::Tokenizer.new("-").next_token 10 | assert_equal t(:IDENT, "--"), ::KDL::V1::Tokenizer.new("--").next_token 11 | end 12 | 13 | def test_string 14 | assert_equal t(:STRING, "foo"), ::KDL::V1::Tokenizer.new('"foo"').next_token 15 | assert_equal t(:STRING, "foo\nbar"), ::KDL::V1::Tokenizer.new('"foo\nbar"').next_token 16 | end 17 | 18 | def test_rawstring 19 | assert_equal t(:RAWSTRING, "foo\\nbar"), ::KDL::V1::Tokenizer.new('r"foo\\nbar"').next_token 20 | assert_equal t(:RAWSTRING, "foo\"bar"), ::KDL::V1::Tokenizer.new('r#"foo"bar"#').next_token 21 | assert_equal t(:RAWSTRING, "foo\"#bar"), ::KDL::V1::Tokenizer.new('r##"foo"#bar"##').next_token 22 | assert_equal t(:RAWSTRING, "\"foo\""), ::KDL::V1::Tokenizer.new('r#""foo""#').next_token 23 | 24 | tokenizer = ::KDL::V1::Tokenizer.new('node r"C:\\Users\\zkat\\"') 25 | assert_equal t(:IDENT, "node"), tokenizer.next_token 26 | assert_equal t(:WS, " ", 1, 5), tokenizer.next_token 27 | assert_equal t(:RAWSTRING, "C:\\Users\\zkat\\", 1, 6), tokenizer.next_token 28 | 29 | tokenizer = ::KDL::V1::Tokenizer.new('other-node r#"hello"world"#') 30 | assert_equal t(:IDENT, "other-node"), tokenizer.next_token 31 | assert_equal t(:WS, " ", 1, 11), tokenizer.next_token 32 | assert_equal t(:RAWSTRING, "hello\"world", 1, 12), tokenizer.next_token 33 | end 34 | 35 | def test_integer 36 | assert_equal t(:INTEGER, 123), ::KDL::V1::Tokenizer.new("123").next_token 37 | assert_equal t(:INTEGER, 0x0123456789abcdef), ::KDL::V1::Tokenizer.new("0x0123456789abcdef").next_token 38 | assert_equal t(:INTEGER, 0o01234567), ::KDL::V1::Tokenizer.new("0o01234567").next_token 39 | assert_equal t(:INTEGER, 0b101001), ::KDL::V1::Tokenizer.new("0b101001").next_token 40 | assert_equal t(:INTEGER, -0x0123456789abcdef), ::KDL::V1::Tokenizer.new("-0x0123456789abcdef").next_token 41 | assert_equal t(:INTEGER, -0o01234567), ::KDL::V1::Tokenizer.new("-0o01234567").next_token 42 | assert_equal t(:INTEGER, -0b101001), ::KDL::V1::Tokenizer.new("-0b101001").next_token 43 | end 44 | 45 | def test_float 46 | assert_equal t(:FLOAT, 1.23), ::KDL::V1::Tokenizer.new("1.23").next_token 47 | end 48 | 49 | def test_boolean 50 | assert_equal t(:TRUE, true), ::KDL::V1::Tokenizer.new("true").next_token 51 | assert_equal t(:FALSE, false), ::KDL::V1::Tokenizer.new("false").next_token 52 | end 53 | 54 | def test_null 55 | assert_equal t(:NULL, nil), ::KDL::V1::Tokenizer.new("null").next_token 56 | end 57 | 58 | def test_symbols 59 | assert_equal t(:LBRACE, '{'), ::KDL::V1::Tokenizer.new("{").next_token 60 | assert_equal t(:RBRACE, '}'), ::KDL::V1::Tokenizer.new("}").next_token 61 | assert_equal t(:EQUALS, '='), ::KDL::V1::Tokenizer.new("=").next_token 62 | end 63 | 64 | def test_whitespace 65 | assert_equal t(:WS, ' '), ::KDL::V1::Tokenizer.new(" ").next_token 66 | assert_equal t(:WS, "\t"), ::KDL::V1::Tokenizer.new("\t").next_token 67 | assert_equal t(:WS, " \t"), ::KDL::V1::Tokenizer.new(" \t").next_token 68 | end 69 | 70 | def test_escline 71 | assert_equal t(:WS, "\\\n"), ::KDL::V1::Tokenizer.new("\\\n").next_token 72 | assert_equal t(:WS, "\\\n"), ::KDL::V1::Tokenizer.new("\\\n//some comment").next_token 73 | assert_equal t(:WS, "\\\n "), ::KDL::V1::Tokenizer.new("\\\n //some comment").next_token 74 | assert_equal t(:STRING, "foo"), ::KDL::V1::Tokenizer.new("\"\\\n\n\nfoo\"").next_token 75 | end 76 | 77 | def test_multiple_tokens 78 | tokenizer = ::KDL::V1::Tokenizer.new("node 1 \"two\" a=3") 79 | 80 | assert_equal t(:IDENT, 'node'), tokenizer.next_token 81 | assert_equal t(:WS, ' ', 1, 5), tokenizer.next_token 82 | assert_equal t(:INTEGER, 1, 1, 6), tokenizer.next_token 83 | assert_equal t(:WS, ' ', 1, 7), tokenizer.next_token 84 | assert_equal t(:STRING, 'two', 1, 8), tokenizer.next_token 85 | assert_equal t(:WS, ' ', 1, 13), tokenizer.next_token 86 | assert_equal t(:IDENT, 'a', 1, 14), tokenizer.next_token 87 | assert_equal t(:EQUALS, '=', 1, 15), tokenizer.next_token 88 | assert_equal t(:INTEGER, 3, 1, 16), tokenizer.next_token 89 | assert_equal t(:EOF, :EOF, 1, 17), tokenizer.next_token 90 | assert_equal eof(1, 17), tokenizer.next_token 91 | end 92 | 93 | def test_single_line_comment 94 | assert_equal t(:EOF, :EOF), ::KDL::V1::Tokenizer.new("// comment").next_token 95 | 96 | tokenizer = ::KDL::V1::Tokenizer.new <<~KDL 97 | node1 98 | // comment 99 | node2 100 | KDL 101 | 102 | assert_equal t(:IDENT, 'node1'), tokenizer.next_token 103 | assert_equal t(:NEWLINE, "\n", 1, 6), tokenizer.next_token 104 | assert_equal t(:NEWLINE, "\n", 2, 11), tokenizer.next_token 105 | assert_equal t(:IDENT, 'node2', 3, 1), tokenizer.next_token 106 | assert_equal t(:NEWLINE, "\n", 3, 6), tokenizer.next_token 107 | assert_equal t(:EOF, :EOF, 4, 1), tokenizer.next_token 108 | assert_equal eof(4, 1), tokenizer.next_token 109 | end 110 | 111 | def test_multiline_comment 112 | tokenizer = ::KDL::V1::Tokenizer.new("foo /*bar=1*/ baz=2") 113 | 114 | assert_equal t(:IDENT, 'foo'), tokenizer.next_token 115 | assert_equal t(:WS, ' ', 1, 4), tokenizer.next_token 116 | assert_equal t(:IDENT, 'baz', 1, 15), tokenizer.next_token 117 | assert_equal t(:EQUALS, '=', 1, 18), tokenizer.next_token 118 | assert_equal t(:INTEGER, 2, 1, 19), tokenizer.next_token 119 | assert_equal t(:EOF, :EOF, 1, 20), tokenizer.next_token 120 | assert_equal eof(1, 20), tokenizer.next_token 121 | end 122 | 123 | def test_utf8 124 | assert_equal t(:IDENT, '😁'), ::KDL::V1::Tokenizer.new("😁").next_token 125 | assert_equal t(:STRING, '😁'), ::KDL::V1::Tokenizer.new('"😁"').next_token 126 | assert_equal t(:IDENT, 'ノード'), ::KDL::V1::Tokenizer.new('ノード').next_token 127 | assert_equal t(:IDENT, 'お名前'), ::KDL::V1::Tokenizer.new('お名前').next_token 128 | assert_equal t(:STRING, '☜(ﾟヮﾟ☜)'), ::KDL::V1::Tokenizer.new('"☜(ﾟヮﾟ☜)"').next_token 129 | 130 | tokenizer = ::KDL::V1::Tokenizer.new <<~KDL 131 | smile "😁" 132 | ノード お名前="☜(ﾟヮﾟ☜)" 133 | KDL 134 | 135 | assert_equal t(:IDENT, 'smile'), tokenizer.next_token 136 | assert_equal t(:WS, ' ', 1, 6), tokenizer.next_token 137 | assert_equal t(:STRING, '😁', 1, 7), tokenizer.next_token 138 | assert_equal t(:NEWLINE, "\n", 1, 10), tokenizer.next_token 139 | assert_equal t(:IDENT, 'ノード', 2, 1), tokenizer.next_token 140 | assert_equal t(:WS, ' ', 2, 4), tokenizer.next_token 141 | assert_equal t(:IDENT, 'お名前', 2, 5), tokenizer.next_token 142 | assert_equal t(:EQUALS, '=', 2, 8), tokenizer.next_token 143 | assert_equal t(:STRING, '☜(ﾟヮﾟ☜)', 2, 9), tokenizer.next_token 144 | assert_equal t(:NEWLINE, "\n", 2, 18), tokenizer.next_token 145 | assert_equal t(:EOF, :EOF, 3, 1), tokenizer.next_token 146 | assert_equal eof(3, 1), tokenizer.next_token 147 | end 148 | 149 | def test_semicolon 150 | tokenizer = ::KDL::V1::Tokenizer.new 'node1; node2' 151 | 152 | assert_equal t(:IDENT, 'node1'), tokenizer.next_token 153 | assert_equal t(:SEMICOLON, ';', 1, 6), tokenizer.next_token 154 | assert_equal t(:WS, ' ', 1, 7), tokenizer.next_token 155 | assert_equal t(:IDENT, 'node2', 1, 8), tokenizer.next_token 156 | assert_equal t(:EOF, :EOF, 1, 13), tokenizer.next_token 157 | assert_equal eof(1, 13), tokenizer.next_token 158 | end 159 | 160 | def test_slash_dash 161 | tokenizer = ::KDL::V1::Tokenizer.new <<~KDL 162 | /-mynode /-"foo" /-key=1 /-{ 163 | a 164 | } 165 | KDL 166 | 167 | assert_equal t(:SLASHDASH, '/-'), tokenizer.next_token 168 | assert_equal t(:IDENT, 'mynode', 1, 3), tokenizer.next_token 169 | assert_equal t(:WS, ' ', 1, 9), tokenizer.next_token 170 | assert_equal t(:SLASHDASH, '/-', 1, 10), tokenizer.next_token 171 | assert_equal t(:STRING, 'foo', 1, 12), tokenizer.next_token 172 | assert_equal t(:WS, ' ', 1, 17), tokenizer.next_token 173 | assert_equal t(:SLASHDASH, '/-', 1, 18), tokenizer.next_token 174 | assert_equal t(:IDENT, 'key', 1, 20), tokenizer.next_token 175 | assert_equal t(:EQUALS, '=', 1, 23), tokenizer.next_token 176 | assert_equal t(:INTEGER, 1, 1, 24), tokenizer.next_token 177 | assert_equal t(:WS, ' ', 1, 25), tokenizer.next_token 178 | assert_equal t(:SLASHDASH, '/-', 1, 26), tokenizer.next_token 179 | assert_equal t(:LBRACE, '{', 1, 28), tokenizer.next_token 180 | assert_equal t(:NEWLINE, "\n", 1, 29), tokenizer.next_token 181 | assert_equal t(:WS, ' ', 2, 1), tokenizer.next_token 182 | assert_equal t(:IDENT, 'a', 2, 3), tokenizer.next_token 183 | assert_equal t(:NEWLINE, "\n", 2, 4), tokenizer.next_token 184 | assert_equal t(:RBRACE, '}', 3, 1), tokenizer.next_token 185 | assert_equal t(:NEWLINE, "\n", 3, 2), tokenizer.next_token 186 | assert_equal t(:EOF, :EOF, 4, 1), tokenizer.next_token 187 | assert_equal eof(4, 1), tokenizer.next_token 188 | end 189 | 190 | def test_multiline_nodes 191 | tokenizer = ::KDL::V1::Tokenizer.new <<~KDL 192 | title \\ 193 | "Some title" 194 | KDL 195 | 196 | assert_equal t(:IDENT, 'title'), tokenizer.next_token 197 | assert_equal t(:WS, " \\\n ", 1, 6), tokenizer.next_token 198 | assert_equal t(:STRING, 'Some title', 2, 3), tokenizer.next_token 199 | assert_equal t(:NEWLINE, "\n", 2, 15), tokenizer.next_token 200 | assert_equal t(:EOF, :EOF, 3, 1), tokenizer.next_token 201 | assert_equal eof(3, 1), tokenizer.next_token 202 | end 203 | 204 | def test_multiline_nodes_with_comment 205 | tokenizer = ::KDL::V1::Tokenizer.new <<~KDL 206 | title \\ // some comment 207 | "Some title" 208 | KDL 209 | 210 | assert_equal t(:IDENT, 'title'), tokenizer.next_token 211 | assert_equal t(:WS, " \\ \n ", 1, 6), tokenizer.next_token 212 | assert_equal t(:STRING, 'Some title', 2, 3), tokenizer.next_token 213 | assert_equal t(:NEWLINE, "\n", 2, 15), tokenizer.next_token 214 | assert_equal t(:EOF, :EOF, 3, 1), tokenizer.next_token 215 | assert_equal eof(3, 1), tokenizer.next_token 216 | end 217 | 218 | private 219 | 220 | def t(type, value, line = 1, col = 1) 221 | [type, ::KDL::V1::Tokenizer::Token.new(type, value, line, col)] 222 | end 223 | 224 | def eof(line = 1, col = 1) 225 | [false, ::KDL::V1::Tokenizer::Token.new(:EOF, :EOF, line, col)] 226 | end 227 | end 228 | -------------------------------------------------------------------------------- /test/tokenizer_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "test_helper" 4 | 5 | class TokenizerTest < Minitest::Test 6 | def test_identifier 7 | assert_equal t(:IDENT, "foo"), ::KDL::Tokenizer.new("foo").next_token 8 | assert_equal t(:IDENT, "foo-bar123"), ::KDL::Tokenizer.new("foo-bar123").next_token 9 | assert_equal t(:IDENT, "-"), ::KDL::Tokenizer.new("-").next_token 10 | assert_equal t(:IDENT, "--"), ::KDL::Tokenizer.new("--").next_token 11 | end 12 | 13 | def test_string 14 | assert_equal t(:STRING, "foo"), ::KDL::Tokenizer.new('"foo"').next_token 15 | assert_equal t(:STRING, "foo\nbar"), ::KDL::Tokenizer.new('"foo\nbar"').next_token 16 | assert_equal t(:STRING, "foo"), ::KDL::V1::Tokenizer.new("\"\\\n\n\nfoo\"").next_token 17 | end 18 | 19 | def test_rawstring 20 | assert_equal t(:RAWSTRING, "foo\\nbar"), ::KDL::Tokenizer.new('#"foo\\nbar"#').next_token 21 | assert_equal t(:RAWSTRING, "foo\"bar"), ::KDL::Tokenizer.new('#"foo"bar"#').next_token 22 | assert_equal t(:RAWSTRING, "foo\"#bar"), ::KDL::Tokenizer.new('##"foo"#bar"##').next_token 23 | assert_equal t(:RAWSTRING, "\"foo\""), ::KDL::Tokenizer.new('#""foo""#').next_token 24 | 25 | tokenizer = ::KDL::Tokenizer.new('node #"C:\\Users\\zkat\\"#') 26 | assert_equal t(:IDENT, "node"), tokenizer.next_token 27 | assert_equal t(:WS, " ", 1, 5), tokenizer.next_token 28 | assert_equal t(:RAWSTRING, "C:\\Users\\zkat\\", 1, 6), tokenizer.next_token 29 | 30 | tokenizer = ::KDL::Tokenizer.new('other-node #"hello"world"#') 31 | assert_equal t(:IDENT, "other-node"), tokenizer.next_token 32 | assert_equal t(:WS, " ", 1, 11), tokenizer.next_token 33 | assert_equal t(:RAWSTRING, "hello\"world", 1, 12), tokenizer.next_token 34 | end 35 | 36 | def test_integer 37 | assert_equal t(:INTEGER, 123), ::KDL::Tokenizer.new("123").next_token 38 | assert_equal t(:INTEGER, 0x0123456789abcdef), ::KDL::Tokenizer.new("0x0123456789abcdef").next_token 39 | assert_equal t(:INTEGER, 0o01234567), ::KDL::Tokenizer.new("0o01234567").next_token 40 | assert_equal t(:INTEGER, 0b101001), ::KDL::Tokenizer.new("0b101001").next_token 41 | assert_equal t(:INTEGER, -0x0123456789abcdef), ::KDL::Tokenizer.new("-0x0123456789abcdef").next_token 42 | assert_equal t(:INTEGER, -0o01234567), ::KDL::Tokenizer.new("-0o01234567").next_token 43 | assert_equal t(:INTEGER, -0b101001), ::KDL::Tokenizer.new("-0b101001").next_token 44 | end 45 | 46 | def test_float 47 | assert_equal t(:FLOAT, 1.23), ::KDL::Tokenizer.new("1.23").next_token 48 | assert_equal t(:FLOAT, Float::INFINITY), ::KDL::Tokenizer.new("#inf").next_token 49 | assert_equal t(:FLOAT, -Float::INFINITY), ::KDL::Tokenizer.new("#-inf").next_token 50 | nan = ::KDL::Tokenizer.new("#nan").next_token 51 | assert_equal :FLOAT, nan[0] 52 | assert_predicate nan[1].value, :nan? 53 | end 54 | 55 | def test_boolean 56 | assert_equal t(:TRUE, true), ::KDL::Tokenizer.new("#true").next_token 57 | assert_equal t(:FALSE, false), ::KDL::Tokenizer.new("#false").next_token 58 | end 59 | 60 | def test_null 61 | assert_equal t(:NULL, nil), ::KDL::Tokenizer.new("#null").next_token 62 | end 63 | 64 | def test_symbols 65 | assert_equal t(:LBRACE, '{'), ::KDL::Tokenizer.new("{").next_token 66 | assert_equal t(:RBRACE, '}'), ::KDL::Tokenizer.new("}").next_token 67 | end 68 | 69 | def test_equals 70 | assert_equal t(:EQUALS, '='), ::KDL::Tokenizer.new("=").next_token 71 | assert_equal t(:EQUALS, ' ='), ::KDL::Tokenizer.new(" =").next_token 72 | assert_equal t(:EQUALS, '= '), ::KDL::Tokenizer.new("= ").next_token 73 | assert_equal t(:EQUALS, ' = '), ::KDL::Tokenizer.new(" = ").next_token 74 | assert_equal t(:EQUALS, ' ='), ::KDL::Tokenizer.new(" =foo").next_token 75 | end 76 | 77 | def test_whitespace 78 | assert_equal t(:WS, ' '), ::KDL::Tokenizer.new(" ").next_token 79 | assert_equal t(:WS, "\t"), ::KDL::Tokenizer.new("\t").next_token 80 | assert_equal t(:WS, " \t"), ::KDL::Tokenizer.new(" \t").next_token 81 | assert_equal t(:WS, "\\\n"), ::KDL::Tokenizer.new("\\\n").next_token 82 | assert_equal t(:WS, "\\\n"), ::KDL::Tokenizer.new("\\\n//some comment").next_token 83 | assert_equal t(:WS, "\\\n "), ::KDL::Tokenizer.new("\\\n //some comment").next_token 84 | assert_equal t(:WS, " \\\n"), ::KDL::Tokenizer.new(" \\\n").next_token 85 | assert_equal t(:WS, " \\\n"), ::KDL::Tokenizer.new(" \\\n//some comment").next_token 86 | assert_equal t(:WS, " \\\n "), ::KDL::Tokenizer.new(" \\\n //some comment").next_token 87 | assert_equal t(:WS, " \\\n \\\n "), ::KDL::Tokenizer.new(" \\\n \\\n ").next_token 88 | end 89 | 90 | def test_multiple_tokens 91 | tokenizer = ::KDL::Tokenizer.new("node 1 \"two\" a=3") 92 | 93 | assert_equal t(:IDENT, 'node'), tokenizer.next_token 94 | assert_equal t(:WS, ' ', 1, 5), tokenizer.next_token 95 | assert_equal t(:INTEGER, 1, 1, 6), tokenizer.next_token 96 | assert_equal t(:WS, ' ', 1, 7), tokenizer.next_token 97 | assert_equal t(:STRING, 'two', 1, 8), tokenizer.next_token 98 | assert_equal t(:WS, ' ', 1, 13), tokenizer.next_token 99 | assert_equal t(:IDENT, 'a', 1, 14), tokenizer.next_token 100 | assert_equal t(:EQUALS, '=', 1, 15), tokenizer.next_token 101 | assert_equal t(:INTEGER, 3, 1, 16), tokenizer.next_token 102 | assert_equal t(:EOF, :EOF, 1, 17), tokenizer.next_token 103 | assert_equal eof(1, 17), tokenizer.next_token 104 | end 105 | 106 | def test_single_line_comment 107 | assert_equal t(:EOF, :EOF), ::KDL::Tokenizer.new("// comment").next_token 108 | 109 | tokenizer = ::KDL::Tokenizer.new <<~KDL 110 | node1 111 | // comment 112 | node2 113 | KDL 114 | 115 | assert_equal t(:IDENT, 'node1'), tokenizer.next_token 116 | assert_equal t(:NEWLINE, "\n", 1, 6), tokenizer.next_token 117 | assert_equal t(:NEWLINE, "\n", 2, 11), tokenizer.next_token 118 | assert_equal t(:IDENT, 'node2', 3, 1), tokenizer.next_token 119 | assert_equal t(:NEWLINE, "\n", 3, 6), tokenizer.next_token 120 | assert_equal t(:EOF, :EOF, 4, 1), tokenizer.next_token 121 | assert_equal eof(4, 1), tokenizer.next_token 122 | end 123 | 124 | def test_multiline_comment 125 | tokenizer = ::KDL::Tokenizer.new("foo /*bar=1*/ baz=2") 126 | 127 | assert_equal t(:IDENT, 'foo'), tokenizer.next_token 128 | assert_equal t(:WS, ' ', 1, 4), tokenizer.next_token 129 | assert_equal t(:IDENT, 'baz', 1, 15), tokenizer.next_token 130 | assert_equal t(:EQUALS, '=', 1, 18), tokenizer.next_token 131 | assert_equal t(:INTEGER, 2, 1, 19), tokenizer.next_token 132 | assert_equal t(:EOF, :EOF, 1, 20), tokenizer.next_token 133 | assert_equal eof(1, 20), tokenizer.next_token 134 | end 135 | 136 | def test_utf8 137 | assert_equal t(:IDENT, '😁'), ::KDL::Tokenizer.new("😁").next_token 138 | assert_equal t(:STRING, '😁'), ::KDL::Tokenizer.new('"😁"').next_token 139 | assert_equal t(:IDENT, 'ノード'), ::KDL::Tokenizer.new('ノード').next_token 140 | assert_equal t(:IDENT, 'お名前'), ::KDL::Tokenizer.new('お名前').next_token 141 | assert_equal t(:STRING, '☜(ﾟヮﾟ☜)'), ::KDL::Tokenizer.new('"☜(ﾟヮﾟ☜)"').next_token 142 | 143 | tokenizer = ::KDL::Tokenizer.new <<~KDL 144 | smile "😁" 145 | ノード お名前="☜(ﾟヮﾟ☜)" 146 | KDL 147 | 148 | assert_equal t(:IDENT, 'smile'), tokenizer.next_token 149 | assert_equal t(:WS, ' ', 1, 6), tokenizer.next_token 150 | assert_equal t(:STRING, '😁', 1, 7), tokenizer.next_token 151 | assert_equal t(:NEWLINE, "\n", 1, 10), tokenizer.next_token 152 | assert_equal t(:IDENT, 'ノード', 2, 1), tokenizer.next_token 153 | assert_equal t(:WS, ' ', 2, 4), tokenizer.next_token 154 | assert_equal t(:IDENT, 'お名前', 2, 5), tokenizer.next_token 155 | assert_equal t(:EQUALS, '=', 2, 8), tokenizer.next_token 156 | assert_equal t(:STRING, '☜(ﾟヮﾟ☜)', 2, 9), tokenizer.next_token 157 | assert_equal t(:NEWLINE, "\n", 2, 18), tokenizer.next_token 158 | assert_equal t(:EOF, :EOF, 3, 1), tokenizer.next_token 159 | assert_equal eof(3, 1), tokenizer.next_token 160 | end 161 | 162 | def test_semicolon 163 | tokenizer = ::KDL::Tokenizer.new 'node1; node2' 164 | 165 | assert_equal t(:IDENT, 'node1'), tokenizer.next_token 166 | assert_equal t(:SEMICOLON, ';', 1, 6), tokenizer.next_token 167 | assert_equal t(:WS, ' ', 1, 7), tokenizer.next_token 168 | assert_equal t(:IDENT, 'node2', 1, 8), tokenizer.next_token 169 | assert_equal t(:EOF, :EOF, 1, 13), tokenizer.next_token 170 | assert_equal eof(1, 13), tokenizer.next_token 171 | end 172 | 173 | def test_slash_dash 174 | tokenizer = ::KDL::Tokenizer.new <<~KDL 175 | /-mynode /-"foo" /-key=1 /-{ 176 | a 177 | } 178 | KDL 179 | 180 | assert_equal t(:SLASHDASH, '/-'), tokenizer.next_token 181 | assert_equal t(:IDENT, 'mynode', 1, 3), tokenizer.next_token 182 | assert_equal t(:WS, ' ', 1, 9), tokenizer.next_token 183 | assert_equal t(:SLASHDASH, '/-', 1, 10), tokenizer.next_token 184 | assert_equal t(:STRING, 'foo', 1, 12), tokenizer.next_token 185 | assert_equal t(:WS, ' ', 1, 17), tokenizer.next_token 186 | assert_equal t(:SLASHDASH, '/-', 1, 18), tokenizer.next_token 187 | assert_equal t(:IDENT, 'key', 1, 20), tokenizer.next_token 188 | assert_equal t(:EQUALS, '=', 1, 23), tokenizer.next_token 189 | assert_equal t(:INTEGER, 1, 1, 24), tokenizer.next_token 190 | assert_equal t(:WS, ' ', 1, 25), tokenizer.next_token 191 | assert_equal t(:SLASHDASH, '/-', 1, 26), tokenizer.next_token 192 | assert_equal t(:LBRACE, '{', 1, 28), tokenizer.next_token 193 | assert_equal t(:NEWLINE, "\n", 1, 29), tokenizer.next_token 194 | assert_equal t(:WS, ' ', 2, 1), tokenizer.next_token 195 | assert_equal t(:IDENT, 'a', 2, 3), tokenizer.next_token 196 | assert_equal t(:NEWLINE, "\n", 2, 4), tokenizer.next_token 197 | assert_equal t(:RBRACE, '}', 3, 1), tokenizer.next_token 198 | assert_equal t(:NEWLINE, "\n", 3, 2), tokenizer.next_token 199 | assert_equal t(:EOF, :EOF, 4, 1), tokenizer.next_token 200 | assert_equal eof(4, 1), tokenizer.next_token 201 | end 202 | 203 | def test_multiline_nodes 204 | tokenizer = ::KDL::Tokenizer.new <<~KDL 205 | title \\ 206 | "Some title" 207 | KDL 208 | 209 | assert_equal t(:IDENT, 'title'), tokenizer.next_token 210 | assert_equal t(:WS, " \\\n ", 1, 6), tokenizer.next_token 211 | assert_equal t(:STRING, 'Some title', 2, 3), tokenizer.next_token 212 | assert_equal t(:NEWLINE, "\n", 2, 15), tokenizer.next_token 213 | assert_equal t(:EOF, :EOF, 3, 1), tokenizer.next_token 214 | assert_equal eof(3, 1), tokenizer.next_token 215 | end 216 | 217 | def test_tokens 218 | tokenizer = ::KDL::Tokenizer.new "node1 {\n foo\n bar\n}" 219 | 220 | assert_equal %i[IDENT WS LBRACE NEWLINE WS IDENT NEWLINE WS IDENT NEWLINE RBRACE EOF], 221 | tokenizer.tokens.map(&:first) 222 | end 223 | 224 | private 225 | 226 | def t(type, value, line = 1, col = 1) 227 | [type, ::KDL::Tokenizer::Token.new(type, value, line, col)] 228 | end 229 | 230 | def eof(line = 1, col = 1) 231 | [false, ::KDL::Tokenizer::Token.new(:EOF, :EOF, line, col)] 232 | end 233 | end 234 | -------------------------------------------------------------------------------- /lib/kdl/types/currency/iso4217_currencies.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module KDL 4 | module Types 5 | class Currency < Value::Custom 6 | # From https://en.wikipedia.org/wiki/ISO_4217#Active_codes 7 | CURRENCIES = { 8 | 'AED' => { numeric_code: 784, minor_unit: 2, name: 'United Arab Emirates dirham' }.freeze, 9 | 'AFN' => { numeric_code: 971, minor_unit: 2, name: 'Afghan afghani' }.freeze, 10 | 'ALL' => { numeric_code: 8, minor_unit: 2, name: 'Albanian lek' }.freeze, 11 | 'AMD' => { numeric_code: 51, minor_unit: 2, name: 'Armenian dram' }.freeze, 12 | 'ANG' => { numeric_code: 532, minor_unit: 2, name: 'Netherlands Antillean guilder' }.freeze, 13 | 'AOA' => { numeric_code: 973, minor_unit: 2, name: 'Angolan kwanza' }.freeze, 14 | 'ARS' => { numeric_code: 32, minor_unit: 2, name: 'Argentine peso' }.freeze, 15 | 'AUD' => { numeric_code: 36, minor_unit: 2, name: 'Australian dollar' }.freeze, 16 | 'AWG' => { numeric_code: 533, minor_unit: 2, name: 'Aruban florin' }.freeze, 17 | 'AZN' => { numeric_code: 944, minor_unit: 2, name: 'Azerbaijani manat' }.freeze, 18 | 'BAM' => { numeric_code: 977, minor_unit: 2, name: 'Bosnia and Herzegovina convertible mark' }.freeze, 19 | 'BBD' => { numeric_code: 52, minor_unit: 2, name: 'Barbados dollar' }.freeze, 20 | 'BDT' => { numeric_code: 50, minor_unit: 2, name: 'Bangladeshi taka' }.freeze, 21 | 'BGN' => { numeric_code: 975, minor_unit: 2, name: 'Bulgarian lev' }.freeze, 22 | 'BHD' => { numeric_code: 48, minor_unit: 3, name: 'Bahraini dinar' }.freeze, 23 | 'BIF' => { numeric_code: 108, minor_unit: 0, name: 'Burundian franc' }.freeze, 24 | 'BMD' => { numeric_code: 60, minor_unit: 2, name: 'Bermudian dollar' }.freeze, 25 | 'BND' => { numeric_code: 96, minor_unit: 2, name: 'Brunei dollar' }.freeze, 26 | 'BOB' => { numeric_code: 68, minor_unit: 2, name: 'Boliviano' }.freeze, 27 | 'BOV' => { numeric_code: 984, minor_unit: 2, name: 'Bolivian Mvdol (funds code)' }.freeze, 28 | 'BRL' => { numeric_code: 986, minor_unit: 2, name: 'Brazilian real' }.freeze, 29 | 'BSD' => { numeric_code: 44, minor_unit: 2, name: 'Bahamian dollar' }.freeze, 30 | 'BTN' => { numeric_code: 64, minor_unit: 2, name: 'Bhutanese ngultrum' }.freeze, 31 | 'BWP' => { numeric_code: 72, minor_unit: 2, name: 'Botswana pula' }.freeze, 32 | 'BYN' => { numeric_code: 933, minor_unit: 2, name: 'Belarusian ruble' }.freeze, 33 | 'BZD' => { numeric_code: 84, minor_unit: 2, name: 'Belize dollar' }.freeze, 34 | 'CAD' => { numeric_code: 124, minor_unit: 2, name: 'Canadian dollar' }.freeze, 35 | 'CDF' => { numeric_code: 976, minor_unit: 2, name: 'Congolese franc' }.freeze, 36 | 'CHE' => { numeric_code: 947, minor_unit: 2, name: 'WIR euro (complementary currency)' }.freeze, 37 | 'CHF' => { numeric_code: 756, minor_unit: 2, name: 'Swiss franc' }.freeze, 38 | 'CHW' => { numeric_code: 948, minor_unit: 2, name: 'WIR franc (complementary currency)' }.freeze, 39 | 'CLF' => { numeric_code: 990, minor_unit: 4, name: 'Unidad de Fomento (funds code)' }.freeze, 40 | 'CLP' => { numeric_code: 152, minor_unit: 0, name: 'Chilean peso' }.freeze, 41 | 'CNY' => { numeric_code: 156, minor_unit: 2, name: 'Chinese yuan[8]' }.freeze, 42 | 'COP' => { numeric_code: 170, minor_unit: 2, name: 'Colombian peso' }.freeze, 43 | 'COU' => { numeric_code: 970, minor_unit: 2, name: 'Unidad de Valor Real (UVR) (funds code)' }.freeze, 44 | 'CRC' => { numeric_code: 188, minor_unit: 2, name: 'Costa Rican colon' }.freeze, 45 | 'CUC' => { numeric_code: 931, minor_unit: 2, name: 'Cuban convertible peso' }.freeze, 46 | 'CUP' => { numeric_code: 192, minor_unit: 2, name: 'Cuban peso' }.freeze, 47 | 'CVE' => { numeric_code: 132, minor_unit: 2, name: 'Cape Verdean escudo' }.freeze, 48 | 'CZK' => { numeric_code: 203, minor_unit: 2, name: 'Czech koruna' }.freeze, 49 | 'DJF' => { numeric_code: 262, minor_unit: 0, name: 'Djiboutian franc' }.freeze, 50 | 'DKK' => { numeric_code: 208, minor_unit: 2, name: 'Danish krone' }.freeze, 51 | 'DOP' => { numeric_code: 214, minor_unit: 2, name: 'Dominican peso' }.freeze, 52 | 'DZD' => { numeric_code: 12, minor_unit: 2, name: 'Algerian dinar' }.freeze, 53 | 'EGP' => { numeric_code: 818, minor_unit: 2, name: 'Egyptian pound' }.freeze, 54 | 'ERN' => { numeric_code: 232, minor_unit: 2, name: 'Eritrean nakfa' }.freeze, 55 | 'ETB' => { numeric_code: 230, minor_unit: 2, name: 'Ethiopian birr' }.freeze, 56 | 'EUR' => { numeric_code: 978, minor_unit: 2, name: 'Euro' }.freeze, 57 | 'FJD' => { numeric_code: 242, minor_unit: 2, name: 'Fiji dollar' }.freeze, 58 | 'FKP' => { numeric_code: 238, minor_unit: 2, name: 'Falkland Islands pound' }.freeze, 59 | 'GBP' => { numeric_code: 826, minor_unit: 2, name: 'Pound sterling' }.freeze, 60 | 'GEL' => { numeric_code: 981, minor_unit: 2, name: 'Georgian lari' }.freeze, 61 | 'GHS' => { numeric_code: 936, minor_unit: 2, name: 'Ghanaian cedi' }.freeze, 62 | 'GIP' => { numeric_code: 292, minor_unit: 2, name: 'Gibraltar pound' }.freeze, 63 | 'GMD' => { numeric_code: 270, minor_unit: 2, name: 'Gambian dalasi' }.freeze, 64 | 'GNF' => { numeric_code: 324, minor_unit: 0, name: 'Guinean franc' }.freeze, 65 | 'GTQ' => { numeric_code: 320, minor_unit: 2, name: 'Guatemalan quetzal' }.freeze, 66 | 'GYD' => { numeric_code: 328, minor_unit: 2, name: 'Guyanese dollar' }.freeze, 67 | 'HKD' => { numeric_code: 344, minor_unit: 2, name: 'Hong Kong dollar' }.freeze, 68 | 'HNL' => { numeric_code: 340, minor_unit: 2, name: 'Honduran lempira' }.freeze, 69 | 'HRK' => { numeric_code: 191, minor_unit: 2, name: 'Croatian kuna' }.freeze, 70 | 'HTG' => { numeric_code: 332, minor_unit: 2, name: 'Haitian gourde' }.freeze, 71 | 'HUF' => { numeric_code: 348, minor_unit: 2, name: 'Hungarian forint' }.freeze, 72 | 'IDR' => { numeric_code: 360, minor_unit: 2, name: 'Indonesian rupiah' }.freeze, 73 | 'ILS' => { numeric_code: 376, minor_unit: 2, name: 'Israeli new shekel' }.freeze, 74 | 'INR' => { numeric_code: 356, minor_unit: 2, name: 'Indian rupee' }.freeze, 75 | 'IQD' => { numeric_code: 368, minor_unit: 3, name: 'Iraqi dinar' }.freeze, 76 | 'IRR' => { numeric_code: 364, minor_unit: 2, name: 'Iranian rial' }.freeze, 77 | 'ISK' => { numeric_code: 352, minor_unit: 0, name: 'Icelandic króna (plural: krónur)' }.freeze, 78 | 'JMD' => { numeric_code: 388, minor_unit: 2, name: 'Jamaican dollar' }.freeze, 79 | 'JOD' => { numeric_code: 400, minor_unit: 3, name: 'Jordanian dinar' }.freeze, 80 | 'JPY' => { numeric_code: 392, minor_unit: 0, name: 'Japanese yen' }.freeze, 81 | 'KES' => { numeric_code: 404, minor_unit: 2, name: 'Kenyan shilling' }.freeze, 82 | 'KGS' => { numeric_code: 417, minor_unit: 2, name: 'Kyrgyzstani som' }.freeze, 83 | 'KHR' => { numeric_code: 116, minor_unit: 2, name: 'Cambodian riel' }.freeze, 84 | 'KMF' => { numeric_code: 174, minor_unit: 0, name: 'Comoro franc' }.freeze, 85 | 'KPW' => { numeric_code: 408, minor_unit: 2, name: 'North Korean won' }.freeze, 86 | 'KRW' => { numeric_code: 410, minor_unit: 0, name: 'South Korean won' }.freeze, 87 | 'KWD' => { numeric_code: 414, minor_unit: 3, name: 'Kuwaiti dinar' }.freeze, 88 | 'KYD' => { numeric_code: 136, minor_unit: 2, name: 'Cayman Islands dollar' }.freeze, 89 | 'KZT' => { numeric_code: 398, minor_unit: 2, name: 'Kazakhstani tenge' }.freeze, 90 | 'LAK' => { numeric_code: 418, minor_unit: 2, name: 'Lao kip' }.freeze, 91 | 'LBP' => { numeric_code: 422, minor_unit: 2, name: 'Lebanese pound' }.freeze, 92 | 'LKR' => { numeric_code: 144, minor_unit: 2, name: 'Sri Lankan rupee' }.freeze, 93 | 'LRD' => { numeric_code: 430, minor_unit: 2, name: 'Liberian dollar' }.freeze, 94 | 'LSL' => { numeric_code: 426, minor_unit: 2, name: 'Lesotho loti' }.freeze, 95 | 'LYD' => { numeric_code: 434, minor_unit: 3, name: 'Libyan dinar' }.freeze, 96 | 'MAD' => { numeric_code: 504, minor_unit: 2, name: 'Moroccan dirham' }.freeze, 97 | 'MDL' => { numeric_code: 498, minor_unit: 2, name: 'Moldovan leu' }.freeze, 98 | 'MGA' => { numeric_code: 969, minor_unit: 2, name: 'Malagasy ariary' }.freeze, 99 | 'MKD' => { numeric_code: 807, minor_unit: 2, name: 'Macedonian denar' }.freeze, 100 | 'MMK' => { numeric_code: 104, minor_unit: 2, name: 'Myanmar kyat' }.freeze, 101 | 'MNT' => { numeric_code: 496, minor_unit: 2, name: 'Mongolian tögrög' }.freeze, 102 | 'MOP' => { numeric_code: 446, minor_unit: 2, name: 'Macanese pataca' }.freeze, 103 | 'MRU' => { numeric_code: 929, minor_unit: 2, name: 'Mauritanian ouguiya' }.freeze, 104 | 'MUR' => { numeric_code: 480, minor_unit: 2, name: 'Mauritian rupee' }.freeze, 105 | 'MVR' => { numeric_code: 462, minor_unit: 2, name: 'Maldivian rufiyaa' }.freeze, 106 | 'MWK' => { numeric_code: 454, minor_unit: 2, name: 'Malawian kwacha' }.freeze, 107 | 'MXN' => { numeric_code: 484, minor_unit: 2, name: 'Mexican peso' }.freeze, 108 | 'MXV' => { numeric_code: 979, minor_unit: 2, name: 'Mexican Unidad de Inversion (UDI) (funds code)' }.freeze, 109 | 'MYR' => { numeric_code: 458, minor_unit: 2, name: 'Malaysian ringgit' }.freeze, 110 | 'MZN' => { numeric_code: 943, minor_unit: 2, name: 'Mozambican metical' }.freeze, 111 | 'NAD' => { numeric_code: 516, minor_unit: 2, name: 'Namibian dollar' }.freeze, 112 | 'NGN' => { numeric_code: 566, minor_unit: 2, name: 'Nigerian naira' }.freeze, 113 | 'NIO' => { numeric_code: 558, minor_unit: 2, name: 'Nicaraguan córdoba' }.freeze, 114 | 'NOK' => { numeric_code: 578, minor_unit: 2, name: 'Norwegian krone' }.freeze, 115 | 'NPR' => { numeric_code: 524, minor_unit: 2, name: 'Nepalese rupee' }.freeze, 116 | 'NZD' => { numeric_code: 554, minor_unit: 2, name: 'New Zealand dollar' }.freeze, 117 | 'OMR' => { numeric_code: 512, minor_unit: 3, name: 'Omani rial' }.freeze, 118 | 'PAB' => { numeric_code: 590, minor_unit: 2, name: 'Panamanian balboa' }.freeze, 119 | 'PEN' => { numeric_code: 604, minor_unit: 2, name: 'Peruvian sol' }.freeze, 120 | 'PGK' => { numeric_code: 598, minor_unit: 2, name: 'Papua New Guinean kina' }.freeze, 121 | 'PHP' => { numeric_code: 608, minor_unit: 2, name: 'Philippine peso' }.freeze, 122 | 'PKR' => { numeric_code: 586, minor_unit: 2, name: 'Pakistani rupee' }.freeze, 123 | 'PLN' => { numeric_code: 985, minor_unit: 2, name: 'Polish złoty' }.freeze, 124 | 'PYG' => { numeric_code: 600, minor_unit: 0, name: 'Paraguayan guaraní' }.freeze, 125 | 'QAR' => { numeric_code: 634, minor_unit: 2, name: 'Qatari riyal' }.freeze, 126 | 'RON' => { numeric_code: 946, minor_unit: 2, name: 'Romanian leu' }.freeze, 127 | 'RSD' => { numeric_code: 941, minor_unit: 2, name: 'Serbian dinar' }.freeze, 128 | 'RUB' => { numeric_code: 643, minor_unit: 2, name: 'Russian ruble' }.freeze, 129 | 'RWF' => { numeric_code: 646, minor_unit: 0, name: 'Rwandan franc' }.freeze, 130 | 'SAR' => { numeric_code: 682, minor_unit: 2, name: 'Saudi riyal' }.freeze, 131 | 'SBD' => { numeric_code: 90, minor_unit: 2, name: 'Solomon Islands dollar' }.freeze, 132 | 'SCR' => { numeric_code: 690, minor_unit: 2, name: 'Seychelles rupee' }.freeze, 133 | 'SDG' => { numeric_code: 938, minor_unit: 2, name: 'Sudanese pound' }.freeze, 134 | 'SEK' => { numeric_code: 752, minor_unit: 2, name: 'Swedish krona (plural: kronor)' }.freeze, 135 | 'SGD' => { numeric_code: 702, minor_unit: 2, name: 'Singapore dollar' }.freeze, 136 | 'SHP' => { numeric_code: 654, minor_unit: 2, name: 'Saint Helena pound' }.freeze, 137 | 'SLL' => { numeric_code: 694, minor_unit: 2, name: 'Sierra Leonean leone' }.freeze, 138 | 'SOS' => { numeric_code: 706, minor_unit: 2, name: 'Somali shilling' }.freeze, 139 | 'SRD' => { numeric_code: 968, minor_unit: 2, name: 'Surinamese dollar' }.freeze, 140 | 'SSP' => { numeric_code: 728, minor_unit: 2, name: 'South Sudanese pound' }.freeze, 141 | 'STN' => { numeric_code: 930, minor_unit: 2, name: 'São Tomé and Príncipe dobra' }.freeze, 142 | 'SVC' => { numeric_code: 222, minor_unit: 2, name: 'Salvadoran colón' }.freeze, 143 | 'SYP' => { numeric_code: 760, minor_unit: 2, name: 'Syrian pound' }.freeze, 144 | 'SZL' => { numeric_code: 748, minor_unit: 2, name: 'Swazi lilangeni' }.freeze, 145 | 'THB' => { numeric_code: 764, minor_unit: 2, name: 'Thai baht' }.freeze, 146 | 'TJS' => { numeric_code: 972, minor_unit: 2, name: 'Tajikistani somoni' }.freeze, 147 | 'TMT' => { numeric_code: 934, minor_unit: 2, name: 'Turkmenistan manat' }.freeze, 148 | 'TND' => { numeric_code: 788, minor_unit: 3, name: 'Tunisian dinar' }.freeze, 149 | 'TOP' => { numeric_code: 776, minor_unit: 2, name: 'Tongan paʻanga' }.freeze, 150 | 'TRY' => { numeric_code: 949, minor_unit: 2, name: 'Turkish lira' }.freeze, 151 | 'TTD' => { numeric_code: 780, minor_unit: 2, name: 'Trinidad and Tobago dollar' }.freeze, 152 | 'TWD' => { numeric_code: 901, minor_unit: 2, name: 'New Taiwan dollar' }.freeze, 153 | 'TZS' => { numeric_code: 834, minor_unit: 2, name: 'Tanzanian shilling' }.freeze, 154 | 'UAH' => { numeric_code: 980, minor_unit: 2, name: 'Ukrainian hryvnia' }.freeze, 155 | 'UGX' => { numeric_code: 800, minor_unit: 0, name: 'Ugandan shilling' }.freeze, 156 | 'USD' => { numeric_code: 840, minor_unit: 2, name: 'United States dollar' }.freeze, 157 | 'USN' => { numeric_code: 997, minor_unit: 2, name: 'United States dollar (next day) (funds code)' }.freeze, 158 | 'UYI' => { numeric_code: 940, minor_unit: 0, name: 'Uruguay Peso en Unidades Indexadas (URUIURUI) (funds code)' }.freeze, 159 | 'UYU' => { numeric_code: 858, minor_unit: 2, name: 'Uruguayan peso' }.freeze, 160 | 'UYW' => { numeric_code: 927, minor_unit: 4, name: 'Unidad previsional' }.freeze, 161 | 'UZS' => { numeric_code: 860, minor_unit: 2, name: 'Uzbekistan som' }.freeze, 162 | 'VED' => { numeric_code: 926, minor_unit: 2, name: 'Venezuelan bolívar digital' }.freeze, 163 | 'VES' => { numeric_code: 928, minor_unit: 2, name: 'Venezuelan bolívar soberano' }.freeze, 164 | 'VND' => { numeric_code: 704, minor_unit: 0, name: 'Vietnamese đồng' }.freeze, 165 | 'VUV' => { numeric_code: 548, minor_unit: 0, name: 'Vanuatu vatu' }.freeze, 166 | 'WST' => { numeric_code: 882, minor_unit: 2, name: 'Samoan tala' }.freeze, 167 | 'XAF' => { numeric_code: 950, minor_unit: 0, name: 'CFA franc BEAC' }.freeze, 168 | 'XAG' => { numeric_code: 961, minor_unit: nil, name: 'Silver (one troy ounce)' }.freeze, 169 | 'XAU' => { numeric_code: 959, minor_unit: nil, name: 'Gold (one troy ounce)' }.freeze, 170 | 'XBA' => { numeric_code: 955, minor_unit: nil, name: 'European Composite Unit (EURCO) (bond market unit)' }.freeze, 171 | 'XBB' => { numeric_code: 956, minor_unit: nil, name: 'European Monetary Unit (E.M.U.-6) (bond market unit)' }.freeze, 172 | 'XBC' => { numeric_code: 957, minor_unit: nil, name: 'European Unit of Account 9 (E.U.A.-9) (bond market unit)' }.freeze, 173 | 'XBD' => { numeric_code: 958, minor_unit: nil, name: 'European Unit of Account 17 (E.U.A.-17) (bond market unit)' }.freeze, 174 | 'XCD' => { numeric_code: 951, minor_unit: 2, name: 'East Caribbean dollar' }.freeze, 175 | 'XDR' => { numeric_code: 960, minor_unit: nil, name: 'Special drawing rights' }.freeze, 176 | 'XOF' => { numeric_code: 952, minor_unit: 0, name: 'CFA franc BCEAO' }.freeze, 177 | 'XPD' => { numeric_code: 964, minor_unit: nil, name: 'Palladium (one troy ounce)' }.freeze, 178 | 'XPF' => { numeric_code: 953, minor_unit: 0, name: 'CFP franc (franc Pacifique)' }.freeze, 179 | 'XPT' => { numeric_code: 962, minor_unit: nil, name: 'Platinum (one troy ounce)' }.freeze, 180 | 'XSU' => { numeric_code: 994, minor_unit: nil, name: 'SUCRE' }.freeze, 181 | 'XTS' => { numeric_code: 963, minor_unit: nil, name: 'Code reserved for testing' }.freeze, 182 | 'XUA' => { numeric_code: 965, minor_unit: nil, name: 'ADB Unit of Account' }.freeze, 183 | 'XXX' => { numeric_code: 999, minor_unit: nil, name: 'No currency' }.freeze, 184 | 'YER' => { numeric_code: 886, minor_unit: 2, name: 'Yemeni rial' }.freeze, 185 | 'ZAR' => { numeric_code: 710, minor_unit: 2, name: 'South African rand' }.freeze, 186 | 'ZMW' => { numeric_code: 967, minor_unit: 2, name: 'Zambian kwacha' }.freeze, 187 | 'ZWL' => { numeric_code: 932, minor_unit: 2, name: 'Zimbabwean dollar' }.freeze 188 | }.freeze 189 | end 190 | end 191 | end 192 | -------------------------------------------------------------------------------- /lib/kdl/tokenizer.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'bigdecimal' 4 | 5 | module KDL 6 | class Tokenizer 7 | class Token 8 | attr_reader :type, :value, :line, :column, :meta 9 | 10 | def initialize(type, value, line, column, meta = {}) 11 | @type = type 12 | @value = value 13 | @line = line 14 | @column = column 15 | @meta = meta 16 | end 17 | 18 | def ==(other) 19 | return false unless other.is_a?(Token) 20 | 21 | type == other.type && value == other.value && line == other.line && column == other.column 22 | end 23 | 24 | def to_s 25 | "#{value.inspect} (#{line}:#{column})" 26 | end 27 | end 28 | 29 | attr_reader :index, :filename 30 | 31 | SYMBOLS = { 32 | '{' => :LBRACE, 33 | '}' => :RBRACE, 34 | ';' => :SEMICOLON, 35 | '=' => :EQUALS 36 | } 37 | 38 | WHITESPACE = ["\u0009", "\u0020", "\u00A0", "\u1680", 39 | "\u2000", "\u2001", "\u2002", "\u2003", 40 | "\u2004", "\u2005", "\u2006", "\u2007", 41 | "\u2008", "\u2009", "\u200A", "\u202F", 42 | "\u205F", "\u3000"] 43 | WS = "[#{Regexp.escape(WHITESPACE.join)}]" 44 | WS_STAR = /\A#{WS}*\z/ 45 | WS_PLUS = /\A#{WS}+\z/ 46 | 47 | NEWLINES = ["\u000A", "\u0085", "\u000B", "\u000C", "\u2028", "\u2029"] 48 | NEWLINES_PATTERN = Regexp.new("(#{NEWLINES.map{Regexp.escape(_1)}.join('|')}|\r\n?)", Regexp::MULTILINE) 49 | 50 | OTHER_NON_IDENTIFIER_CHARS = ("\x0".."\x20").to_a - WHITESPACE 51 | 52 | NON_IDENTIFIER_CHARS = Regexp.escape "#{SYMBOLS.keys.join}()[]/\\\"##{WHITESPACE.join}#{OTHER_NON_IDENTIFIER_CHARS.join}" 53 | IDENTIFIER_CHARS = /[^#{NON_IDENTIFIER_CHARS}]/ 54 | INITIAL_IDENTIFIER_CHARS = /[^#{NON_IDENTIFIER_CHARS}0-9]/ 55 | 56 | FORBIDDEN = [ 57 | *"\u0000".."\u0008", 58 | *"\u000E".."\u001F", 59 | "\u007F", 60 | *"\u200E".."\u200F", 61 | *"\u202A".."\u202E", 62 | *"\u2066".."\u2069", 63 | "\uFEFF" 64 | ] 65 | 66 | VERSION_PATTERN = /\A\/-[#{WHITESPACE.join}]*kdl-version[#{WHITESPACE.join}]+(\d+)[#{WHITESPACE.join}]*[#{NEWLINES.join}]/ 67 | 68 | def initialize(str, start = 0, filename: nil) 69 | @str = debom(str) 70 | @start = start 71 | @index = start 72 | @buffer = +"" 73 | @filename = filename 74 | @context = nil 75 | @rawstring_hashes = nil 76 | @done = false 77 | @previous_context = nil 78 | @line = 1 79 | @column = 1 80 | @type_context = false 81 | @last_token = nil 82 | end 83 | 84 | def version_directive 85 | if m = @str.match(VERSION_PATTERN) 86 | m[1].to_i 87 | end 88 | end 89 | 90 | def done? 91 | @done 92 | end 93 | 94 | def [](i) 95 | @str[i].tap do |c| 96 | raise_error "Forbidden character: #{c.inspect}" if FORBIDDEN.include?(c) 97 | end 98 | end 99 | 100 | def tokens 101 | a = [] 102 | while !done? 103 | a << next_token 104 | end 105 | a 106 | end 107 | 108 | def next_token 109 | @context = nil 110 | @previous_context = nil 111 | @line_at_start = @line 112 | @column_at_start = @column 113 | loop do 114 | c = self[@index] 115 | case @context 116 | when nil 117 | case c 118 | when '"' 119 | if self[@index + 1] == '"' && self[@index + 2] == '"' 120 | nl = expect_newline(@index + 3) 121 | self.context = :multiline_string 122 | @buffer = +'' 123 | traverse(3 + nl.length) 124 | else 125 | self.context = :string 126 | @buffer = +'' 127 | traverse(1) 128 | end 129 | when '#' 130 | if self[@index + 1] == '"' 131 | if self[@index + 2] == '"' && self[@index + 3] == '"' 132 | nl = expect_newline(@index + 4) 133 | self.context = :multiline_rawstring 134 | @rawstring_hashes = 1 135 | @buffer = +'' 136 | traverse(4 + nl.length) 137 | next 138 | else 139 | self.context = :rawstring 140 | traverse(2) 141 | @rawstring_hashes = 1 142 | @buffer = +'' 143 | next 144 | end 145 | elsif self[@index + 1] == '#' 146 | i = @index + 2 147 | @rawstring_hashes = 2 148 | while self[i] == '#' 149 | @rawstring_hashes += 1 150 | i += 1 151 | end 152 | if self[i] == '"' 153 | if self[i + 1] == '"' && self[i + 2] == '"' 154 | nl = expect_newline(i + 3) 155 | self.context = :multiline_rawstring 156 | traverse(@rawstring_hashes + 3 + nl.length) 157 | @buffer = +'' 158 | next 159 | else 160 | self.context = :rawstring 161 | traverse(@rawstring_hashes + 1) 162 | @buffer = +'' 163 | next 164 | end 165 | end 166 | end 167 | self.context = :keyword 168 | @buffer = +c 169 | traverse(1) 170 | when '-' 171 | n = self[@index + 1] 172 | if n =~ /[0-9]/ 173 | n2 = self[@index + 2] 174 | if n == '0' && n2 =~ /[box]/ 175 | self.context = integer_context(n2) 176 | traverse(3) 177 | else 178 | self.context = :decimal 179 | traverse(1) 180 | end 181 | else 182 | self.context = :ident 183 | traverse(1) 184 | end 185 | @buffer = +c 186 | when /[0-9+]/ 187 | n = self[@index + 1] 188 | if c == '0' && n =~ /[box]/ 189 | traverse(2) 190 | @buffer = +'' 191 | self.context = integer_context(n) 192 | else 193 | self.context = :decimal 194 | @buffer = +c 195 | traverse(1) 196 | end 197 | when '\\' 198 | t = Tokenizer.new(@str, @index + 1, filename:) 199 | la = t.next_token 200 | if la[0] == :NEWLINE || la[0] == :EOF || (la[0] == :WS && (lan = t.next_token[0]) == :NEWLINE || lan == :EOF) 201 | traverse_to(t.index) 202 | @buffer = "#{c}#{la[1].value}" 203 | @buffer << "\n" if lan == :NEWLINE 204 | self.context = :whitespace 205 | else 206 | raise_error "Unexpected '\\' (#{la[0]})" 207 | end 208 | when '=' 209 | self.context = :equals 210 | @buffer = +c 211 | traverse(1) 212 | when *SYMBOLS.keys 213 | return token(SYMBOLS[c], -c).tap { traverse(1) } 214 | when *NEWLINES, "\r" 215 | nl = expect_newline 216 | return token(:NEWLINE, -nl).tap do 217 | traverse(nl.length) 218 | end 219 | when "/" 220 | if self[@index + 1] == '/' 221 | self.context = :single_line_comment 222 | traverse(2) 223 | elsif self[@index + 1] == '*' 224 | self.context = :multi_line_comment 225 | @comment_nesting = 1 226 | traverse(2) 227 | elsif self[@index + 1] == '-' 228 | return token(:SLASHDASH, '/-').tap { traverse(2) } 229 | else 230 | self.context = :ident 231 | @buffer = +c 232 | traverse(1) 233 | end 234 | when *WHITESPACE 235 | self.context = :whitespace 236 | @buffer = +c 237 | traverse(1) 238 | when nil 239 | return [false, token(:EOF, :EOF)[1]] if @done 240 | 241 | @done = true 242 | return token(:EOF, :EOF) 243 | when INITIAL_IDENTIFIER_CHARS 244 | self.context = :ident 245 | @buffer = +c 246 | traverse(1) 247 | when '(' 248 | @type_context = true 249 | return token(:LPAREN, -c).tap { traverse(1) } 250 | when ')' 251 | @type_context = false 252 | return token(:RPAREN, -c).tap { traverse(1) } 253 | else 254 | raise_error "Unexpected character #{c.inspect}" 255 | end 256 | when :ident 257 | case c 258 | when IDENTIFIER_CHARS 259 | traverse(1) 260 | @buffer << c 261 | else 262 | case @buffer 263 | when 'true', 'false', 'null', 'inf', '-inf', 'nan' 264 | raise_error "Identifier cannot be a literal" 265 | when /\A\.\d/ 266 | raise_error "Identifier cannot look like an illegal float" 267 | else 268 | return token(:IDENT, -@buffer) 269 | end 270 | end 271 | when :keyword 272 | case c 273 | when /[a-z\-]/ 274 | traverse(1) 275 | @buffer << c 276 | else 277 | case @buffer 278 | when '#true' then return token(:TRUE, true) 279 | when '#false' then return token(:FALSE, false) 280 | when '#null' then return token(:NULL, nil) 281 | when '#inf' then return token(:FLOAT, Float::INFINITY) 282 | when '#-inf' then return token(:FLOAT, -Float::INFINITY) 283 | when '#nan' then return token(:FLOAT, Float::NAN) 284 | else raise_error "Unknown keyword #{@buffer.inspect}" 285 | end 286 | end 287 | when :string 288 | case c 289 | when '\\' 290 | @buffer << c 291 | c2 = self[@index + 1] 292 | @buffer << c2 293 | if c2.match?(NEWLINES_PATTERN) 294 | i = 2 295 | while self[@index + i]&.match?(NEWLINES_PATTERN) 296 | @buffer << self[@index + i] 297 | i+=1 298 | end 299 | traverse(i) 300 | else 301 | traverse(2) 302 | end 303 | when '"' 304 | return token(:STRING, -unescape(@buffer)).tap { traverse(1) } 305 | when *NEWLINES, "\r" 306 | raise_error "Unexpected NEWLINE in string literal" 307 | when nil 308 | raise_error "Unterminated string literal" 309 | else 310 | @buffer << c 311 | traverse(1) 312 | end 313 | when :multiline_string 314 | case c 315 | when '\\' 316 | @buffer << c 317 | @buffer << self[@index + 1] 318 | traverse(2) 319 | when '"' 320 | if self[@index + 1] == '"' && self[@index + 2] == '"' 321 | return token(:STRING, -unescape_non_ws(dedent(unescape_ws(@buffer)))).tap { traverse(3) } 322 | end 323 | @buffer << c 324 | traverse(1) 325 | when nil 326 | raise_error "Unterminated multi-line string literal" 327 | else 328 | @buffer << c 329 | traverse(1) 330 | end 331 | when :rawstring 332 | raise_error "Unterminated rawstring literal" if c.nil? 333 | 334 | case c 335 | when '"' 336 | h = 0 337 | h += 1 while self[@index + 1 + h] == '#' && h < @rawstring_hashes 338 | if h == @rawstring_hashes 339 | return token(:RAWSTRING, -@buffer).tap { traverse(1 + h) } 340 | end 341 | when *NEWLINES, "\r" 342 | raise_error "Unexpected NEWLINE in rawstring literal" 343 | end 344 | 345 | @buffer << c 346 | traverse(1) 347 | when :multiline_rawstring 348 | raise_error "Unterminated multi-line rawstring literal" if c.nil? 349 | 350 | if c == '"' && self[@index + 1] == '"' && self[@index + 2] == '"' && self[@index + 3] == '#' 351 | h = 1 352 | h += 1 while self[@index + 3 + h] == '#' && h < @rawstring_hashes 353 | if h == @rawstring_hashes 354 | return token(:RAWSTRING, -dedent(@buffer)).tap { traverse(3 + h) } 355 | end 356 | end 357 | 358 | @buffer << c 359 | traverse(1) 360 | when :decimal 361 | case c 362 | when /[0-9.\-+_eE]/ 363 | traverse(1) 364 | @buffer << c 365 | else 366 | return parse_decimal(@buffer) 367 | end 368 | when :hexadecimal 369 | case c 370 | when /[0-9a-fA-F_]/ 371 | traverse(1) 372 | @buffer << c 373 | else 374 | return parse_hexadecimal(@buffer) 375 | end 376 | when :octal 377 | case c 378 | when /[0-7_]/ 379 | traverse(1) 380 | @buffer << c 381 | else 382 | return parse_octal(@buffer) 383 | end 384 | when :binary 385 | case c 386 | when /[01_]/ 387 | traverse(1) 388 | @buffer << c 389 | else 390 | return parse_binary(@buffer) 391 | end 392 | when :single_line_comment 393 | case c 394 | when *NEWLINES, "\r" 395 | self.context = nil 396 | @column_at_start = @column 397 | next 398 | when nil 399 | @done = true 400 | return token(:EOF, :EOF) 401 | else 402 | traverse(1) 403 | end 404 | when :multi_line_comment 405 | if c == '/' && self[@index + 1] == '*' 406 | @comment_nesting += 1 407 | traverse(2) 408 | elsif c == '*' && self[@index + 1] == '/' 409 | @comment_nesting -= 1 410 | traverse(2) 411 | if @comment_nesting == 0 412 | revert_context 413 | end 414 | else 415 | traverse(1) 416 | end 417 | when :whitespace 418 | if WHITESPACE.include?(c) 419 | traverse(1) 420 | @buffer << c 421 | elsif c == '=' 422 | self.context = :equals 423 | @buffer << c 424 | traverse(1) 425 | elsif c == "/" && self[@index + 1] == '*' 426 | self.context = :multi_line_comment 427 | @comment_nesting = 1 428 | traverse(2) 429 | elsif c == "\\" 430 | t = Tokenizer.new(@str, @index + 1, filename:) 431 | la = t.next_token 432 | if la[0] == :NEWLINE || la[0] == :EOF || (la[0] == :WS && (lan = t.next_token[0]) == :NEWLINE || lan == :EOF) 433 | traverse_to(t.index) 434 | @buffer << "#{c}#{la[1].value}" 435 | @buffer << "\n" if lan == :NEWLINE 436 | else 437 | raise_error "Unexpected '\\' (#{la[0]})" 438 | end 439 | else 440 | return token(:WS, -@buffer) 441 | end 442 | when :equals 443 | t = Tokenizer.new(@str, @index, filename:) 444 | la = t.next_token 445 | if la[0] == :WS 446 | @buffer << la[1].value 447 | traverse_to(t.index) 448 | end 449 | return token(:EQUALS, -@buffer) 450 | else 451 | # :nocov: 452 | raise_error "Unknown context `#{@context}'" 453 | # :nocov: 454 | end 455 | end 456 | end 457 | 458 | private 459 | 460 | def token(type, value, **meta) 461 | @last_token = [type, Token.new(type, value, @line_at_start, @column_at_start, meta)] 462 | end 463 | 464 | def traverse(n = 1) 465 | n.times do |i| 466 | case self[@index + i] 467 | when "\r" 468 | @column = 1 469 | when *NEWLINES 470 | @line += 1 471 | @column = 1 472 | else 473 | @column += 1 474 | end 475 | end 476 | @index += n 477 | end 478 | 479 | def traverse_to(i) 480 | traverse(i - @index) 481 | end 482 | 483 | def raise_error(error) 484 | case error 485 | when String then raise ParseError.new(error, @filename, @line, @column) 486 | when Error then raise error 487 | else raise ParseError.new(error.message, @filename, @line, @column) 488 | end 489 | end 490 | 491 | def context=(val) 492 | if @type_context && !allowed_in_type?(val) 493 | raise_error "#{val} context not allowed in type declaration" 494 | elsif @last_token && @last_token[0] == :RPAREN && !allowed_after_type?(val) 495 | raise_error 'Comments are not allowed after a type declaration' 496 | end 497 | @previous_context = @context 498 | @context = val 499 | end 500 | 501 | def allowed_in_type?(val) 502 | %i[ident string rawstring multi_line_comment whitespace].include?(val) 503 | end 504 | 505 | def allowed_after_type?(val) 506 | !%i[single_line_comment].include?(val) 507 | end 508 | 509 | def revert_context 510 | @context = @previous_context 511 | @previous_context = nil 512 | end 513 | 514 | def expect_newline(i = @index) 515 | c = self[i] 516 | case c 517 | when "\r" 518 | n = self[i + 1] 519 | if n == "\n" 520 | "#{c}#{n}" 521 | else 522 | c 523 | end 524 | when *NEWLINES 525 | c 526 | else 527 | raise_error "Expected NEWLINE, found '#{c}'" 528 | end 529 | end 530 | 531 | def integer_context(n) 532 | case n 533 | when 'b' then :binary 534 | when 'o' then :octal 535 | when 'x' then :hexadecimal 536 | end 537 | end 538 | 539 | def parse_decimal(s) 540 | return parse_float(s) if s =~ /[.E]/i 541 | 542 | token(:INTEGER, Integer(munch_underscores(s), 10), format: '%d') 543 | rescue => e 544 | if s[0] =~ INITIAL_IDENTIFIER_CHARS && s[1..-1].each_char.all? { |c| c =~ IDENTIFIER_CHARS } 545 | token(:IDENT, -s) 546 | else 547 | raise_error(e) 548 | end 549 | end 550 | 551 | def parse_float(s) 552 | match, _, fraction, exponent = *s.match(/^([-+]?[\d_]+)(?:\.([\d_]+))?(?:[eE]([-+]?[\d_]+))?$/) 553 | raise_error "Invalid floating point value #{s}" if match.nil? 554 | 555 | s = munch_underscores(s) 556 | 557 | decimals = fraction.nil? ? 0 : fraction.size 558 | value = Float(s) 559 | scientific = value.abs >= 100 || (exponent && exponent.to_i.abs >= 2) 560 | if value.infinite? || (value.zero? && exponent.to_i < 0) 561 | token(:FLOAT, BigDecimal(s)) 562 | else 563 | token(:FLOAT, value, format: scientific ? "%.#{decimals}E" : nil) 564 | end 565 | end 566 | 567 | def parse_hexadecimal(s) 568 | token(:INTEGER, Integer(munch_underscores(s), 16)) 569 | rescue ArgumentError => e 570 | raise_error(e) 571 | end 572 | 573 | def parse_octal(s) 574 | token(:INTEGER, Integer(munch_underscores(s), 8)) 575 | rescue ArgumentError => e 576 | raise_error(e) 577 | end 578 | 579 | def parse_binary(s) 580 | token(:INTEGER, Integer(munch_underscores(s), 2)) 581 | rescue ArgumentError => e 582 | raise_error(e) 583 | end 584 | 585 | def munch_underscores(s) 586 | s.chomp('_').squeeze('_') 587 | end 588 | 589 | def unescape_ws(string) 590 | string.gsub(/\\(\\|\s+)/) do |m| 591 | case m 592 | when '\\\\' then '\\\\' 593 | else '' 594 | end 595 | end 596 | end 597 | 598 | UNESCAPE = /\\(?:[#{WHITESPACE.join}#{NEWLINES.join}\r]+|[^u])/ 599 | UNESCAPE_NON_WS = /\\(?:[^u])/ 600 | 601 | def unescape_non_ws(string) 602 | unescape(string, UNESCAPE_NON_WS) 603 | end 604 | 605 | def unescape(string, rgx = UNESCAPE) 606 | string 607 | .gsub(rgx) { |m| replace_esc(m) } 608 | .gsub(/\\u\{[0-9a-fA-F]+\}/) do |m| 609 | digits = m[3..-2] 610 | raise_error "Invalid code point #{m}" if digits.length > 6 611 | i = Integer(digits, 16) 612 | if i < 0 || i > 0x10FFFF || (0xD800..0xDFFF).include?(i) 613 | raise_error "Invalid code point #{m}" 614 | end 615 | i.chr(Encoding::UTF_8) 616 | end 617 | end 618 | 619 | def replace_esc(m) 620 | case m 621 | when '\n' then "\n" 622 | when '\r' then "\r" 623 | when '\t' then "\t" 624 | when '\\\\' then "\\" 625 | when '\"' then "\"" 626 | when '\b' then "\b" 627 | when '\f' then "\f" 628 | when '\s' then ' ' 629 | when /\\[#{WHITESPACE.join}#{NEWLINES.join}]+/ then '' 630 | else raise_error "Unexpected escape #{m.inspect}" 631 | end 632 | end 633 | 634 | def dedent(string) 635 | split = string.split(NEWLINES_PATTERN) 636 | return "" if split.empty? 637 | 638 | lines = split.partition.with_index { |_, i| i.even? }.first 639 | if split.last.match?(NEWLINES_PATTERN) 640 | indent = "" 641 | else 642 | *lines, indent = lines 643 | end 644 | return "" if lines.empty? 645 | raise_error "Invalid multiline string final line" unless indent.match?(WS_STAR) 646 | valid = /\A#{Regexp.escape(indent)}(.*)/ 647 | 648 | lines.map do |line| 649 | case line 650 | when WS_STAR then "" 651 | when valid then $1 652 | else raise_error "Invalid multiline string indentation" 653 | end 654 | end.join("\n") 655 | end 656 | 657 | def debom(str) 658 | return str unless str.start_with?("\uFEFF") 659 | 660 | str[1..] 661 | end 662 | end 663 | end 664 | --------------------------------------------------------------------------------