├── log └── .gitkeep ├── .ruby-version ├── .ruby-gemset ├── gemfiles ├── .bundle │ └── config ├── avro1_11_rails6_1.gemfile ├── avro1_11_rails7_0.gemfile └── avro1_11_rails7_1.gemfile ├── .github └── CODEOWNERS ├── .rspec ├── lib ├── avromatic │ ├── version.rb │ ├── io.rb │ ├── model │ │ ├── coercion_error.rb │ │ ├── validation_error.rb │ │ ├── unknown_attribute_error.rb │ │ ├── field_helper.rb │ │ ├── value_object.rb │ │ ├── types │ │ │ ├── timestamp_micros_type.rb │ │ │ ├── timestamp_millis_type.rb │ │ │ ├── null_type.rb │ │ │ ├── integer_type.rb │ │ │ ├── boolean_type.rb │ │ │ ├── date_type.rb │ │ │ ├── fixed_type.rb │ │ │ ├── float_type.rb │ │ │ ├── string_type.rb │ │ │ ├── abstract_type.rb │ │ │ ├── enum_type.rb │ │ │ ├── array_type.rb │ │ │ ├── decimal_type.rb │ │ │ ├── record_type.rb │ │ │ ├── abstract_timestamp_type.rb │ │ │ ├── custom_type.rb │ │ │ ├── map_type.rb │ │ │ ├── union_type.rb │ │ │ └── type_factory.rb │ │ ├── nested_models.rb │ │ ├── custom_type_registry.rb │ │ ├── custom_type_configuration.rb │ │ ├── builder.rb │ │ ├── validation.rb │ │ ├── configurable.rb │ │ ├── configuration.rb │ │ ├── messaging_serialization.rb │ │ ├── message_decoder.rb │ │ ├── raw_serialization.rb │ │ └── attributes.rb │ ├── railtie.rb │ ├── rspec.rb │ ├── io │ │ ├── union_datum.rb │ │ ├── datum_writer.rb │ │ └── datum_reader.rb │ ├── model.rb │ ├── messaging.rb │ └── model_registry.rb └── avromatic.rb ├── spec ├── avro │ ├── dsl │ │ └── test │ │ │ ├── encode_key.rb │ │ │ ├── with_union.rb │ │ │ ├── key.rb │ │ │ ├── optional_boolean.rb │ │ │ ├── key_conflict.rb │ │ │ ├── key_overlap.rb │ │ │ ├── recursive.rb │ │ │ ├── with_map.rb │ │ │ ├── key_with_optional.rb │ │ │ ├── with_array.rb │ │ │ ├── value.rb │ │ │ ├── encode_value.rb │ │ │ ├── named_fields.rb │ │ │ ├── named_type.rb │ │ │ ├── repeated_name.rb │ │ │ ├── with_varchar.rb │ │ │ ├── optional_record.rb │ │ │ ├── optional_array.rb │ │ │ ├── defaults.rb │ │ │ ├── nested_record.rb │ │ │ ├── null_in_union.rb │ │ │ ├── wrapper.rb │ │ │ ├── optional_union.rb │ │ │ ├── real_union.rb │ │ │ ├── primitive_types.rb │ │ │ ├── logical_types.rb │ │ │ └── nested_nested_record.rb │ └── schema │ │ └── test │ │ ├── encode_key.avsc │ │ ├── key.avsc │ │ ├── key_overlap.avsc │ │ ├── with_union.avsc │ │ ├── key_conflict.avsc │ │ ├── optional_boolean.avsc │ │ ├── with_map.avsc │ │ ├── with_array.avsc │ │ ├── encode_value.avsc │ │ ├── recursive.avsc │ │ ├── key_with_optional.avsc │ │ ├── named_fields.avsc │ │ ├── value.avsc │ │ ├── repeated_name.avsc │ │ ├── named_type.avsc │ │ ├── with_varchar.avsc │ │ ├── optional_record.avsc │ │ ├── defaults.avsc │ │ ├── optional_array.avsc │ │ ├── nested_record.avsc │ │ ├── logical_types.avsc │ │ ├── wrapper.avsc │ │ ├── real_union.avsc │ │ ├── null_in_union.avsc │ │ ├── optional_union.avsc │ │ ├── logical_types_with_decimal.avsc │ │ ├── primitive_types.avsc │ │ └── nested_nested_record.avsc ├── support │ ├── helpers │ │ └── logical_types_helper.rb │ └── contexts │ │ └── logical_types_serialization.rb ├── avromatic │ ├── model_spec.rb │ ├── io │ │ ├── union_datum_spec.rb │ │ ├── datum_writer_spec.rb │ │ └── datum_reader_spec.rb │ ├── model │ │ ├── builder_nested_models_spec.rb │ │ ├── message_decoder_spec.rb │ │ ├── builder_validation_spec.rb │ │ └── messaging_serialization_spec.rb │ └── model_registry_spec.rb ├── spec_helper.rb └── avromatic_spec.rb ├── Gemfile ├── bin ├── setup └── console ├── .gitignore ├── .overcommit.yml ├── .rubocop.yml ├── Appraisals ├── Rakefile ├── LICENSE.txt ├── avromatic.gemspec ├── .circleci └── config.yml └── CHANGELOG.md /log/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.ruby-version: -------------------------------------------------------------------------------- 1 | 3.0.6 2 | -------------------------------------------------------------------------------- /.ruby-gemset: -------------------------------------------------------------------------------- 1 | avromatic 2 | -------------------------------------------------------------------------------- /gemfiles/.bundle/config: -------------------------------------------------------------------------------- 1 | --- 2 | BUNDLE_RETRY: "1" 3 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @jturkel @salsify/pim-core-backend 2 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --format documentation 2 | --color 3 | --require spec_helper 4 | -------------------------------------------------------------------------------- /lib/avromatic/version.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Avromatic 4 | VERSION = '5.0.0' 5 | end 6 | -------------------------------------------------------------------------------- /spec/avro/dsl/test/encode_key.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | record :encode_key, namespace: :test do 4 | required :id, :int 5 | end 6 | -------------------------------------------------------------------------------- /spec/avro/dsl/test/with_union.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | record :with_union, namespace: :test do 4 | optional :s, :string 5 | end 6 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | source 'https://rubygems.org' 4 | 5 | # Specify your gem's dependencies in avromatic.gemspec 6 | gemspec 7 | -------------------------------------------------------------------------------- /spec/avro/dsl/test/key.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | namespace 'test' 4 | 5 | record :key do 6 | required :a, :int 7 | required :b, :string 8 | end 9 | -------------------------------------------------------------------------------- /spec/avro/dsl/test/optional_boolean.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | namespace :test 4 | 5 | record :optional_boolean do 6 | optional :b, :boolean 7 | end 8 | -------------------------------------------------------------------------------- /lib/avromatic/io.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'avromatic/io/datum_reader' 4 | require 'avromatic/io/datum_writer' 5 | require 'avromatic/io/union_datum' 6 | -------------------------------------------------------------------------------- /spec/avro/dsl/test/key_conflict.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | record :key_conflict, namespace: 'test' do 4 | required :id, :bytes 5 | required :b, :string 6 | end 7 | -------------------------------------------------------------------------------- /spec/avro/dsl/test/key_overlap.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | record :key_overlap, namespace: 'test' do 4 | required :id, :long 5 | required :b, :string 6 | end 7 | -------------------------------------------------------------------------------- /spec/avro/dsl/test/recursive.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | record :recursive, namespace: :test do 4 | required :s, :string 5 | optional :child, :recursive 6 | end 7 | -------------------------------------------------------------------------------- /spec/avro/dsl/test/with_map.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | record :with_map, namespace: :test do 4 | required :pairs, :map, values: :int, default: { a: 1 } 5 | end 6 | -------------------------------------------------------------------------------- /bin/setup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | IFS=$'\n\t' 4 | set -vx 5 | 6 | bundle install 7 | appraisal install 8 | # Do any other automated setup that you need to do here 9 | -------------------------------------------------------------------------------- /lib/avromatic/model/coercion_error.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Avromatic 4 | module Model 5 | class CoercionError < StandardError 6 | end 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /spec/avro/dsl/test/key_with_optional.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | record :key_with_optional, namespace: :test do 4 | required :id, :long 5 | optional :name, :string 6 | end 7 | -------------------------------------------------------------------------------- /spec/avro/dsl/test/with_array.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | record :with_array, namespace: :test do 4 | required :names, :array, items: :string, default: ['first'] 5 | end 6 | -------------------------------------------------------------------------------- /lib/avromatic/model/validation_error.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Avromatic 4 | module Model 5 | class ValidationError < StandardError 6 | end 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.bundle/ 2 | /.yardoc 3 | /Gemfile.lock 4 | /_yardoc/ 5 | /coverage/ 6 | /doc/ 7 | /pkg/ 8 | /spec/reports/ 9 | /tmp/ 10 | /gemfiles/*.lock 11 | /log/* 12 | 13 | *.iml 14 | .idea 15 | -------------------------------------------------------------------------------- /spec/avro/dsl/test/value.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | record :value, namespace: :test do 4 | required :action, :enum, symbols: [:CREATE, :UPDATE, :DESTROY] 5 | required :id, :long 6 | end 7 | -------------------------------------------------------------------------------- /spec/avro/dsl/test/encode_value.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | record :encode_value, namespace: :test do 4 | required :str1, :string, default: 'X' 5 | required :str2, :string, default: 'Y' 6 | end 7 | -------------------------------------------------------------------------------- /spec/avro/dsl/test/named_fields.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | namespace 'test' 4 | 5 | record :named_fields do 6 | required :sub, :record do 7 | required :s, :string 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /spec/avro/schema/test/encode_key.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "encode_key", 4 | "namespace": "test", 5 | "fields": [ 6 | { 7 | "name": "id", 8 | "type": "int" 9 | } 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /gemfiles/avro1_11_rails6_1.gemfile: -------------------------------------------------------------------------------- 1 | # This file was generated by Appraisal 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "avro", "~> 1.11.0" 6 | gem "activesupport", "~> 6.1.0" 7 | gem "activemodel", "~> 6.1.0" 8 | 9 | gemspec path: "../" 10 | -------------------------------------------------------------------------------- /gemfiles/avro1_11_rails7_0.gemfile: -------------------------------------------------------------------------------- 1 | # This file was generated by Appraisal 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "avro", "~> 1.11.0" 6 | gem "activesupport", "~> 7.0.0" 7 | gem "activemodel", "~> 7.0.0" 8 | 9 | gemspec path: "../" 10 | -------------------------------------------------------------------------------- /gemfiles/avro1_11_rails7_1.gemfile: -------------------------------------------------------------------------------- 1 | # This file was generated by Appraisal 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "avro", "~> 1.11.0" 6 | gem "activesupport", "~> 7.0.0" 7 | gem "activemodel", "~> 7.0.0" 8 | 9 | gemspec path: "../" 10 | -------------------------------------------------------------------------------- /spec/avro/dsl/test/named_type.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | namespace :test 4 | 5 | fixed :six, size: 6 6 | 7 | record :named_type do 8 | required :six_str, :six, default: ''.ljust(6) 9 | optional :optional_six, :six 10 | end 11 | -------------------------------------------------------------------------------- /spec/avro/dsl/test/repeated_name.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | namespace :test 4 | 5 | record :sub do 6 | required :i, :int 7 | end 8 | 9 | record :repeated_name do 10 | required :old, :sub 11 | required :new, :sub 12 | end 13 | -------------------------------------------------------------------------------- /.overcommit.yml: -------------------------------------------------------------------------------- 1 | PreCommit: 2 | RuboCop: 3 | enabled: true 4 | required: false 5 | on_warn: fail 6 | 7 | HardTabs: 8 | enabled: true 9 | required: false 10 | 11 | CommitMsg: 12 | TrailingPeriod: 13 | enabled: false 14 | 15 | -------------------------------------------------------------------------------- /spec/avro/dsl/test/with_varchar.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | namespace :test 4 | 5 | record :varchar do 6 | required :length, :int 7 | required :data, :bytes 8 | end 9 | 10 | record :with_varchar do 11 | required :str, :varchar 12 | end 13 | -------------------------------------------------------------------------------- /spec/avro/dsl/test/optional_record.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | namespace :test 4 | 5 | record :message do 6 | required :body, :string 7 | end 8 | 9 | record :optional_record do 10 | required :id, :int 11 | optional :message, :message 12 | end 13 | -------------------------------------------------------------------------------- /spec/avro/dsl/test/optional_array.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | namespace :test 4 | 5 | record :message do 6 | required :body, :string 7 | end 8 | 9 | record :optional_array do 10 | required :id, :int 11 | optional :messages, array(:message) 12 | end 13 | -------------------------------------------------------------------------------- /spec/avro/schema/test/key.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "key", 4 | "namespace": "test", 5 | "fields": [ 6 | { 7 | "name": "a", 8 | "type": "int" 9 | }, 10 | { 11 | "name": "b", 12 | "type": "string" 13 | } 14 | ] 15 | } 16 | -------------------------------------------------------------------------------- /spec/avro/schema/test/key_overlap.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "key_overlap", 4 | "namespace": "test", 5 | "fields": [ 6 | { 7 | "name": "id", 8 | "type": "long" 9 | }, 10 | { 11 | "name": "b", 12 | "type": "string" 13 | } 14 | ] 15 | } 16 | -------------------------------------------------------------------------------- /spec/avro/schema/test/with_union.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "with_union", 4 | "namespace": "test", 5 | "fields": [ 6 | { 7 | "name": "s", 8 | "type": [ 9 | "null", 10 | "string" 11 | ], 12 | "default": null 13 | } 14 | ] 15 | } 16 | -------------------------------------------------------------------------------- /spec/avro/dsl/test/defaults.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | namespace 'test' 4 | 5 | record :defaults do 6 | required :defaulted_enum, :enum, symbols: [:A, :B], default: :A 7 | required :defaulted_string, :string, default: 'fnord' 8 | required :defaulted_int, :int, default: 42 9 | end 10 | -------------------------------------------------------------------------------- /spec/avro/schema/test/key_conflict.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "key_conflict", 4 | "namespace": "test", 5 | "fields": [ 6 | { 7 | "name": "id", 8 | "type": "bytes" 9 | }, 10 | { 11 | "name": "b", 12 | "type": "string" 13 | } 14 | ] 15 | } 16 | -------------------------------------------------------------------------------- /spec/avro/dsl/test/nested_record.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | namespace :test 4 | 5 | record :nested_record do 6 | required :str, :string, default: 'A' 7 | 8 | required :sub, :record do 9 | required :str, :string, default: 'B' 10 | required :i, :int, default: 0 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /spec/avro/schema/test/optional_boolean.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "optional_boolean", 4 | "namespace": "test", 5 | "fields": [ 6 | { 7 | "name": "b", 8 | "type": [ 9 | "null", 10 | "boolean" 11 | ], 12 | "default": null 13 | } 14 | ] 15 | } 16 | -------------------------------------------------------------------------------- /spec/avro/dsl/test/null_in_union.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | namespace :test 4 | 5 | record :i_rec do 6 | required :i, :int 7 | end 8 | 9 | record :s_rec do 10 | required :s, :string 11 | end 12 | 13 | record :null_in_union do 14 | required :values, array(union(:i_rec, null, :s_rec)) 15 | end 16 | -------------------------------------------------------------------------------- /spec/avro/schema/test/with_map.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "with_map", 4 | "namespace": "test", 5 | "fields": [ 6 | { 7 | "name": "pairs", 8 | "type": { 9 | "type": "map", 10 | "values": "int" 11 | }, 12 | "default": { 13 | "a": 1 14 | } 15 | } 16 | ] 17 | } 18 | -------------------------------------------------------------------------------- /spec/avro/dsl/test/wrapper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | namespace 'test' 4 | 5 | record :wrapped1 do 6 | required :i, :int 7 | end 8 | 9 | record :wrapped2 do 10 | required :i, :int 11 | end 12 | 13 | record :wrapper do 14 | required :sub1, :wrapped1 15 | required :sub2, :wrapped1 16 | required :sub3, :wrapped2 17 | end 18 | -------------------------------------------------------------------------------- /spec/avro/schema/test/with_array.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "with_array", 4 | "namespace": "test", 5 | "fields": [ 6 | { 7 | "name": "names", 8 | "type": { 9 | "type": "array", 10 | "items": "string" 11 | }, 12 | "default": [ 13 | "first" 14 | ] 15 | } 16 | ] 17 | } 18 | -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | inherit_gem: 2 | salsify_rubocop: conf/rubocop.yml 3 | 4 | AllCops: 5 | TargetRubyVersion: 2.7 6 | Exclude: 7 | - 'vendor/**/*' 8 | - 'gemfiles/**/*' 9 | 10 | Style/MultilineBlockChain: 11 | Enabled: false 12 | 13 | Style/NumericPredicate: 14 | Enabled: false 15 | 16 | Style/FrozenStringLiteralComment: 17 | Enabled: true 18 | -------------------------------------------------------------------------------- /spec/avro/dsl/test/optional_union.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | namespace :test 4 | 5 | record :foo do 6 | required :foo_message, :string 7 | end 8 | 9 | record :bar do 10 | required :bar_message, :string 11 | end 12 | 13 | record :optional_union do 14 | required :header, :string 15 | optional :message, :union, types: [:foo, :bar] 16 | end 17 | -------------------------------------------------------------------------------- /spec/avro/schema/test/encode_value.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "encode_value", 4 | "namespace": "test", 5 | "fields": [ 6 | { 7 | "name": "str1", 8 | "type": "string", 9 | "default": "X" 10 | }, 11 | { 12 | "name": "str2", 13 | "type": "string", 14 | "default": "Y" 15 | } 16 | ] 17 | } 18 | -------------------------------------------------------------------------------- /spec/avro/dsl/test/real_union.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | namespace :test 4 | 5 | record :foo do 6 | required :foo_message, :string 7 | end 8 | 9 | record :bar do 10 | required :bar_message, :string 11 | end 12 | 13 | record :real_union do 14 | required :header, :string 15 | required :message, :union, types: [:foo, :bar, :boolean] 16 | end 17 | -------------------------------------------------------------------------------- /spec/avro/schema/test/recursive.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "recursive", 4 | "namespace": "test", 5 | "fields": [ 6 | { 7 | "name": "s", 8 | "type": "string" 9 | }, 10 | { 11 | "name": "child", 12 | "type": [ 13 | "null", 14 | "test.recursive" 15 | ], 16 | "default": null 17 | } 18 | ] 19 | } 20 | -------------------------------------------------------------------------------- /lib/avromatic/railtie.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Avromatic 4 | class Railtie < Rails::Railtie 5 | initializer 'avromatic.configure' do 6 | Avromatic.configure do |config| 7 | config.logger = Rails.logger 8 | end 9 | 10 | Rails.configuration.to_prepare do 11 | Avromatic.prepare! 12 | end 13 | end 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /spec/avro/schema/test/key_with_optional.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "key_with_optional", 4 | "namespace": "test", 5 | "fields": [ 6 | { 7 | "name": "id", 8 | "type": "long" 9 | }, 10 | { 11 | "name": "name", 12 | "type": [ 13 | "null", 14 | "string" 15 | ], 16 | "default": null 17 | } 18 | ] 19 | } 20 | -------------------------------------------------------------------------------- /spec/support/helpers/logical_types_helper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module LogicalTypesHelper 4 | 5 | def with_logical_types 6 | yield if logical_types? 7 | end 8 | 9 | def without_logical_types 10 | yield unless logical_types? 11 | end 12 | 13 | private 14 | 15 | def logical_types? 16 | Avro::Schema.instance_methods.include?(:logical_type) 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /spec/avro/dsl/test/primitive_types.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | namespace :test 4 | 5 | record :primitive_types do 6 | required :s, :string 7 | required :b, :bytes 8 | required :tf, :boolean 9 | required :i, :int 10 | required :l, :long 11 | required :f, :float 12 | required :d, :double 13 | required :n, :null 14 | required :fx, :fixed, size: 7 15 | required :e, :enum, symbols: [:A, :B] 16 | end 17 | -------------------------------------------------------------------------------- /bin/console: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | require 'bundler/setup' 5 | require 'avromatic' 6 | 7 | # You can add fixtures and/or initialization code here to make experimenting 8 | # with your gem easier. You can also use a different console, if you like. 9 | 10 | # (If you use this, don't forget to add pry to your Gemfile!) 11 | # require "pry" 12 | # Pry.start 13 | 14 | require 'irb' 15 | IRB.start 16 | -------------------------------------------------------------------------------- /spec/avro/dsl/test/logical_types.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | record :logical_types, namespace: :test do 4 | required :date, :int, logical_type: 'date' 5 | required :ts_msec, :long, logical_type: 'timestamp-millis' 6 | required :ts_usec, :long, logical_type: 'timestamp-micros' 7 | required :decimal, :bytes, logical_type: 'decimal', precision: 4, scale: 2 8 | required :unknown, :int, logical_type: 'foobar' 9 | end 10 | -------------------------------------------------------------------------------- /spec/avro/schema/test/named_fields.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "named_fields", 4 | "namespace": "test", 5 | "fields": [ 6 | { 7 | "name": "sub", 8 | "type": { 9 | "type": "record", 10 | "name": "__named_fields_sub_record", 11 | "namespace": "test", 12 | "fields": [ 13 | { 14 | "name": "s", 15 | "type": "string" 16 | } 17 | ] 18 | } 19 | } 20 | ] 21 | } 22 | -------------------------------------------------------------------------------- /lib/avromatic/model/unknown_attribute_error.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Avromatic 4 | module Model 5 | class UnknownAttributeError < StandardError 6 | attr_reader :unknown_attributes, :allowed_attributes 7 | 8 | def initialize(message, unknown_attributes:, allowed_attributes:) 9 | super(message) 10 | @unknown_attributes = unknown_attributes 11 | @allowed_attributes = allowed_attributes 12 | end 13 | end 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /Appraisals: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | appraise 'avro1_11-rails6_1' do 4 | gem 'avro', '~> 1.11.0' 5 | gem 'activesupport', '~> 6.1.0' 6 | gem 'activemodel', '~> 6.1.0' 7 | end 8 | 9 | appraise 'avro1_11-rails7_0' do 10 | gem 'avro', '~> 1.11.0' 11 | gem 'activesupport', '~> 7.0.0' 12 | gem 'activemodel', '~> 7.0.0' 13 | end 14 | 15 | appraise 'avro1_11-rails7_1' do 16 | gem 'avro', '~> 1.11.0' 17 | gem 'activesupport', '~> 7.1.1' 18 | gem 'activemodel', '~> 7.1.1' 19 | end 20 | -------------------------------------------------------------------------------- /spec/avro/schema/test/value.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "value", 4 | "namespace": "test", 5 | "fields": [ 6 | { 7 | "name": "action", 8 | "type": { 9 | "name": "__value_action_enum", 10 | "type": "enum", 11 | "namespace": "test", 12 | "symbols": [ 13 | "CREATE", 14 | "UPDATE", 15 | "DESTROY" 16 | ] 17 | } 18 | }, 19 | { 20 | "name": "id", 21 | "type": "long" 22 | } 23 | ] 24 | } 25 | -------------------------------------------------------------------------------- /spec/avro/schema/test/repeated_name.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "repeated_name", 4 | "namespace": "test", 5 | "fields": [ 6 | { 7 | "name": "old", 8 | "type": { 9 | "type": "record", 10 | "name": "sub", 11 | "namespace": "test", 12 | "fields": [ 13 | { 14 | "name": "i", 15 | "type": "int" 16 | } 17 | ] 18 | } 19 | }, 20 | { 21 | "name": "new", 22 | "type": "test.sub" 23 | } 24 | ] 25 | } 26 | -------------------------------------------------------------------------------- /spec/avro/schema/test/named_type.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "named_type", 4 | "namespace": "test", 5 | "fields": [ 6 | { 7 | "name": "six_str", 8 | "type": { 9 | "name": "six", 10 | "type": "fixed", 11 | "namespace": "test", 12 | "size": 6 13 | }, 14 | "default": " " 15 | }, 16 | { 17 | "name": "optional_six", 18 | "type": [ 19 | "null", 20 | "test.six" 21 | ], 22 | "default": null 23 | } 24 | ] 25 | } 26 | -------------------------------------------------------------------------------- /spec/avro/schema/test/with_varchar.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "with_varchar", 4 | "namespace": "test", 5 | "fields": [ 6 | { 7 | "name": "str", 8 | "type": { 9 | "type": "record", 10 | "name": "varchar", 11 | "namespace": "test", 12 | "fields": [ 13 | { 14 | "name": "length", 15 | "type": "int" 16 | }, 17 | { 18 | "name": "data", 19 | "type": "bytes" 20 | } 21 | ] 22 | } 23 | } 24 | ] 25 | } 26 | -------------------------------------------------------------------------------- /lib/avromatic/rspec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'uri' 4 | require 'webmock/rspec' 5 | require 'avro_schema_registry/test/fake_server' 6 | 7 | RSpec.configure do |config| 8 | config.before(:each) do 9 | # Strip the username/password from the URL so WebMock can match the URL 10 | registry_uri = URI(Avromatic.registry_url) 11 | registry_uri.userinfo = '' 12 | 13 | WebMock.stub_request(:any, /^#{registry_uri}/).to_rack(AvroSchemaRegistry::FakeServer) 14 | AvroSchemaRegistry::FakeServer.clear 15 | Avromatic.build_schema_registry! 16 | Avromatic.build_messaging! 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /lib/avromatic/io/union_datum.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Avromatic 4 | module IO 5 | class UnionDatum 6 | attr_reader :member_index, :datum 7 | 8 | def initialize(member_index, datum) 9 | @member_index = member_index 10 | @datum = datum 11 | end 12 | 13 | def ==(other) 14 | other.is_a?(Avromatic::IO::UnionDatum) && 15 | member_index == other.member_index && 16 | datum == other.datum 17 | end 18 | alias_method :eql?, :== 19 | 20 | def hash 21 | 31 * datum.hash + member_index 22 | end 23 | end 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /spec/avro/schema/test/optional_record.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "optional_record", 4 | "namespace": "test", 5 | "fields": [ 6 | { 7 | "name": "id", 8 | "type": "int" 9 | }, 10 | { 11 | "name": "message", 12 | "type": [ 13 | "null", 14 | { 15 | "type": "record", 16 | "name": "message", 17 | "namespace": "test", 18 | "fields": [ 19 | { 20 | "name": "body", 21 | "type": "string" 22 | } 23 | ] 24 | } 25 | ], 26 | "default": null 27 | } 28 | ] 29 | } 30 | -------------------------------------------------------------------------------- /spec/avro/schema/test/defaults.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "defaults", 4 | "namespace": "test", 5 | "fields": [ 6 | { 7 | "name": "defaulted_enum", 8 | "type": { 9 | "name": "__defaults_defaulted_enum_enum", 10 | "type": "enum", 11 | "namespace": "test", 12 | "symbols": [ 13 | "A", 14 | "B" 15 | ] 16 | }, 17 | "default": "A" 18 | }, 19 | { 20 | "name": "defaulted_string", 21 | "type": "string", 22 | "default": "fnord" 23 | }, 24 | { 25 | "name": "defaulted_int", 26 | "type": "int", 27 | "default": 42 28 | } 29 | ] 30 | } 31 | -------------------------------------------------------------------------------- /spec/avro/schema/test/optional_array.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "optional_array", 4 | "namespace": "test", 5 | "fields": [ 6 | { 7 | "name": "id", 8 | "type": "int" 9 | }, 10 | { 11 | "name": "messages", 12 | "type": [ 13 | "null", 14 | { 15 | "type": "array", 16 | "items": { 17 | "type": "record", 18 | "name": "message", 19 | "namespace": "test", 20 | "fields": [ 21 | { 22 | "name": "body", 23 | "type": "string" 24 | } 25 | ] 26 | } 27 | } 28 | ], 29 | "default": null 30 | } 31 | ] 32 | } 33 | -------------------------------------------------------------------------------- /spec/avro/schema/test/nested_record.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "nested_record", 4 | "namespace": "test", 5 | "fields": [ 6 | { 7 | "name": "str", 8 | "type": "string", 9 | "default": "A" 10 | }, 11 | { 12 | "name": "sub", 13 | "type": { 14 | "type": "record", 15 | "name": "__nested_record_sub_record", 16 | "namespace": "test", 17 | "fields": [ 18 | { 19 | "name": "str", 20 | "type": "string", 21 | "default": "B" 22 | }, 23 | { 24 | "name": "i", 25 | "type": "int", 26 | "default": 0 27 | } 28 | ] 29 | } 30 | } 31 | ] 32 | } 33 | -------------------------------------------------------------------------------- /spec/avro/schema/test/logical_types.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "logical_types", 4 | "namespace": "test", 5 | "fields": [ 6 | { 7 | "name": "date", 8 | "type": { 9 | "type": "int", 10 | "logicalType": "date" 11 | } 12 | }, 13 | { 14 | "name": "ts_msec", 15 | "type": { 16 | "type": "long", 17 | "logicalType": "timestamp-millis" 18 | } 19 | }, 20 | { 21 | "name": "ts_usec", 22 | "type": { 23 | "type": "long", 24 | "logicalType": "timestamp-micros" 25 | } 26 | }, 27 | { 28 | "name": "unknown", 29 | "type": { 30 | "type": "int", 31 | "logicalType": "foobar" 32 | } 33 | } 34 | ] 35 | } 36 | -------------------------------------------------------------------------------- /lib/avromatic/model/field_helper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Avromatic 4 | module Model 5 | module FieldHelper 6 | extend self 7 | 8 | # An optional field is represented as a union where the first member 9 | # is null. 10 | def optional?(field) 11 | field.type.type_sym == :union && 12 | field.type.schemas.first.type_sym == :null 13 | end 14 | 15 | def required?(field) 16 | !optional?(field) 17 | end 18 | 19 | def nullable?(field) 20 | optional?(field) || field.type.type_sym == :null 21 | end 22 | 23 | def boolean?(field) 24 | field.type.type_sym == :boolean || 25 | (optional?(field) && field.type.schemas.last.type_sym == :boolean) 26 | end 27 | end 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /lib/avromatic/model/value_object.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Avromatic 4 | module Model 5 | # This module is used to override the comparisons defined by 6 | # Virtus::Equalizer which is pulled in by Virtus::ValueObject. 7 | module ValueObject 8 | def eql?(other) 9 | other.instance_of?(self.class) && attributes == other.attributes 10 | end 11 | alias_method :==, :eql? 12 | 13 | def hash 14 | attributes.hash 15 | end 16 | 17 | def inspect 18 | "#<#{self.class.name} #{attributes.map { |k, v| "#{k}: #{v.inspect}" }.join(', ')}>" 19 | end 20 | 21 | def to_s 22 | format('#<%s:0x00%x>', class_name: self.class.name, identifier: object_id.abs * 2) 23 | end 24 | end 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /spec/avro/schema/test/wrapper.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "wrapper", 4 | "namespace": "test", 5 | "fields": [ 6 | { 7 | "name": "sub1", 8 | "type": { 9 | "type": "record", 10 | "name": "wrapped1", 11 | "namespace": "test", 12 | "fields": [ 13 | { 14 | "name": "i", 15 | "type": "int" 16 | } 17 | ] 18 | } 19 | }, 20 | { 21 | "name": "sub2", 22 | "type": "test.wrapped1" 23 | }, 24 | { 25 | "name": "sub3", 26 | "type": { 27 | "type": "record", 28 | "name": "wrapped2", 29 | "namespace": "test", 30 | "fields": [ 31 | { 32 | "name": "i", 33 | "type": "int" 34 | } 35 | ] 36 | } 37 | } 38 | ] 39 | } 40 | -------------------------------------------------------------------------------- /spec/avro/schema/test/real_union.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "real_union", 4 | "namespace": "test", 5 | "fields": [ 6 | { 7 | "name": "header", 8 | "type": "string" 9 | }, 10 | { 11 | "name": "message", 12 | "type": [ 13 | { 14 | "type": "record", 15 | "name": "foo", 16 | "namespace": "test", 17 | "fields": [ 18 | { 19 | "name": "foo_message", 20 | "type": "string" 21 | } 22 | ] 23 | }, 24 | { 25 | "type": "record", 26 | "name": "bar", 27 | "namespace": "test", 28 | "fields": [ 29 | { 30 | "name": "bar_message", 31 | "type": "string" 32 | } 33 | ] 34 | }, 35 | "boolean" 36 | ] 37 | } 38 | ] 39 | } 40 | -------------------------------------------------------------------------------- /spec/avromatic/model_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | describe Avromatic::Model do 6 | let(:args) { { schema_name: 'test_schema' } } 7 | 8 | describe ".build" do 9 | it "delegates to Avromatic::Model::Builder" do 10 | builder = instance_double(Avromatic::Model::Builder, mod: Module.new) 11 | allow(Avromatic::Model::Builder).to receive(:new).and_return(builder) 12 | described_class.build(**args) 13 | expect(Avromatic::Model::Builder).to have_received(:new).with(args) 14 | expect(builder).to have_received(:mod) 15 | end 16 | end 17 | 18 | describe ".model" do 19 | it "delegates to Avromatic::Model::Builder" do 20 | allow(Avromatic::Model::Builder).to receive(:model) 21 | described_class.model(**args) 22 | expect(Avromatic::Model::Builder).to have_received(:model).with(args) 23 | end 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /spec/avro/schema/test/null_in_union.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "null_in_union", 4 | "namespace": "test", 5 | "fields": [ 6 | { 7 | "name": "values", 8 | "type": { 9 | "type": "array", 10 | "items": [ 11 | { 12 | "type": "record", 13 | "name": "i_rec", 14 | "namespace": "test", 15 | "fields": [ 16 | { 17 | "name": "i", 18 | "type": "int" 19 | } 20 | ] 21 | }, 22 | "null", 23 | { 24 | "type": "record", 25 | "name": "s_rec", 26 | "namespace": "test", 27 | "fields": [ 28 | { 29 | "name": "s", 30 | "type": "string" 31 | } 32 | ] 33 | } 34 | ] 35 | } 36 | } 37 | ] 38 | } 39 | -------------------------------------------------------------------------------- /spec/avro/schema/test/optional_union.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "optional_union", 4 | "namespace": "test", 5 | "fields": [ 6 | { 7 | "name": "header", 8 | "type": "string" 9 | }, 10 | { 11 | "name": "message", 12 | "type": [ 13 | "null", 14 | { 15 | "type": "record", 16 | "name": "foo", 17 | "namespace": "test", 18 | "fields": [ 19 | { 20 | "name": "foo_message", 21 | "type": "string" 22 | } 23 | ] 24 | }, 25 | { 26 | "type": "record", 27 | "name": "bar", 28 | "namespace": "test", 29 | "fields": [ 30 | { 31 | "name": "bar_message", 32 | "type": "string" 33 | } 34 | ] 35 | } 36 | ], 37 | "default": null 38 | } 39 | ] 40 | } 41 | -------------------------------------------------------------------------------- /lib/avromatic/model/types/timestamp_micros_type.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'avromatic/model/types/abstract_timestamp_type' 4 | 5 | module Avromatic 6 | module Model 7 | module Types 8 | 9 | # This subclass is used to truncate timestamp values to microseconds. 10 | class TimestampMicrosType < Avromatic::Model::Types::AbstractTimestampType 11 | 12 | def name 13 | 'timestamp-micros' 14 | end 15 | 16 | def referenced_model_classes 17 | EMPTY_ARRAY 18 | end 19 | 20 | private 21 | 22 | def truncated?(value) 23 | value.nsec % 1_000 == 0 24 | end 25 | 26 | def coerce_time(input) 27 | # value is coerced to a local Time 28 | # The Avro representation of a timestamp is Epoch seconds, independent 29 | # of time zone. 30 | ::Time.at(input.to_i, input.usec) 31 | end 32 | end 33 | end 34 | end 35 | end 36 | -------------------------------------------------------------------------------- /spec/avro/schema/test/logical_types_with_decimal.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "logical_types_with_decimal", 4 | "namespace": "test", 5 | "fields": [ 6 | { 7 | "name": "date", 8 | "type": { 9 | "type": "int", 10 | "logicalType": "date" 11 | } 12 | }, 13 | { 14 | "name": "ts_msec", 15 | "type": { 16 | "type": "long", 17 | "logicalType": "timestamp-millis" 18 | } 19 | }, 20 | { 21 | "name": "ts_usec", 22 | "type": { 23 | "type": "long", 24 | "logicalType": "timestamp-micros" 25 | } 26 | }, 27 | { 28 | "name": "decimal", 29 | "type": { 30 | "type": "bytes", 31 | "logicalType": "decimal", 32 | "precision": 4, 33 | "scale": 2 34 | } 35 | }, 36 | { 37 | "name": "unknown", 38 | "type": { 39 | "type": "int", 40 | "logicalType": "foobar" 41 | } 42 | } 43 | ] 44 | } 45 | -------------------------------------------------------------------------------- /lib/avromatic/model/types/timestamp_millis_type.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'avromatic/model/types/abstract_timestamp_type' 4 | 5 | module Avromatic 6 | module Model 7 | module Types 8 | 9 | # This subclass is used to truncate timestamp values to milliseconds. 10 | class TimestampMillisType < Avromatic::Model::Types::AbstractTimestampType 11 | 12 | def name 13 | 'timestamp-millis' 14 | end 15 | 16 | def referenced_model_classes 17 | EMPTY_ARRAY 18 | end 19 | 20 | private 21 | 22 | def truncated?(value) 23 | value.nsec % 1_000_000 == 0 24 | end 25 | 26 | def coerce_time(input) 27 | # value is coerced to a local Time 28 | # The Avro representation of a timestamp is Epoch seconds, independent 29 | # of time zone. 30 | ::Time.at(input.to_i, input.usec / 1000 * 1000) 31 | end 32 | end 33 | end 34 | end 35 | end 36 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'bundler/gem_tasks' 4 | require 'rspec/core/rake_task' 5 | require 'appraisal/task' 6 | require 'avro/builder' 7 | 8 | RSpec::Core::RakeTask.new(:default_spec) 9 | 10 | Appraisal::Task.new 11 | 12 | if !ENV['APPRAISAL_INITIALIZED'] 13 | task default: :appraisal 14 | task spec: :appraisal 15 | else 16 | task default: :default_spec 17 | end 18 | 19 | namespace :avro do 20 | desc 'Generate Avro schema files used by specs' 21 | task :generate_spec do 22 | root = 'spec/avro/dsl' 23 | Avro::Builder.add_load_path(root) 24 | Dir["#{root}/**/*.rb"].each do |dsl_file| 25 | puts "Generating Avro schema from #{dsl_file}" 26 | output_file = dsl_file.sub('/dsl/', '/schema/').sub(/\.rb$/, '.avsc') 27 | schema = Avro::Builder.build(File.read(dsl_file)) 28 | FileUtils.mkdir_p(File.dirname(output_file)) 29 | File.write(output_file, schema.end_with?("\n") ? schema : schema << "\n") 30 | end 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /lib/avromatic/model/types/null_type.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'avromatic/model/types/abstract_type' 4 | 5 | module Avromatic 6 | module Model 7 | module Types 8 | class NullType < AbstractType 9 | VALUE_CLASSES = [::NilClass].freeze 10 | 11 | def value_classes 12 | VALUE_CLASSES 13 | end 14 | 15 | def name 16 | 'null' 17 | end 18 | 19 | def coerce(input) 20 | if input.nil? 21 | nil 22 | else 23 | raise ArgumentError.new("Could not coerce '#{input.inspect}' to #{name}") 24 | end 25 | end 26 | 27 | def coercible?(input) 28 | input.nil? 29 | end 30 | 31 | alias_method :coerced?, :coercible? 32 | 33 | def serialize(_value, _strict) 34 | nil 35 | end 36 | 37 | def referenced_model_classes 38 | EMPTY_ARRAY 39 | end 40 | end 41 | end 42 | end 43 | end 44 | -------------------------------------------------------------------------------- /lib/avromatic/model/types/integer_type.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'avromatic/model/types/abstract_type' 4 | 5 | module Avromatic 6 | module Model 7 | module Types 8 | class IntegerType < AbstractType 9 | VALUE_CLASSES = [::Integer].freeze 10 | 11 | def value_classes 12 | VALUE_CLASSES 13 | end 14 | 15 | def name 16 | 'integer' 17 | end 18 | 19 | def coerce(input) 20 | if input.nil? || input.is_a?(::Integer) 21 | input 22 | else 23 | raise ArgumentError.new("Could not coerce '#{input.inspect}' to #{name}") 24 | end 25 | end 26 | 27 | def coercible?(input) 28 | input.nil? || input.is_a?(::Integer) 29 | end 30 | 31 | alias_method :coerced?, :coercible? 32 | 33 | def serialize(value, _strict) 34 | value 35 | end 36 | 37 | def referenced_model_classes 38 | EMPTY_ARRAY 39 | end 40 | end 41 | end 42 | end 43 | end 44 | -------------------------------------------------------------------------------- /lib/avromatic/model/types/boolean_type.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'avromatic/model/types/abstract_type' 4 | 5 | module Avromatic 6 | module Model 7 | module Types 8 | class BooleanType < AbstractType 9 | VALUE_CLASSES = [::TrueClass, ::FalseClass].freeze 10 | 11 | def value_classes 12 | VALUE_CLASSES 13 | end 14 | 15 | def name 16 | 'boolean' 17 | end 18 | 19 | def coerce(input) 20 | if coercible?(input) 21 | input 22 | else 23 | raise ArgumentError.new("Could not coerce '#{input.inspect}' to #{name}") 24 | end 25 | end 26 | 27 | def coercible?(input) 28 | input.nil? || input.is_a?(::TrueClass) || input.is_a?(::FalseClass) 29 | end 30 | 31 | alias_method :coerced?, :coercible? 32 | 33 | def serialize(value, _strict) 34 | value 35 | end 36 | 37 | def referenced_model_classes 38 | EMPTY_ARRAY 39 | end 40 | end 41 | end 42 | end 43 | end 44 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | $LOAD_PATH.unshift File.expand_path('../lib', __dir__) 4 | require 'simplecov' 5 | 6 | SimpleCov.start do 7 | add_filter 'spec' 8 | minimum_coverage 95 9 | end 10 | 11 | require 'avro/builder' 12 | require 'avromatic' 13 | require 'active_support/core_ext/hash/keys' 14 | 15 | Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].sort.each { |f| require f } 16 | 17 | RSpec.configure do |config| 18 | config.extend LogicalTypesHelper 19 | 20 | config.before do 21 | Avromatic.logger = Logger.new('log/test.log') 22 | Avromatic.registry_url = 'http://username:password@registry.example.com' 23 | Avromatic.use_schema_fingerprint_lookup = true 24 | Avromatic.schema_store = AvroTurf::SchemaStore.new(path: 'spec/avro/schema') 25 | Avromatic.custom_type_registry.clear 26 | Avromatic.nested_models = Avromatic::ModelRegistry.new 27 | 28 | Time.zone = 'GMT' 29 | end 30 | 31 | config.filter_run_when_matching :focus 32 | end 33 | 34 | # This needs to be required after the before block that sets 35 | # Avromatic.registry_url 36 | require 'avromatic/rspec' 37 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Salsify, Inc 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /lib/avromatic/model/types/date_type.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'avromatic/model/types/abstract_type' 4 | 5 | module Avromatic 6 | module Model 7 | module Types 8 | class DateType < AbstractType 9 | VALUE_CLASSES = [::Date].freeze 10 | 11 | def value_classes 12 | VALUE_CLASSES 13 | end 14 | 15 | def name 16 | 'date' 17 | end 18 | 19 | def coerce(input) 20 | if input.is_a?(::Time) || input.is_a?(::DateTime) 21 | ::Date.new(input.year, input.month, input.day) 22 | elsif input.nil? || input.is_a?(::Date) 23 | input 24 | else 25 | raise ArgumentError.new("Could not coerce '#{input.inspect}' to #{name}") 26 | end 27 | end 28 | 29 | def coercible?(input) 30 | input.nil? || input.is_a?(::Date) || input.is_a?(::Time) 31 | end 32 | 33 | alias_method :coerced?, :coercible? 34 | 35 | def serialize(value, _strict) 36 | value 37 | end 38 | 39 | def referenced_model_classes 40 | EMPTY_ARRAY 41 | end 42 | end 43 | end 44 | end 45 | end 46 | -------------------------------------------------------------------------------- /lib/avromatic/model/types/fixed_type.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'avromatic/model/types/abstract_type' 4 | 5 | module Avromatic 6 | module Model 7 | module Types 8 | class FixedType < AbstractType 9 | VALUE_CLASSES = [::String].freeze 10 | 11 | attr_reader :size 12 | 13 | def initialize(size) 14 | super() 15 | @size = size 16 | end 17 | 18 | def name 19 | "fixed(#{size})" 20 | end 21 | 22 | def value_classes 23 | VALUE_CLASSES 24 | end 25 | 26 | def coerce(input) 27 | if coercible?(input) 28 | input 29 | else 30 | raise ArgumentError.new("Could not coerce '#{input.inspect}' to #{name}") 31 | end 32 | end 33 | 34 | def coercible?(input) 35 | input.nil? || (input.is_a?(::String) && input.length == size) 36 | end 37 | 38 | alias_method :coerced?, :coercible? 39 | 40 | def serialize(value, _strict) 41 | value 42 | end 43 | 44 | def referenced_model_classes 45 | EMPTY_ARRAY 46 | end 47 | end 48 | end 49 | end 50 | end 51 | -------------------------------------------------------------------------------- /spec/avro/schema/test/primitive_types.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "primitive_types", 4 | "namespace": "test", 5 | "fields": [ 6 | { 7 | "name": "s", 8 | "type": "string" 9 | }, 10 | { 11 | "name": "b", 12 | "type": "bytes" 13 | }, 14 | { 15 | "name": "tf", 16 | "type": "boolean" 17 | }, 18 | { 19 | "name": "i", 20 | "type": "int" 21 | }, 22 | { 23 | "name": "l", 24 | "type": "long" 25 | }, 26 | { 27 | "name": "f", 28 | "type": "float" 29 | }, 30 | { 31 | "name": "d", 32 | "type": "double" 33 | }, 34 | { 35 | "name": "n", 36 | "type": "null" 37 | }, 38 | { 39 | "name": "fx", 40 | "type": { 41 | "name": "__primitive_types_fx_fixed", 42 | "type": "fixed", 43 | "namespace": "test", 44 | "size": 7 45 | } 46 | }, 47 | { 48 | "name": "e", 49 | "type": { 50 | "name": "__primitive_types_e_enum", 51 | "type": "enum", 52 | "namespace": "test", 53 | "symbols": [ 54 | "A", 55 | "B" 56 | ] 57 | } 58 | } 59 | ] 60 | } 61 | -------------------------------------------------------------------------------- /lib/avromatic/model/nested_models.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'active_support/inflector/methods' 4 | 5 | module Avromatic 6 | module Model 7 | # This module handles integration with the ModelRegistry and support 8 | # for nested model reuse. 9 | module NestedModels 10 | extend ActiveSupport::Concern 11 | 12 | module ClassMethods 13 | # Register this model if it can be used as a nested model. 14 | def register! 15 | return unless key_avro_schema.nil? && value_avro_schema.type_sym == :record 16 | 17 | processed = Set.new 18 | roots = [self] 19 | until roots.empty? 20 | model = roots.shift 21 | # Avoid any nested model dependency cycles by ignoring already processed models 22 | next unless processed.add?(model) 23 | 24 | nested_models.ensure_registered_model(model) 25 | roots.concat(model.referenced_model_classes) 26 | end 27 | end 28 | 29 | def referenced_model_classes 30 | attribute_definitions.values.flat_map { |definition| definition.type.referenced_model_classes }.freeze 31 | end 32 | end 33 | end 34 | end 35 | end 36 | -------------------------------------------------------------------------------- /lib/avromatic/io/datum_writer.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Avromatic 4 | module IO 5 | # Subclass DatumWriter to use additional information about union member 6 | # index. 7 | class DatumWriter < Avro::IO::DatumWriter 8 | def write_union(writers_schema, datum, encoder) 9 | optional = writers_schema.schemas.first.type_sym == :null 10 | if datum.is_a?(Avromatic::IO::UnionDatum) 11 | index_of_schema = datum.member_index 12 | # Avromatic does not treat the null of an optional field as part of the union 13 | index_of_schema += 1 if optional 14 | datum = datum.datum 15 | elsif optional && writers_schema.schemas.size == 2 16 | # Optimize for the common case of a union that's just an optional field 17 | index_of_schema = datum.nil? ? 0 : 1 18 | else 19 | index_of_schema = writers_schema.schemas.find_index do |schema| 20 | Avro::Schema.validate(schema, datum) 21 | end 22 | end 23 | 24 | raise Avro::IO::AvroTypeError.new(writers_schema, datum) unless index_of_schema 25 | 26 | encoder.write_long(index_of_schema) 27 | write_data(writers_schema.schemas[index_of_schema], datum, encoder) 28 | end 29 | end 30 | end 31 | end 32 | -------------------------------------------------------------------------------- /lib/avromatic/model/types/float_type.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'avromatic/model/types/abstract_type' 4 | 5 | module Avromatic 6 | module Model 7 | module Types 8 | class FloatType < AbstractType 9 | VALUE_CLASSES = [::Float].freeze 10 | INPUT_CLASSES = [::Float, ::Integer].freeze 11 | 12 | def value_classes 13 | VALUE_CLASSES 14 | end 15 | 16 | def input_classes 17 | INPUT_CLASSES 18 | end 19 | 20 | def name 21 | 'float' 22 | end 23 | 24 | def coerce(input) 25 | if input.nil? || input.is_a?(::Float) 26 | input 27 | elsif input.is_a?(::Integer) 28 | input.to_f 29 | else 30 | raise ArgumentError.new("Could not coerce '#{input.inspect}' to #{name}") 31 | end 32 | end 33 | 34 | def coercible?(input) 35 | input.nil? || input.is_a?(::Float) || input.is_a?(::Integer) 36 | end 37 | 38 | def coerced?(input) 39 | input.nil? || input.is_a?(::Float) 40 | end 41 | 42 | def serialize(value, _strict) 43 | value 44 | end 45 | 46 | def referenced_model_classes 47 | EMPTY_ARRAY 48 | end 49 | end 50 | end 51 | end 52 | end 53 | -------------------------------------------------------------------------------- /lib/avromatic/model/types/string_type.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'avromatic/model/types/abstract_type' 4 | 5 | module Avromatic 6 | module Model 7 | module Types 8 | class StringType < AbstractType 9 | VALUE_CLASSES = [::String].freeze 10 | INPUT_CLASSES = [::String, ::Symbol].freeze 11 | 12 | def value_classes 13 | VALUE_CLASSES 14 | end 15 | 16 | def input_classes 17 | INPUT_CLASSES 18 | end 19 | 20 | def name 21 | 'string' 22 | end 23 | 24 | def coerce(input) 25 | if input.nil? || input.is_a?(::String) 26 | input 27 | elsif input.is_a?(::Symbol) 28 | input.to_s 29 | else 30 | raise ArgumentError.new("Could not coerce '#{input.inspect}' to #{name}") 31 | end 32 | end 33 | 34 | def coercible?(input) 35 | input.nil? || input.is_a?(::String) || input.is_a?(::Symbol) 36 | end 37 | 38 | def coerced?(value) 39 | value.nil? || value.is_a?(::String) 40 | end 41 | 42 | def serialize(value, _strict) 43 | value 44 | end 45 | 46 | def referenced_model_classes 47 | EMPTY_ARRAY 48 | end 49 | end 50 | end 51 | end 52 | end 53 | -------------------------------------------------------------------------------- /lib/avromatic/model/types/abstract_type.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Avromatic 4 | module Model 5 | module Types 6 | class AbstractType 7 | EMPTY_ARRAY = [].freeze 8 | private_constant :EMPTY_ARRAY 9 | 10 | def value_classes 11 | raise "#{__method__} must be overridden by #{self.class.name}" 12 | end 13 | 14 | def input_classes 15 | value_classes 16 | end 17 | 18 | def name 19 | raise "#{__method__} must be overridden by #{self.class.name}" 20 | end 21 | 22 | def coerce(_input) 23 | raise "#{__method__} must be overridden by #{self.class.name}" 24 | end 25 | 26 | def coercible?(_input) 27 | raise "#{__method__} must be overridden by #{self.class.name}" 28 | end 29 | 30 | def coerced?(_value) 31 | raise "#{__method__} must be overridden by #{self.class.name}" 32 | end 33 | 34 | # Note we use positional args rather than keyword args to reduce 35 | # memory allocations 36 | def serialize(_value, _strict) 37 | raise "#{__method__} must be overridden by #{self.class.name}" 38 | end 39 | 40 | def referenced_model_classes 41 | raise "#{__method__} must be overridden by #{self.class.name}" 42 | end 43 | end 44 | end 45 | end 46 | end 47 | -------------------------------------------------------------------------------- /spec/avromatic/io/union_datum_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | describe Avromatic::IO::UnionDatum do 4 | describe "#==" do 5 | it "returns true when the member index and datum are equal" do 6 | member1 = described_class.new(1, foo: 'bar') 7 | member2 = described_class.new(1, foo: 'bar') 8 | expect(member1).to eq(member2) 9 | end 10 | 11 | it "returns false when the member indexes are not equal" do 12 | member1 = described_class.new(1, foo: 'bar') 13 | member2 = described_class.new(2, foo: 'bar') 14 | expect(member1).not_to eq(member2) 15 | end 16 | 17 | it "returns false when the datums are not equal" do 18 | member1 = described_class.new(1, foo: 'bar') 19 | member2 = described_class.new(1, foo: 'baz') 20 | expect(member1).not_to eq(member2) 21 | end 22 | end 23 | 24 | describe "#hash" do 25 | it "returns the same value for equal UnionDatums" do 26 | member1 = described_class.new(1, foo: 'bar') 27 | member2 = described_class.new(1, foo: 'bar') 28 | expect(member1.hash).to eq(member2.hash) 29 | end 30 | 31 | it "returns different values when the member indexes are different" do 32 | member1 = described_class.new(1, foo: 'bar') 33 | member2 = described_class.new(2, foo: 'bar') 34 | expect(member1.hash).not_to eq(member2.hash) 35 | end 36 | 37 | it "returns different values when the datums are different" do 38 | member1 = described_class.new(1, foo: 'bar') 39 | member2 = described_class.new(1, foo: 'baz') 40 | expect(member1.hash).not_to eq(member2.hash) 41 | end 42 | end 43 | end 44 | -------------------------------------------------------------------------------- /lib/avromatic/model/types/enum_type.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'avromatic/model/types/abstract_type' 4 | 5 | module Avromatic 6 | module Model 7 | module Types 8 | class EnumType < AbstractType 9 | VALUE_CLASSES = [::String].freeze 10 | INPUT_CLASSES = [::String, ::Symbol].freeze 11 | 12 | attr_reader :allowed_values 13 | 14 | def initialize(allowed_values) 15 | super() 16 | @allowed_values = allowed_values.to_set 17 | end 18 | 19 | def name 20 | "enum#{allowed_values.to_a}" 21 | end 22 | 23 | def value_classes 24 | VALUE_CLASSES 25 | end 26 | 27 | def input_classes 28 | INPUT_CLASSES 29 | end 30 | 31 | def coerce(input) 32 | if input.nil? 33 | input 34 | elsif coercible?(input) 35 | input.to_s 36 | else 37 | raise ArgumentError.new("Could not coerce '#{input.inspect}' to #{name}") 38 | end 39 | end 40 | 41 | def coerced?(input) 42 | input.nil? || (input.is_a?(::String) && allowed_values.include?(input)) 43 | end 44 | 45 | def coercible?(input) 46 | input.nil? || 47 | (input.is_a?(::String) && allowed_values.include?(input)) || 48 | (input.is_a?(::Symbol) && allowed_values.include?(input.to_s)) 49 | end 50 | 51 | def serialize(value, _strict) 52 | value 53 | end 54 | 55 | def referenced_model_classes 56 | EMPTY_ARRAY 57 | end 58 | end 59 | end 60 | end 61 | end 62 | -------------------------------------------------------------------------------- /lib/avromatic/model/types/array_type.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'avromatic/model/types/abstract_type' 4 | 5 | module Avromatic 6 | module Model 7 | module Types 8 | class ArrayType < AbstractType 9 | VALUE_CLASSES = [::Array].freeze 10 | 11 | attr_reader :value_type 12 | 13 | def initialize(value_type:) 14 | super() 15 | @value_type = value_type 16 | end 17 | 18 | def value_classes 19 | VALUE_CLASSES 20 | end 21 | 22 | def name 23 | "array[#{value_type.name}]" 24 | end 25 | 26 | def coerce(input) 27 | if input.nil? 28 | input 29 | elsif input.is_a?(::Array) 30 | input.map { |element_input| value_type.coerce(element_input) } 31 | else 32 | raise ArgumentError.new("Could not coerce '#{input.inspect}' to #{name}") 33 | end 34 | end 35 | 36 | def coercible?(input) 37 | input.nil? || (input.is_a?(::Array) && input.all? { |element_input| value_type.coercible?(element_input) }) 38 | end 39 | 40 | def coerced?(value) 41 | value.nil? || (value.is_a?(::Array) && value.all? { |element| value_type.coerced?(element) }) 42 | end 43 | 44 | def serialize(value, strict) 45 | if value.nil? 46 | value 47 | else 48 | value.map { |element| value_type.serialize(element, strict) } 49 | end 50 | end 51 | 52 | def referenced_model_classes 53 | value_type.referenced_model_classes 54 | end 55 | end 56 | end 57 | end 58 | end 59 | -------------------------------------------------------------------------------- /lib/avromatic/io/datum_reader.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Avromatic 4 | module IO 5 | # Subclass DatumReader to include additional information about the union 6 | # member index used. The code modified below is based on salsify/avro, 7 | # branch 'salsify-master' with the tag 'v1.9.0.3' 8 | class DatumReader < Avro::IO::DatumReader 9 | 10 | def read_data(writers_schema, readers_schema, decoder) 11 | # schema resolution: reader's schema is a union, writer's schema is not 12 | return super unless writers_schema.type_sym != :union && readers_schema.type_sym == :union 13 | 14 | rs_index = readers_schema.schemas.find_index do |s| 15 | self.class.match_schemas(writers_schema, s) 16 | end 17 | 18 | raise Avro::IO::SchemaMatchException.new(writers_schema, readers_schema) unless rs_index 19 | 20 | datum = read_data(writers_schema, readers_schema.schemas[rs_index], decoder) 21 | optional = readers_schema.schemas.first.type_sym == :null 22 | 23 | if readers_schema.schemas.size == 2 && optional 24 | # Avromatic does not treat the union of null and 1 other type as a union 25 | datum 26 | elsif datum.nil? 27 | # Avromatic does not treat the null of an optional field as part of the union 28 | nil 29 | else 30 | # Avromatic does not treat the null of an optional field as part of the union so 31 | # adjust the member index accordingly 32 | member_index = optional ? rs_index - 1 : rs_index 33 | Avromatic::IO::UnionDatum.new(member_index, datum) 34 | end 35 | end 36 | end 37 | end 38 | end 39 | -------------------------------------------------------------------------------- /lib/avromatic/model/types/decimal_type.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'bigdecimal' 4 | require 'bigdecimal/util' 5 | require 'avromatic/model/types/abstract_type' 6 | 7 | module Avromatic 8 | module Model 9 | module Types 10 | class DecimalType < AbstractType 11 | VALUE_CLASSES = [::BigDecimal].freeze 12 | INPUT_CLASSES = [::BigDecimal, ::Float, ::Integer].freeze 13 | 14 | attr_reader :precision, :scale 15 | 16 | def initialize(precision:, scale: 0) 17 | super() 18 | @precision = precision 19 | @scale = scale 20 | end 21 | 22 | def value_classes 23 | VALUE_CLASSES 24 | end 25 | 26 | def input_classes 27 | INPUT_CLASSES 28 | end 29 | 30 | def name 31 | "decimal(#{precision}, #{scale})" 32 | end 33 | 34 | def coerce(input) 35 | case input 36 | when ::NilClass, ::BigDecimal 37 | input 38 | when ::Float, ::Integer 39 | input.to_d 40 | else 41 | raise ArgumentError.new("Could not coerce '#{input.inspect}' to #{name}") 42 | end 43 | end 44 | 45 | def coercible?(input) 46 | input.nil? || input_classes.any? { |input_class| input.is_a?(input_class) } 47 | end 48 | 49 | def coerced?(value) 50 | value.nil? || value_classes.any? { |value_class| value.is_a?(value_class) } 51 | end 52 | 53 | def serialize(value, _strict) 54 | value 55 | end 56 | 57 | def referenced_model_classes 58 | EMPTY_ARRAY 59 | end 60 | end 61 | end 62 | end 63 | end 64 | -------------------------------------------------------------------------------- /lib/avromatic/model/types/record_type.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'avromatic/model/types/abstract_type' 4 | 5 | module Avromatic 6 | module Model 7 | module Types 8 | class RecordType < AbstractType 9 | attr_reader :record_class, :value_classes, :input_classes 10 | 11 | def initialize(record_class:) 12 | super() 13 | @record_class = record_class 14 | @value_classes = [record_class].freeze 15 | @input_classes = [record_class, Hash].freeze 16 | end 17 | 18 | def name 19 | record_class.name.to_s.freeze 20 | end 21 | 22 | def coerce(input) 23 | if input.nil? || input.is_a?(record_class) 24 | input 25 | elsif input.is_a?(Hash) 26 | record_class.new(input) 27 | else 28 | raise ArgumentError.new("Could not coerce '#{input.inspect}' to #{name}") 29 | end 30 | end 31 | 32 | def coercible?(input) 33 | # TODO: Is there a better way to figure this out? 34 | input.nil? || input.is_a?(record_class) || coerce(input).valid? 35 | rescue StandardError 36 | false 37 | end 38 | 39 | def coerced?(value) 40 | value.nil? || value.is_a?(record_class) 41 | end 42 | 43 | def serialize(value, strict) 44 | if value.nil? 45 | value 46 | elsif strict 47 | value.avro_value_datum 48 | else 49 | value.value_attributes_for_avro 50 | end 51 | end 52 | 53 | def referenced_model_classes 54 | [record_class].freeze 55 | end 56 | end 57 | end 58 | end 59 | end 60 | -------------------------------------------------------------------------------- /spec/support/contexts/logical_types_serialization.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # This examples expects let-variables to be defined for: 4 | # decoded: a model instance based on the encoded_value 5 | shared_examples_for "logical type encoding and decoding" do 6 | context "logical types" do 7 | let(:schema_name) { 'test.logical_types_with_decimal' } 8 | let(:test_class) do 9 | Avromatic::Model.model(schema_name: schema_name) 10 | end 11 | let(:now) do 12 | # ensure that the Time value has nanoseconds 13 | time = Time.now 14 | if time.nsec % 1000 == 0 15 | Time.at(time.to_i, (time.nsec + rand(999) + 1) / 1000.0) 16 | else 17 | time 18 | end 19 | end 20 | 21 | with_logical_types do 22 | context "supported" do 23 | let(:values) do 24 | { 25 | date: Date.today, 26 | decimal: BigDecimal('5.2'), 27 | ts_msec: now, 28 | ts_usec: now, 29 | unknown: 42 30 | } 31 | end 32 | 33 | it "encodes and decodes instances" do 34 | expect(decoded).to eq(instance) 35 | end 36 | end 37 | end 38 | 39 | without_logical_types do 40 | context "unsupported" do 41 | let(:epoch_start) { Date.new(1970, 1, 1) } 42 | let(:values) do 43 | { 44 | date: (Date.today - epoch_start).to_i, 45 | decimal: BigDecimal('1.5432'), 46 | ts_msec: now.to_i + now.usec / 1000 * 1000, 47 | ts_usec: now.to_i * 1_000_000 + now.usec, 48 | unknown: 42 49 | } 50 | end 51 | 52 | it "encodes and decodes instances" do 53 | expect(decoded).to eq(instance) 54 | end 55 | end 56 | end 57 | end 58 | end 59 | -------------------------------------------------------------------------------- /lib/avromatic/model/types/abstract_timestamp_type.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'active_support/time_with_zone' 4 | 5 | module Avromatic 6 | module Model 7 | module Types 8 | class AbstractTimestampType < AbstractType 9 | VALUE_CLASSES = [::Time].freeze 10 | INPUT_CLASSES = [::Time, ::DateTime, ::ActiveSupport::TimeWithZone].freeze 11 | 12 | def value_classes 13 | VALUE_CLASSES 14 | end 15 | 16 | def input_classes 17 | INPUT_CLASSES 18 | end 19 | 20 | def coerce(input) 21 | if input.nil? || coerced?(input) 22 | input 23 | elsif input.is_a?(::Time) || input.is_a?(::DateTime) 24 | coerce_time(input) 25 | else 26 | raise ArgumentError.new("Could not coerce '#{input.inspect}' to #{name}") 27 | end 28 | end 29 | 30 | def coercible?(input) 31 | input.nil? || input.is_a?(::Time) || input.is_a?(::DateTime) 32 | end 33 | 34 | def coerced?(value) 35 | # ActiveSupport::TimeWithZone overrides is_a? is to make it look like a Time 36 | # even though it's not which can lead to unexpected behavior if we don't force 37 | # a coercion 38 | value.is_a?(::Time) && value.class != ActiveSupport::TimeWithZone && truncated?(value) 39 | end 40 | 41 | def serialize(value, _strict) 42 | value 43 | end 44 | 45 | private 46 | 47 | def truncated?(_value) 48 | raise "#{__method__} must be overridden by #{self.class.name}" 49 | end 50 | 51 | def coerce_time(_input) 52 | raise "#{__method__} must be overridden by #{self.class.name}" 53 | end 54 | end 55 | end 56 | end 57 | end 58 | -------------------------------------------------------------------------------- /lib/avromatic/model/custom_type_registry.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'avromatic/model/custom_type_configuration' 4 | 5 | module Avromatic 6 | module Model 7 | class CustomTypeRegistry 8 | 9 | delegate :clear, to: :custom_types 10 | 11 | def initialize 12 | @custom_types = {} 13 | end 14 | 15 | # @param fullname [String] The fullname of the Avro type. 16 | # @param value_class [Class] Optional class to use for the attribute. 17 | # If unspecified then the default class for the Avro field is used. 18 | # @block If a block is specified then the CustomType is yielded for 19 | # additional configuration. 20 | def register_type(fullname, value_class = nil) 21 | custom_types[fullname.to_s] = Avromatic::Model::CustomTypeConfiguration.new(value_class).tap do |type| 22 | yield(type) if block_given? 23 | end 24 | end 25 | 26 | # @object [Avro::Schema] Custom type may be fetched based on a Avro field 27 | # or schema. 28 | def registered?(object) 29 | field_type = object.is_a?(Avro::Schema::Field) ? object.type : object 30 | custom_types.include?(field_type.fullname) if field_type.is_a?(Avro::Schema::NamedSchema) 31 | end 32 | 33 | # @object [Avro::Schema] Custom type may be fetched based on a Avro field 34 | # or schema. If there is no custom type, then an exception will be thrown. 35 | # @field_class [Object] Value class that has been determined for a field. 36 | def fetch(object) 37 | field_type = object.is_a?(Avro::Schema::Field) ? object.type : object 38 | fullname = field_type.fullname if field_type.is_a?(Avro::Schema::NamedSchema) 39 | custom_types.fetch(fullname) 40 | end 41 | 42 | private 43 | 44 | attr_reader :custom_types 45 | end 46 | end 47 | end 48 | -------------------------------------------------------------------------------- /lib/avromatic/model/custom_type_configuration.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Avromatic 4 | module Model 5 | 6 | # Instances of this class contains the configuration for custom handling of 7 | # a named type (record, enum, fixed). 8 | class CustomTypeConfiguration 9 | 10 | attr_accessor :to_avro, :from_avro, :value_class 11 | 12 | def initialize(value_class) 13 | @value_class = value_class 14 | end 15 | 16 | # A deserializer method is used when assigning to the model. It is used both when 17 | # deserializing a model instance from Avro and when directly instantiating 18 | # an instance. The deserializer method must accept a single argument and return 19 | # the value to store in the model for the attribute. 20 | def deserializer 21 | proc = from_avro_proc 22 | wrap_proc(proc) if proc 23 | end 24 | 25 | # A serializer method is used when preparing attributes to be serialized using 26 | # Avro. The serializer method must accept a single argument of the model value 27 | # for the attribute and return a value in a form that Avro can serialize 28 | # for the attribute. 29 | def serializer 30 | proc = to_avro_proc 31 | wrap_proc(proc) if proc 32 | end 33 | 34 | private 35 | 36 | def to_avro_proc 37 | to_avro || value_class_method(:to_avro) 38 | end 39 | 40 | def from_avro_proc 41 | from_avro || value_class_method(:from_avro) 42 | end 43 | 44 | def value_class_method(method_name) 45 | value_class && value_class.respond_to?(method_name) && 46 | value_class.method(method_name).to_proc 47 | end 48 | 49 | # Wrap the supplied Proc to handle nil. 50 | def wrap_proc(proc) 51 | ->(value) { proc.call(value) if value } 52 | end 53 | end 54 | end 55 | end 56 | -------------------------------------------------------------------------------- /lib/avromatic/model.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'avromatic/model/builder' 4 | require 'avromatic/model/coercion_error' 5 | require 'avromatic/model/message_decoder' 6 | require 'avromatic/model/custom_type_registry' 7 | require 'avromatic/model/validation_error' 8 | require 'avromatic/model/unknown_attribute_error' 9 | 10 | module Avromatic 11 | module Model 12 | 13 | # Returns a module that can be included in a class to define a model 14 | # based on Avro schema(s). 15 | # 16 | # Example: 17 | # class MyTopic 18 | # include Avromatic::Model.build(schema_name: :topic_value, 19 | # key_schema_name: :topic_key) 20 | # end 21 | # 22 | # Either schema(_name) or value_schema(_name) must be specified. 23 | # 24 | # value_schema(_name) is handled identically to schema(_name) and is 25 | # treated like an alias for use when both a value and a key schema are 26 | # specified. 27 | # 28 | # Options: 29 | # value_schema_name: 30 | # The full name of an Avro schema. The schema will be loaded 31 | # using the schema store. 32 | # value_schema: 33 | # An Avro::Schema. 34 | # schema_name: 35 | # The full name of an Avro schema. The schema will be loaded 36 | # using the schema store. 37 | # schema: 38 | # An Avro::Schema. 39 | # key_schema_name: 40 | # The full name of an Avro schema for the key. When an instance of 41 | # the model is encoded, this schema will be used to encode the key. 42 | # The schema will be loaded using the schema store. 43 | # key_schema: 44 | # An Avro::Schema for the key. 45 | def self.build(**options) 46 | Builder.new(**options).mod 47 | end 48 | 49 | # Returns an anonymous class, that can be assigned to a constant, 50 | # defined based on Avro schema(s). See Avromatic::Model.build. 51 | def self.model(**options) 52 | Builder.model(**options) 53 | end 54 | end 55 | end 56 | -------------------------------------------------------------------------------- /spec/avromatic/io/datum_writer_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | describe Avromatic::IO::DatumWriter do 4 | let(:encoder) { instance_double(Avro::IO::BinaryEncoder) } 5 | let(:schema_name) { 'test.real_union' } 6 | let(:test_class) do 7 | Avromatic::Model.model(schema_name: schema_name) 8 | end 9 | let(:values) do 10 | { 11 | header: 'foo', 12 | message: { bar_message: 'bar' } 13 | } 14 | end 15 | let(:instance) { test_class.new(values) } 16 | let(:use_custom_datum_writer) { true } 17 | let(:datum_writer) { described_class.new(test_class.value_avro_schema) } 18 | let(:union_schema) do 19 | test_class.value_avro_schema.fields.find { |f| f.name == 'message' }.type 20 | end 21 | let(:datum) { instance.value_attributes_for_avro['message'] } 22 | 23 | before do 24 | allow(Avromatic).to receive(:use_custom_datum_writer).and_return(use_custom_datum_writer) 25 | allow(Avro::Schema).to receive(:validate).and_call_original 26 | allow(encoder).to receive(:write_long) 27 | allow(datum_writer).to receive(:write_data) 28 | end 29 | 30 | describe "#write_union" do 31 | before { datum_writer.write_union(union_schema, datum, encoder) } 32 | 33 | context "when the datum includes union member index" do 34 | it "does not call Avro::Schema.validate" do 35 | expect(Avro::Schema).not_to have_received(:validate) 36 | end 37 | 38 | it "calls write_data to encode the union" do 39 | expect(datum_writer).to have_received(:write_data).with(union_schema.schemas[1], datum.datum, encoder) 40 | end 41 | end 42 | 43 | context "when the datum does not include union member index" do 44 | let(:use_custom_datum_writer) { false } 45 | 46 | it "calls validate to find the matching schema" do 47 | expect(Avro::Schema).to have_received(:validate).twice 48 | end 49 | 50 | it "calls write_data to encode the union" do 51 | expect(datum_writer).to have_received(:write_data).with(union_schema.schemas[1], datum, encoder) 52 | end 53 | end 54 | end 55 | end 56 | -------------------------------------------------------------------------------- /lib/avromatic/model/builder.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'active_support/concern' 4 | require 'active_model' 5 | require 'avromatic/model/configuration' 6 | require 'avromatic/model/value_object' 7 | require 'avromatic/model/configurable' 8 | require 'avromatic/model/field_helper' 9 | require 'avromatic/model/nested_models' 10 | require 'avromatic/model/validation' 11 | require 'avromatic/model/types/type_factory' 12 | require 'avromatic/model/attributes' 13 | require 'avromatic/model/raw_serialization' 14 | require 'avromatic/model/messaging_serialization' 15 | 16 | module Avromatic 17 | module Model 18 | 19 | # This class implements generating models from Avro schemas. 20 | class Builder 21 | 22 | attr_reader :mod, :config 23 | 24 | # For options see Avromatic::Model.build 25 | def self.model(**options, &block) 26 | Class.new do 27 | include Avromatic::Model::Builder.new(**options).mod 28 | 29 | # Name is required for attribute validations on an anonymous class. 30 | def self.name 31 | super || (@name ||= config.avro_schema.name.classify) 32 | end 33 | 34 | class_eval(&block) if block 35 | end 36 | end 37 | 38 | # For options see Avromatic::Model.build 39 | def initialize(**options) 40 | @mod = Module.new 41 | @config = Avromatic::Model::Configuration.new(**options) 42 | define_included_method 43 | end 44 | 45 | def inclusions 46 | [ 47 | Avromatic::Model::Configurable, 48 | Avromatic::Model::NestedModels, 49 | Avromatic::Model::Validation, 50 | Avromatic::Model::Attributes, 51 | Avromatic::Model::ValueObject, 52 | Avromatic::Model::RawSerialization, 53 | Avromatic::Model::MessagingSerialization 54 | ] 55 | end 56 | 57 | private 58 | 59 | def define_included_method 60 | local_mod = mod 61 | local_builder = self 62 | mod.define_singleton_method(:included) do |model_class| 63 | model_class.include(*local_builder.inclusions) 64 | model_class.config = local_builder.config 65 | model_class.add_avro_fields(local_mod) 66 | end 67 | end 68 | end 69 | end 70 | end 71 | -------------------------------------------------------------------------------- /avromatic.gemspec: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | lib = File.expand_path('lib', __dir__) 4 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 5 | require 'avromatic/version' 6 | 7 | Gem::Specification.new do |spec| 8 | spec.name = 'avromatic' 9 | spec.version = Avromatic::VERSION 10 | spec.authors = ['Salsify Engineering'] 11 | spec.email = ['engineering@salsify.com'] 12 | 13 | spec.summary = 'Generate Ruby models from Avro schemas' 14 | spec.description = spec.summary 15 | spec.homepage = 'https://github.com/salsify/avromatic.git' 16 | spec.license = 'MIT' 17 | 18 | if spec.respond_to?(:metadata) 19 | spec.metadata['allowed_push_host'] = 'https://rubygems.org' 20 | spec.metadata['rubygems_mfa_required'] = 'true' 21 | else 22 | raise 'RubyGems 2.0 or newer is required to set allowed_push_host.' 23 | end 24 | 25 | spec.files = `git ls-files -z`.split("\x0").select { |f| f.match(%r{^(bin/|lib/|LICENSE.txt)}) } 26 | spec.bindir = 'exe' 27 | spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } 28 | spec.require_paths = ['lib'] 29 | 30 | spec.metadata['rubygems_mfa_required'] = 'true' 31 | 32 | spec.required_ruby_version = '>= 2.7' 33 | 34 | spec.add_runtime_dependency 'activemodel', '>= 5.2', '< 7.2' 35 | spec.add_runtime_dependency 'activesupport', '>= 5.2', '< 7.2' 36 | spec.add_runtime_dependency 'avro', '>= 1.11.0', '< 1.12' 37 | spec.add_runtime_dependency 'avro_schema_registry-client', '>= 0.4.0' 38 | spec.add_runtime_dependency 'avro_turf' 39 | spec.add_runtime_dependency 'ice_nine' 40 | 41 | spec.add_development_dependency 'appraisal' 42 | spec.add_development_dependency 'avro-builder', '>= 0.12.0' 43 | spec.add_development_dependency 'bundler', '~> 2.0' 44 | spec.add_development_dependency 'overcommit', '0.35.0' 45 | spec.add_development_dependency 'rake', '~> 13.0' 46 | spec.add_development_dependency 'rspec', '~> 3.8' 47 | spec.add_development_dependency 'rspec_junit_formatter' 48 | spec.add_development_dependency 'salsify_rubocop', '~> 1.27.1' 49 | spec.add_development_dependency 'simplecov' 50 | spec.add_development_dependency 'webmock' 51 | # For AvroSchemaRegistry::FakeServer 52 | spec.add_development_dependency 'sinatra' 53 | end 54 | -------------------------------------------------------------------------------- /lib/avromatic/messaging.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'avro_turf/messaging' 4 | require 'avromatic/io' 5 | 6 | module Avromatic 7 | # Subclass AvroTurf::Messaging to use a custom DatumReader and DatumWriter 8 | class Messaging < AvroTurf::Messaging 9 | attr_reader :registry 10 | 11 | def decode(data, schema_name: nil, namespace: @namespace) 12 | readers_schema = schema_name && @schema_store.find(schema_name, namespace) 13 | stream = StringIO.new(data) 14 | decoder = Avro::IO::BinaryDecoder.new(stream) 15 | 16 | # The first byte is MAGIC!!! 17 | magic_byte = decoder.read(1) 18 | 19 | raise "Expected data to begin with a magic byte, got `#{magic_byte.inspect}`" if magic_byte != MAGIC_BYTE 20 | 21 | # The schema id is a 4-byte big-endian integer. 22 | schema_id = decoder.read(4).unpack1('N') 23 | 24 | writers_schema = @schemas_by_id.fetch(schema_id) do 25 | schema_json = @registry.fetch(schema_id) 26 | @schemas_by_id[schema_id] = Avro::Schema.parse(schema_json) 27 | end 28 | 29 | # The following line differs from the parent class to use a custom DatumReader 30 | reader_class = Avromatic.use_custom_datum_reader ? Avromatic::IO::DatumReader : Avro::IO::DatumReader 31 | reader = reader_class.new(writers_schema, readers_schema) 32 | reader.read(decoder) 33 | end 34 | 35 | def encode(message, schema_name: nil, namespace: @namespace, subject: nil) 36 | schema = @schema_store.find(schema_name, namespace) 37 | 38 | # Schemas are registered under the full name of the top level Avro record 39 | # type, or `subject` if it's provided. 40 | schema_id = @registry.register(subject || schema.fullname, schema) 41 | 42 | stream = StringIO.new 43 | encoder = Avro::IO::BinaryEncoder.new(stream) 44 | 45 | # Always start with the magic byte. 46 | encoder.write(MAGIC_BYTE) 47 | 48 | # The schema id is encoded as a 4-byte big-endian integer. 49 | encoder.write([schema_id].pack('N')) 50 | 51 | # The following line differs from the parent class to use a custom DatumWriter 52 | writer_class = Avromatic.use_custom_datum_writer ? Avromatic::IO::DatumWriter : Avro::IO::DatumWriter 53 | writer = writer_class.new(schema) 54 | 55 | # The actual message comes last. 56 | writer.write(message, encoder) 57 | 58 | stream.string 59 | end 60 | end 61 | end 62 | -------------------------------------------------------------------------------- /lib/avromatic/model/types/custom_type.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'avromatic/model/types/abstract_type' 4 | 5 | module Avromatic 6 | module Model 7 | module Types 8 | class CustomType < AbstractType 9 | IDENTITY_PROC = Proc.new { |value| value } 10 | 11 | attr_reader :custom_type_configuration, :value_classes, :default_type 12 | 13 | def initialize(custom_type_configuration:, default_type:) 14 | super() 15 | @custom_type_configuration = custom_type_configuration 16 | @default_type = default_type 17 | @deserializer = custom_type_configuration.deserializer || IDENTITY_PROC 18 | @serializer = custom_type_configuration.serializer || IDENTITY_PROC 19 | @value_classes = if custom_type_configuration.value_class 20 | [custom_type_configuration.value_class].freeze 21 | else 22 | default_type.value_classes 23 | end 24 | end 25 | 26 | def input_classes 27 | # We don't know the valid input classes for a custom type 28 | end 29 | 30 | def name 31 | if custom_type_configuration.value_class 32 | custom_type_configuration.value_class.name.to_s.freeze 33 | else 34 | default_type.name 35 | end 36 | end 37 | 38 | def coerce(input) 39 | if input.nil? 40 | input 41 | else 42 | @deserializer.call(input) 43 | end 44 | rescue StandardError => e 45 | # TODO: Don't swallow this 46 | raise ArgumentError.new("Could not coerce '#{input.inspect}' to #{name}: #{e.message}") 47 | end 48 | 49 | def coercible?(input) 50 | # TODO: Delegate this to optional configuration 51 | input.nil? || !coerce(input).nil? 52 | rescue ArgumentError 53 | false 54 | end 55 | 56 | def coerced?(value) 57 | # TODO: Delegate this to optional configuration 58 | coerce(value) == value 59 | rescue ArgumentError 60 | false 61 | end 62 | 63 | def serialize(value, _strict) 64 | @serializer.call(value) 65 | end 66 | 67 | def referenced_model_classes 68 | default_type.referenced_model_classes 69 | end 70 | end 71 | end 72 | end 73 | end 74 | -------------------------------------------------------------------------------- /lib/avromatic/model/types/map_type.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'avromatic/model/types/abstract_type' 4 | 5 | module Avromatic 6 | module Model 7 | module Types 8 | class MapType < AbstractType 9 | VALUE_CLASSES = [::Hash].freeze 10 | 11 | attr_reader :value_type, :key_type 12 | 13 | def initialize(key_type:, value_type:) 14 | super() 15 | @key_type = key_type 16 | @value_type = value_type 17 | end 18 | 19 | def name 20 | "map[#{key_type.name} => #{value_type.name}]" 21 | end 22 | 23 | def value_classes 24 | VALUE_CLASSES 25 | end 26 | 27 | def coerce(input) 28 | if input.nil? 29 | input 30 | elsif input.is_a?(::Hash) 31 | input.each_with_object({}) do |(key_input, value_input), result| 32 | result[key_type.coerce(key_input)] = value_type.coerce(value_input) 33 | end 34 | else 35 | raise ArgumentError.new("Could not coerce '#{input.inspect}' to #{name}") 36 | end 37 | end 38 | 39 | def coercible?(input) 40 | if input.nil? 41 | true 42 | elsif input.is_a?(Hash) 43 | input.all? do |key_input, value_input| 44 | key_type.coercible?(key_input) && value_type.coercible?(value_input) 45 | end 46 | else 47 | false 48 | end 49 | end 50 | 51 | def coerced?(value) 52 | if value.nil? 53 | true 54 | elsif value.is_a?(Hash) 55 | value.all? do |element_key, element_value| 56 | key_type.coerced?(element_key) && value_type.coerced?(element_value) 57 | end 58 | else 59 | false 60 | end 61 | end 62 | 63 | def serialize(value, strict) 64 | if value.nil? 65 | value 66 | else 67 | value.each_with_object({}) do |(element_key, element_value), result| 68 | result[key_type.serialize(element_key, strict)] = value_type.serialize(element_value, strict) 69 | end 70 | end 71 | end 72 | 73 | def referenced_model_classes 74 | # According to Avro's spec, keys can only be strings, so we can safely disregard #key_type here. 75 | value_type.referenced_model_classes 76 | end 77 | end 78 | end 79 | end 80 | end 81 | -------------------------------------------------------------------------------- /lib/avromatic/model_registry.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'active_support/core_ext/string/access' 4 | 5 | module Avromatic 6 | # The ModelRegistry class is used to store and fetch nested models by 7 | # their fullname. An optional namespace prefix can be removed from the full 8 | # name that is used to store and fetch models. 9 | class ModelRegistry 10 | 11 | def initialize(remove_namespace_prefix: nil) 12 | @prefix = remove_namespace_prefix 13 | @hash = Hash.new 14 | end 15 | 16 | def clear 17 | @hash.clear 18 | end 19 | 20 | def [](fullname) 21 | @hash.fetch(fullname) 22 | end 23 | alias_method :fetch, :[] 24 | 25 | def register(model) 26 | raise 'models with a key schema are not supported' if model.key_avro_schema 27 | 28 | name = model_fullname(model) 29 | raise "'#{name}' has already been registered" if registered?(name) 30 | 31 | @hash[name] = model 32 | end 33 | 34 | def registered?(fullname_or_model) 35 | fullname = fullname_or_model.is_a?(String) ? fullname_or_model : model_fullname(fullname_or_model) 36 | @hash.key?(fullname) 37 | end 38 | 39 | def model_fullname(model) 40 | name = model.avro_schema.fullname 41 | remove_prefix(name) 42 | end 43 | 44 | def ensure_registered_model(model) 45 | name = model_fullname(model) 46 | if registered?(name) 47 | existing_model = fetch(name) 48 | unless existing_model.equal?(model) 49 | raise "Attempted to replace existing Avromatic model #{model_debug_name(existing_model)} with new model " \ 50 | "#{model_debug_name(model)} as '#{name}'. Perhaps '#{model_debug_name(model)}' needs to be eager loaded " \ 51 | 'via the Avromatic eager_load_models setting?' 52 | end 53 | else 54 | register(model) 55 | end 56 | end 57 | 58 | def remove_prefix(name) 59 | return name if @prefix.nil? 60 | 61 | value = 62 | case @prefix 63 | when String 64 | name.start_with?(@prefix) ? name.from(@prefix.length) : name 65 | when Regexp 66 | name.sub(@prefix, '') 67 | else 68 | raise "unsupported `remove_namespace_prefix` value: #{@prefix}" 69 | end 70 | 71 | value.start_with?('.') ? value.from(1) : value 72 | end 73 | 74 | private 75 | 76 | def model_debug_name(model) 77 | model.name || model.to_s 78 | end 79 | end 80 | end 81 | -------------------------------------------------------------------------------- /spec/avro/dsl/test/nested_nested_record.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | namespace :test 4 | 5 | record :nested_nested_record do 6 | # primitive types 7 | required :n, :null 8 | required :b, :boolean 9 | required :i, :int 10 | required :l, :long 11 | required :f, :float 12 | required :d, :double 13 | required :bs, :bytes 14 | required :str, :string 15 | 16 | # logical types 17 | required :date, :int, logical_type: 'date' 18 | required :ts_msec, :long, logical_type: 'timestamp-millis' 19 | required :ts_usec, :long, logical_type: 'timestamp-micros' 20 | required :unknown, :int, logical_type: 'foobar' 21 | 22 | # complex types 23 | required :e, :enum, symbols: [:A, :B] 24 | required :a, :array, items: :int 25 | required :m, :map, values: :int 26 | required :u, :union, types: [:string, :int] 27 | required :fx, :fixed, size: 2 28 | 29 | required :sub, :record do 30 | # primitive types 31 | required :n, :null 32 | required :b, :boolean 33 | required :i, :int 34 | required :l, :long 35 | required :f, :float 36 | required :d, :double 37 | required :bs, :bytes 38 | required :str, :string 39 | 40 | # logical types 41 | required :date, :int, logical_type: 'date' 42 | required :ts_msec, :long, logical_type: 'timestamp-millis' 43 | required :ts_usec, :long, logical_type: 'timestamp-micros' 44 | required :unknown, :int, logical_type: 'foobar' 45 | 46 | # complex types 47 | required :e, :enum, symbols: [:A, :B] 48 | required :a, :array, items: :int 49 | required :m, :map, values: :int 50 | required :u, :union, types: [:string, :int] 51 | required :fx, :fixed, size: 2 52 | 53 | required :subsub, :record do 54 | # primitive types 55 | required :n, :null 56 | required :b, :boolean 57 | required :i, :int 58 | required :l, :long 59 | required :f, :float 60 | required :d, :double 61 | required :bs, :bytes 62 | required :str, :string 63 | 64 | # logical types 65 | required :date, :int, logical_type: 'date' 66 | required :ts_msec, :long, logical_type: 'timestamp-millis' 67 | required :ts_usec, :long, logical_type: 'timestamp-micros' 68 | required :unknown, :int, logical_type: 'foobar' 69 | 70 | # complex types 71 | required :e, :enum, symbols: [:A, :B] 72 | required :a, :array, items: :int 73 | required :m, :map, values: :int 74 | required :u, :union, types: [:string, :int] 75 | required :fx, :fixed, size: 2 76 | end 77 | end 78 | end 79 | -------------------------------------------------------------------------------- /lib/avromatic/model/validation.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Avromatic 4 | module Model 5 | module Validation 6 | extend ActiveSupport::Concern 7 | include ActiveModel::Validations 8 | 9 | EMPTY_ARRAY = [].freeze 10 | 11 | def self.missing_nested_attributes(attribute, value) 12 | if value.is_a?(Array) 13 | results = [] 14 | value.each_with_index do |element, index| 15 | nested_results = missing_nested_attributes("#{attribute}[#{index}]", element) 16 | results.concat(nested_results) 17 | end 18 | results 19 | elsif value.is_a?(Hash) 20 | results = [] 21 | value.each do |key, element| 22 | nested_results = missing_nested_attributes("#{attribute}['#{key}']", element) 23 | results.concat(nested_results) 24 | end 25 | results 26 | elsif value.respond_to?(:missing_avro_attributes) 27 | value.missing_avro_attributes.map do |missing_child_attribute| 28 | "#{attribute}.#{missing_child_attribute}" 29 | end 30 | else 31 | EMPTY_ARRAY 32 | end 33 | end 34 | 35 | included do 36 | # Support the ActiveModel::Validations interface for backwards compatibility 37 | validate do |model| 38 | model.missing_avro_attributes.each do |missing_attribute| 39 | errors.add(:base, "#{missing_attribute} can't be nil") 40 | end 41 | end 42 | end 43 | 44 | def avro_validate! 45 | results = missing_avro_attributes 46 | if results.present? 47 | raise Avromatic::Model::ValidationError.new("#{self.class.name}(#{attributes.inspect}) cannot be " \ 48 | "serialized because the following attributes are nil: #{results.join(', ')}") 49 | end 50 | end 51 | 52 | def missing_avro_attributes 53 | return @missing_attributes if instance_variable_defined?(:@missing_attributes) 54 | 55 | missing_attributes = [] 56 | 57 | self.class.attribute_definitions.each_value do |attribute_definition| 58 | value = send(attribute_definition.name) 59 | field = attribute_definition.field 60 | if value.nil? && field.type.type_sym != :null && attribute_definition.required? 61 | missing_attributes << field.name 62 | else 63 | missing_attributes.concat(Avromatic::Model::Validation.missing_nested_attributes(field.name, value)) 64 | end 65 | end 66 | 67 | @missing_attributes = missing_attributes.freeze if recursively_immutable? 68 | 69 | missing_attributes 70 | end 71 | end 72 | end 73 | end 74 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | jobs: 3 | lint: 4 | docker: 5 | - image: cimg/ruby:3.0.6 6 | working_directory: ~/avromatic 7 | steps: 8 | - checkout 9 | - restore_cache: 10 | keys: 11 | - v2-gems-ruby-3.0.6-{{ checksum "avromatic.gemspec" }}-{{ checksum "Gemfile" }} 12 | - v2-gems-ruby-3.0.6- 13 | - run: 14 | name: Install Gems 15 | command: | 16 | if ! bundle check --path=vendor/bundle; then 17 | bundle install --path=vendor/bundle --jobs=4 --retry=3 18 | bundle clean 19 | fi 20 | - save_cache: 21 | key: v2-gems-ruby-3.0.6-{{ checksum "avromatic.gemspec" }}-{{ checksum "Gemfile" }} 22 | paths: 23 | - "vendor/bundle" 24 | - "gemfiles/vendor/bundle" 25 | - run: 26 | name: Run Rubocop 27 | command: bundle exec rubocop 28 | test: 29 | parameters: 30 | gemfile: 31 | type: string 32 | ruby-version: 33 | type: string 34 | docker: 35 | - image: cimg/ruby:<< parameters.ruby-version >> 36 | environment: 37 | CIRCLE_TEST_REPORTS: "test-results" 38 | BUNDLE_GEMFILE: << parameters.gemfile >> 39 | working_directory: ~/avromatic 40 | steps: 41 | - checkout 42 | - restore_cache: 43 | keys: 44 | - v2-gems-ruby-<< parameters.ruby-version >>-{{ checksum "avromatic.gemspec" }}-{{ checksum "<< parameters.gemfile >>" }} 45 | - v2-gems-ruby-<< parameters.ruby-version >>- 46 | - run: 47 | name: Install Gems 48 | command: | 49 | if ! bundle check --path=vendor/bundle; then 50 | bundle install --path=vendor/bundle --jobs=4 --retry=3 51 | bundle clean 52 | fi 53 | - save_cache: 54 | key: v2-gems-ruby-<< parameters.ruby-version >>-{{ checksum "avromatic.gemspec" }}-{{ checksum "<< parameters.gemfile >>" }} 55 | paths: 56 | - "vendor/bundle" 57 | - "gemfiles/vendor/bundle" 58 | - run: 59 | name: Run Tests 60 | command: | 61 | bundle exec rspec --format RspecJunitFormatter --out $CIRCLE_TEST_REPORTS/rspec/junit.xml --format progress spec 62 | - store_test_results: 63 | path: "test-results" 64 | workflows: 65 | build: 66 | jobs: 67 | - lint 68 | - test: 69 | matrix: 70 | parameters: 71 | gemfile: 72 | - gemfiles/avro1_11_rails6_1.gemfile 73 | - gemfiles/avro1_11_rails7_0.gemfile 74 | - gemfiles/avro1_11_rails7_1.gemfile 75 | ruby-version: 76 | - 3.0.6 77 | - 3.1.4 78 | - 3.2.2 79 | - 3.3.0 80 | -------------------------------------------------------------------------------- /lib/avromatic/model/configurable.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Avromatic 4 | module Model 5 | 6 | # This concern adds methods for configuration for a model generated from 7 | # Avro schema(s). 8 | module Configurable 9 | extend ActiveSupport::Concern 10 | 11 | # Wraps a reference to a field so we can access both the string and symbolized versions of the name 12 | # without repeated memory allocations. 13 | class FieldReference 14 | attr_reader :name, :name_sym 15 | 16 | def initialize(name) 17 | @name = -name 18 | @name_sym = name.to_sym 19 | end 20 | end 21 | 22 | included do 23 | class_attribute :config, instance_accessor: false, instance_predicate: false 24 | end 25 | 26 | module ClassMethods 27 | delegate :avro_schema, :value_avro_schema, :key_avro_schema, :mutable?, :immutable?, 28 | :avro_schema_subject, :value_avro_schema_subject, :key_avro_schema_subject, to: :config 29 | 30 | def value_avro_field_names 31 | @value_avro_field_names ||= value_avro_schema.fields.map(&:name).map(&:to_sym).freeze 32 | end 33 | 34 | def key_avro_field_names 35 | @key_avro_field_names ||= key_avro_schema.fields.map(&:name).map(&:to_sym).freeze 36 | end 37 | 38 | def value_avro_field_references 39 | @value_avro_field_references ||= value_avro_schema.fields.map do |field| 40 | Avromatic::Model::Configurable::FieldReference.new(field.name) 41 | end.freeze 42 | end 43 | 44 | def key_avro_field_references 45 | @key_avro_field_references ||= key_avro_schema.fields.map do |field| 46 | Avromatic::Model::Configurable::FieldReference.new(field.name) 47 | end.freeze 48 | end 49 | 50 | def value_avro_fields_by_name 51 | @value_avro_fields_by_name ||= mapped_by_name(value_avro_schema) 52 | end 53 | 54 | def key_avro_fields_by_name 55 | @key_avro_fields_by_name ||= mapped_by_name(key_avro_schema) 56 | end 57 | 58 | def nested_models 59 | config.nested_models || Avromatic.nested_models 60 | end 61 | 62 | private 63 | 64 | def mapped_by_name(schema) 65 | schema.fields.each_with_object(Hash.new) do |field, result| 66 | result[field.name.to_sym] = field 67 | end 68 | end 69 | end 70 | 71 | delegate :avro_schema, :value_avro_schema, :key_avro_schema, 72 | :avro_schema_subject, :value_avro_schema_subject, :key_avro_schema_subject, 73 | :value_avro_field_names, :key_avro_field_names, 74 | :value_avro_field_references, :key_avro_field_references, 75 | :mutable?, :immutable?, 76 | to: :class 77 | end 78 | end 79 | end 80 | -------------------------------------------------------------------------------- /lib/avromatic/model/configuration.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Avromatic 4 | module Model 5 | 6 | # This class holds configuration for a model built from Avro schema(s). 7 | class Configuration 8 | 9 | attr_reader :avro_schema, :key_avro_schema, :nested_models, :mutable, 10 | :allow_optional_key_fields, :avro_schema_subject, :key_avro_schema_subject 11 | alias_method :mutable?, :mutable 12 | delegate :schema_store, to: Avromatic 13 | 14 | # Either schema(_name) or value_schema(_name), but not both, must be 15 | # specified. 16 | # 17 | # @param options [Hash] 18 | # @option options [Avro::Schema] :schema 19 | # @option options [String, Symbol] :schema_name 20 | # @option options [String, Symbol] :schema_subject 21 | # @option options [Avro::Schema] :value_schema 22 | # @option options [String, Symbol] :value_schema_name 23 | # @option options [String, Symbol] :value_schema_subject 24 | # @option options [Avro::Schema] :key_schema 25 | # @option options [String, Symbol] :key_schema_name 26 | # @option options [String, Symbol] :key_schema_subject 27 | # @option options [Avromatic::ModelRegistry] :nested_models 28 | # @option options [Boolean] :mutable, default false 29 | # @option options [Boolean] :allow_optional_key_fields, default false 30 | def initialize(**options) 31 | @avro_schema = find_avro_schema(**options) 32 | @avro_schema_subject = options[:schema_subject] || options[:value_schema_subject] 33 | raise ArgumentError.new('value_schema(_name) or schema(_name) must be specified') unless avro_schema 34 | 35 | @key_avro_schema = find_schema_by_option(:key_schema, **options) 36 | @key_avro_schema_subject = options[:key_schema_subject] 37 | @nested_models = options[:nested_models] 38 | @mutable = options.fetch(:mutable, false) 39 | @allow_optional_key_fields = options.fetch(:allow_optional_key_fields, false) 40 | end 41 | 42 | alias_method :value_avro_schema, :avro_schema 43 | alias_method :value_avro_schema_subject, :avro_schema_subject 44 | 45 | def immutable? 46 | !mutable? 47 | end 48 | 49 | private 50 | 51 | def find_avro_schema(**options) 52 | if (options[:value_schema] || options[:value_schema_name]) && 53 | (options[:schema] || options[:schema_name]) 54 | raise ArgumentError.new('Only one of value_schema(_name) and schema(_name) can be specified') 55 | end 56 | 57 | find_schema_by_option(:value_schema, **options) || find_schema_by_option(:schema, **options) 58 | end 59 | 60 | def find_schema_by_option(option_name, **options) 61 | schema_name_option = :"#{option_name}_name" 62 | options[option_name] || 63 | (options[schema_name_option] && schema_store.find(options[schema_name_option])) 64 | end 65 | end 66 | end 67 | end 68 | -------------------------------------------------------------------------------- /lib/avromatic/model/messaging_serialization.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Avromatic 4 | module Model 5 | 6 | # This concern adds support for serialization based on AvroTurf::Messaging. 7 | # This serialization leverages a schema registry to prefix encoded values 8 | # with an id for the schema. 9 | module MessagingSerialization 10 | extend ActiveSupport::Concern 11 | 12 | delegate :avro_messaging, to: :class 13 | private :avro_messaging 14 | 15 | module Encode 16 | def avro_message_value 17 | avro_messaging.encode( 18 | value_attributes_for_avro, 19 | schema_name: value_avro_schema.fullname, 20 | subject: value_avro_schema_subject 21 | ) 22 | end 23 | 24 | def avro_message_key 25 | raise 'Model has no key schema' unless key_avro_schema 26 | 27 | avro_messaging.encode( 28 | key_attributes_for_avro, 29 | schema_name: key_avro_schema.fullname, 30 | subject: key_avro_schema_subject 31 | ) 32 | end 33 | end 34 | include Encode 35 | 36 | # This module provides methods to decode an Avro-encoded value and 37 | # an optional Avro-encoded key as a new model instance. 38 | module Decode 39 | 40 | # If two arguments are specified then the first is interpreted as the 41 | # message key and the second is the message value. If there is only one 42 | # arg then it is used as the message value. 43 | def avro_message_decode(*args) 44 | new(avro_message_attributes(*args)) 45 | end 46 | 47 | def avro_message_attributes(*args) 48 | message_key, message_value = args.size > 1 ? args : [nil, args.first] 49 | key_attributes = message_key && 50 | avro_messaging.decode(message_key, schema_name: key_avro_schema.fullname) 51 | value_attributes = avro_messaging 52 | .decode(message_value, schema_name: avro_schema.fullname) 53 | 54 | value_attributes.merge!(key_attributes) if key_attributes 55 | value_attributes 56 | end 57 | end 58 | 59 | module Registration 60 | def register_schemas! 61 | register_schema(key_avro_schema, subject: key_avro_schema_subject) if key_avro_schema 62 | register_schema(value_avro_schema, subject: value_avro_schema_subject) 63 | nil 64 | end 65 | 66 | private 67 | 68 | def register_schema(schema, subject: nil) 69 | avro_messaging.registry.register(subject || schema.fullname, schema) 70 | end 71 | end 72 | 73 | module ClassMethods 74 | # The messaging object acts as an intermediary talking to the schema 75 | # registry and using returned/specified schemas to decode/encode. 76 | def avro_messaging 77 | Avromatic.messaging 78 | end 79 | 80 | include Decode 81 | include Registration 82 | end 83 | end 84 | end 85 | end 86 | -------------------------------------------------------------------------------- /lib/avromatic.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'avromatic/version' 4 | require 'avro_schema_registry-client' 5 | require 'ice_nine' 6 | require 'ice_nine/core_ext/object' 7 | require 'avromatic/model' 8 | require 'avromatic/model_registry' 9 | require 'avromatic/messaging' 10 | require 'active_support/core_ext/string/inflections' 11 | 12 | module Avromatic 13 | class << self 14 | attr_accessor :schema_registry, :registry_url, :schema_store, :logger, 15 | :messaging, :custom_type_registry, :nested_models, 16 | :use_custom_datum_reader, :use_custom_datum_writer, 17 | :use_schema_fingerprint_lookup, :allow_unknown_attributes 18 | 19 | delegate :register_type, to: :custom_type_registry 20 | end 21 | 22 | self.nested_models = ModelRegistry.new 23 | self.logger = Logger.new($stdout) 24 | self.custom_type_registry = Avromatic::Model::CustomTypeRegistry.new 25 | self.use_custom_datum_reader = true 26 | self.use_custom_datum_writer = true 27 | self.use_schema_fingerprint_lookup = true 28 | self.allow_unknown_attributes = false 29 | 30 | def self.configure 31 | yield self 32 | end 33 | 34 | def self.build_schema_registry 35 | raise 'Avromatic must be configured with a registry_url' unless registry_url 36 | 37 | if use_schema_fingerprint_lookup 38 | AvroSchemaRegistry::CachedClient.new( 39 | AvroSchemaRegistry::Client.new(registry_url, logger: logger) 40 | ) 41 | else 42 | AvroTurf::CachedConfluentSchemaRegistry.new( 43 | AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: logger) 44 | ) 45 | end 46 | end 47 | 48 | def self.build_schema_registry! 49 | self.schema_registry = build_schema_registry 50 | end 51 | 52 | def self.build_messaging 53 | raise 'Avromatic must be configured with a schema_store' unless schema_store 54 | 55 | Avromatic::Messaging.new( 56 | registry: schema_registry || build_schema_registry, 57 | schema_store: schema_store, 58 | logger: logger 59 | ) 60 | end 61 | 62 | def self.build_messaging! 63 | self.messaging = build_messaging 64 | end 65 | 66 | # This method is called as a Rails to_prepare hook after the application 67 | # first initializes during boot-up and prior to each code reloading. 68 | def self.prepare! 69 | nested_models.clear 70 | if schema_store 71 | if schema_store.respond_to?(:clear_schemas) 72 | schema_store.clear_schemas 73 | elsif schema_store.respond_to?(:clear) 74 | schema_store.clear 75 | end 76 | end 77 | 78 | eager_load_models! 79 | end 80 | 81 | def self.eager_load_models=(models) 82 | @eager_load_model_names = Array(models).map { |model| model.is_a?(Class) ? model.name : model }.freeze 83 | end 84 | 85 | def self.eager_load_models 86 | @eager_load_model_names 87 | end 88 | 89 | def self.eager_load_models! 90 | @eager_load_model_names&.each { |model_name| model_name.constantize.register! } 91 | end 92 | private_class_method :eager_load_models! 93 | end 94 | 95 | require 'avromatic/railtie' if defined?(Rails) 96 | -------------------------------------------------------------------------------- /spec/avromatic/model/builder_nested_models_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | describe Avromatic::Model::Builder, 'nested_models' do 4 | let(:schema) do 5 | Avro::Builder.build_schema do 6 | record :rec do 7 | required :sub, :record, type_name: :sub_rec, type_namespace: 'test.bar' do 8 | required :s, :string 9 | end 10 | optional :opt_sub, :sub_rec 11 | end 12 | end 13 | end 14 | let(:schema2) do 15 | Avro::Builder.build_schema do 16 | record :rec2 do 17 | required :same, :record, type_name: :sub_rec, type_namespace: 'test.bar' do 18 | required :s, :string 19 | end 20 | end 21 | end 22 | end 23 | 24 | context "when the nested_models option is not specified" do 25 | let(:model) { described_class.model(schema: schema) } 26 | let(:model2) { described_class.model(schema: schema2) } 27 | 28 | it "registers nested models in the Avromatic registry" do 29 | expect(model.nested_models['test.bar.sub_rec']) 30 | .to equal(Avromatic.nested_models['test.bar.sub_rec']) 31 | end 32 | 33 | it "reuses nested models for multiple fields" do 34 | expect(model.attribute_definitions[:sub].type.record_class) 35 | .to equal(model.attribute_definitions[:opt_sub].type.record_class) 36 | end 37 | 38 | it "reuses nested models for multiple models" do 39 | expect(model.attribute_definitions[:sub].type.record_class) 40 | .to equal(model2.attribute_definitions[:same].type.record_class) 41 | end 42 | end 43 | 44 | context "when the nested_models option is specified" do 45 | let(:registry) { Avromatic::ModelRegistry.new } 46 | let(:model) { described_class.model(schema: schema, nested_models: registry) } 47 | let(:registry2) { Avromatic::ModelRegistry.new } 48 | let(:model2) { described_class.model(schema: schema2, nested_models: registry2) } 49 | 50 | it "uses the specified registry for nested models" do 51 | aggregate_failures do 52 | expect(model.nested_models['test.bar.sub_rec']).to equal(model.attribute_definitions[:sub].type.record_class) 53 | expect(Avromatic.nested_models.registered?('test.bar.sub_rec')).to be(false) 54 | end 55 | end 56 | 57 | it "reuses nested models for multiple fields" do 58 | expect(model.attribute_definitions[:sub].type.record_class) 59 | .to equal(model.attribute_definitions[:opt_sub].type.record_class) 60 | end 61 | 62 | it "does not reuse nested models for models with different registries" do 63 | expect(model.attribute_definitions[:sub].type.record_class) 64 | .not_to equal(model2.attribute_definitions[:same].type.record_class) 65 | end 66 | end 67 | 68 | context "when another instance of the nested model has already been registered" do 69 | let!(:outer_model) { described_class.model(schema: schema) } 70 | 71 | it "raises an error" do 72 | expect do 73 | described_class.model(schema: schema.fields_hash['sub'].type) 74 | end.to raise_error(including('Attempted to replace existing Avromatic model')) 75 | end 76 | end 77 | end 78 | -------------------------------------------------------------------------------- /lib/avromatic/model/types/union_type.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'avromatic/io' 4 | require 'avromatic/model/types/abstract_type' 5 | 6 | module Avromatic 7 | module Model 8 | module Types 9 | class UnionType < AbstractType 10 | attr_reader :member_types, :value_classes, :input_classes 11 | 12 | def initialize(member_types:) 13 | super() 14 | @member_types = member_types 15 | @value_classes = member_types.flat_map(&:value_classes) 16 | @input_classes = member_types.flat_map(&:input_classes).uniq 17 | end 18 | 19 | def name 20 | "union[#{member_types.map(&:name).join(', ')}]" 21 | end 22 | 23 | def coerce(input) 24 | return input if coerced?(input) 25 | 26 | result = nil 27 | if input.is_a?(Avromatic::IO::UnionDatum) 28 | result = member_types[input.member_index].coerce(input.datum) 29 | else 30 | member_types.find do |member_type| 31 | result = safe_coerce(member_type, input) 32 | end 33 | end 34 | 35 | raise ArgumentError.new("Could not coerce '#{input.inspect}' to #{name}") if result.nil? 36 | 37 | result 38 | end 39 | 40 | def coerced?(value) 41 | return false if value.is_a?(Avromatic::IO::UnionDatum) 42 | 43 | value.nil? || member_types.any? do |member_type| 44 | member_type.coerced?(value) 45 | end 46 | end 47 | 48 | def coercible?(input) 49 | return true if value.is_a?(Avromatic::IO::UnionDatum) 50 | 51 | coerced?(input) || member_types.any? do |member_type| 52 | member_type.coercible?(input) 53 | end 54 | end 55 | 56 | def serialize(value, strict) 57 | # Avromatic does not treat the null of an optional field as part of the union 58 | return nil if value.nil? 59 | 60 | member_index = find_index(value) 61 | if member_index.nil? 62 | raise ArgumentError.new("Expected #{value.inspect} to be one of #{value_classes.map(&:name)}") 63 | end 64 | 65 | serialized_value = member_types[member_index].serialize(value, strict) 66 | if !strict && Avromatic.use_custom_datum_writer 67 | serialized_value = Avromatic::IO::UnionDatum.new(member_index, serialized_value) 68 | end 69 | serialized_value 70 | end 71 | 72 | def referenced_model_classes 73 | member_types.flat_map(&:referenced_model_classes).tap(&:uniq!).freeze 74 | end 75 | 76 | private 77 | 78 | def find_index(value) 79 | # TODO: Cache this? 80 | member_types.find_index do |member_type| 81 | member_type.value_classes.any? { |value_class| value.is_a?(value_class) } 82 | end 83 | end 84 | 85 | def safe_coerce(member_type, input) 86 | member_type.coerce(input) if member_type.coercible?(input) 87 | rescue StandardError 88 | nil 89 | end 90 | end 91 | end 92 | end 93 | end 94 | -------------------------------------------------------------------------------- /spec/avromatic_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | describe Avromatic do 4 | it "has a version number" do 5 | expect(Avromatic::VERSION).not_to be_nil 6 | end 7 | 8 | describe ".build_schema_registry" do 9 | context "when the registry_url is unset" do 10 | before { Avromatic.registry_url = nil } 11 | 12 | it "raises an error" do 13 | expect { Avromatic.build_schema_registry } 14 | .to raise_error('Avromatic must be configured with a registry_url') 15 | end 16 | end 17 | 18 | context "when the registry_url is set" do 19 | let(:registry_url) { 'http://registry.example.com' } 20 | 21 | before do 22 | Avromatic.registry_url = registry_url 23 | end 24 | 25 | it "returns an AvroSchemaRegistry::CachedClient", :aggregate_failures do 26 | allow(AvroSchemaRegistry::Client).to receive(:new).and_call_original 27 | expect(Avromatic.build_schema_registry).to be_a(AvroSchemaRegistry::CachedClient) 28 | expect(AvroSchemaRegistry::Client).to have_received(:new) 29 | .with(Avromatic.registry_url, logger: Avromatic.logger) 30 | end 31 | 32 | context "when use_schema_fingerprint_lookup is false" do 33 | before { Avromatic.use_schema_fingerprint_lookup = false } 34 | 35 | it "returns a CachedConfluentSchemaRegistry client", :aggregate_failures do 36 | allow(AvroTurf::ConfluentSchemaRegistry).to receive(:new).and_call_original 37 | schema_registry = Avromatic.build_schema_registry 38 | expect(schema_registry).to be_a(AvroTurf::CachedConfluentSchemaRegistry) 39 | expect(schema_registry).not_to be_a(AvroSchemaRegistry::CachedClient) 40 | expect(AvroTurf::ConfluentSchemaRegistry).to have_received(:new) 41 | .with(Avromatic.registry_url, logger: Avromatic.logger) 42 | end 43 | end 44 | 45 | it "does not cache the schema registry client" do 46 | expect(Avromatic.build_schema_registry).not_to equal(Avromatic.build_schema_registry) 47 | end 48 | end 49 | end 50 | 51 | context "eager loading models" do 52 | before do 53 | stub_const('NestedRecord', Avromatic::Model.model(schema_name: 'test.nested_record')) 54 | stub_const('NestedNestedRecord', Avromatic::Model.model(schema_name: 'test.nested_nested_record')) 55 | described_class.nested_models.clear 56 | end 57 | 58 | describe "#prepare!" do 59 | before do 60 | stub_const('ValueModel', Avromatic::Model.model(schema_name: 'test.value')) 61 | allow(Avromatic.schema_store).to receive(:clear) 62 | end 63 | 64 | it "clears the registry" do 65 | described_class.prepare! 66 | expect(described_class.nested_models.registered?('test.value')).to be(false) 67 | end 68 | 69 | it "clears the schema store" do 70 | described_class.prepare! 71 | expect(Avromatic.schema_store).to have_received(:clear) 72 | end 73 | 74 | it "registers models" do 75 | described_class.eager_load_models = ['NestedRecord'] 76 | expect(described_class.eager_load_models).to eq(['NestedRecord']) 77 | described_class.prepare! 78 | expect(described_class.nested_models.registered?('test.value')).to be(false) 79 | end 80 | 81 | it "registers nested models" do 82 | described_class.eager_load_models = ['NestedNestedRecord'] 83 | expect(described_class.eager_load_models).to eq(['NestedNestedRecord']) 84 | described_class.prepare! 85 | expect(described_class.nested_models.registered?('test.__nested_nested_record_sub_record')).to be(true) 86 | expect(described_class.nested_models.registered?('test.__nested_nested_record_sub_subsub_record')).to be(true) 87 | end 88 | end 89 | end 90 | end 91 | -------------------------------------------------------------------------------- /lib/avromatic/model/message_decoder.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Avromatic 4 | module Model 5 | 6 | # This class is used to decode messages encoded using Avro to their 7 | # corresponding models. 8 | class MessageDecoder 9 | MAGIC_BYTE = [0].pack('C').freeze 10 | 11 | class UnexpectedKeyError < StandardError 12 | attr_reader :key_schema_name, :value_schema_name 13 | 14 | def initialize(key_schema_name, value_schema_name) 15 | super("Unexpected schemas #{[key_schema_name, value_schema_name]}") 16 | @key_schema_name = key_schema_name 17 | @value_schema_name = value_schema_name 18 | end 19 | end 20 | 21 | class MagicByteError < StandardError 22 | def initialize(magic_byte) 23 | super("Expected data to begin with a magic byte, got '#{magic_byte}'") 24 | end 25 | end 26 | 27 | class DuplicateKeyError < StandardError 28 | def initialize(*models) 29 | super("Multiple models #{models} have the same key "\ 30 | "'#{Avromatic::Model::MessageDecoder.model_key(models.first)}'") 31 | end 32 | end 33 | 34 | def self.model_key(model) 35 | [model.key_avro_schema && model.key_avro_schema.fullname, 36 | model.value_avro_schema.fullname] 37 | end 38 | 39 | delegate :model_key, to: :class 40 | 41 | # @param *models [generated models] Models to register for decoding. 42 | # @param schema_registry [AvroTurf::ConfluentSchemaRegistry] Optional schema 43 | # registry client. 44 | # @param registry_url [String] Optional URL for schema registry server. 45 | def initialize(*models, schema_registry: nil, registry_url: nil) 46 | @model_map = build_model_map(models) 47 | @schema_names_by_id = {} 48 | @schema_registry = schema_registry || 49 | Avromatic.schema_registry || 50 | (registry_url && AvroTurf::ConfluentSchemaRegistry.new(registry_url, logger: Avromatic.logger)) || 51 | Avromatic.build_schema_registry 52 | end 53 | 54 | # @return [Avromatic model] 55 | def decode(*args) 56 | model, message_key, message_value = extract_decode_args(*args) 57 | model.avro_message_decode(message_key, message_value) 58 | end 59 | 60 | # @return [Hash] 61 | def decode_hash(*args) 62 | model, message_key, message_value = extract_decode_args(*args) 63 | model.avro_message_attributes(message_key, message_value) 64 | end 65 | 66 | # @return [Avromatic model class] 67 | def model(*args) 68 | extract_decode_args(*args).first 69 | end 70 | 71 | private 72 | 73 | attr_reader :schema_names_by_id, :model_map, :schema_registry 74 | 75 | def extract_key_and_value(*args) 76 | args.size > 1 ? args.take(2) : [nil, args.first] 77 | end 78 | 79 | def model_key_for_message(message_key, message_value) 80 | value_schema_name = schema_name_for_data(message_value) 81 | key_schema_name = schema_name_for_data(message_key) if message_key 82 | [key_schema_name, value_schema_name] 83 | end 84 | 85 | # If two arguments are specified then the first is interpreted as the 86 | # message key and the second is the message value. If there is only one 87 | # arg then it is used as the message value. 88 | def extract_decode_args(*args) 89 | message_key, message_value = extract_key_and_value(*args) 90 | model_key = model_key_for_message(message_key, message_value) 91 | raise UnexpectedKeyError.new(*model_key) unless model_map.key?(model_key) 92 | 93 | [model_map[model_key], message_key, message_value] 94 | end 95 | 96 | def schema_name_for_data(data) 97 | validate_magic_byte!(data) 98 | schema_id = extract_schema_id(data) 99 | lookup_schema_name(schema_id) 100 | end 101 | 102 | def lookup_schema_name(schema_id) 103 | schema_names_by_id.fetch(schema_id) do 104 | schema = Avro::Schema.parse(schema_registry.fetch(schema_id)) 105 | schema_names_by_id[schema_id] = schema.fullname 106 | end 107 | end 108 | 109 | def extract_schema_id(data) 110 | data[1..4].unpack1('N') 111 | end 112 | 113 | def validate_magic_byte!(data) 114 | first_byte = data[0] 115 | raise MagicByteError.new(first_byte) if first_byte != MAGIC_BYTE 116 | end 117 | 118 | def build_model_map(models) 119 | models.each_with_object(Hash.new) do |model, map| 120 | key = model_key(model) 121 | raise DuplicateKeyError.new(map[key], model) if map.key?(key) && !model.equal?(map[key]) 122 | 123 | map[key] = model 124 | end 125 | end 126 | end 127 | end 128 | end 129 | -------------------------------------------------------------------------------- /spec/avromatic/io/datum_reader_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | describe Avromatic::IO::DatumReader do 4 | let(:test_class) do 5 | Avromatic::Model.model(schema_name: schema_name) 6 | end 7 | let(:instance) { test_class.new(values) } 8 | let(:avro_message_value) { instance.avro_message_value } 9 | let(:attributes) { test_class.avro_message_attributes(avro_message_value) } 10 | 11 | describe "#read" do 12 | let(:decoder) { instance_double(Avro::IO::BinaryDecoder) } 13 | 14 | context "errors" do 15 | it "raises an error if schemas do not match" do 16 | reader = described_class.new(Avro::Schema.real_parse('int'), Avro::Schema.real_parse('string')) 17 | expect { reader.read(decoder) }.to raise_error(Avro::IO::SchemaMatchException) 18 | end 19 | 20 | context "reader's schema is a union" do 21 | let(:reader_schema) do 22 | Avro::Schema.parse([:null, :string].to_json) 23 | end 24 | 25 | it "raises an error if the writer's schema does not match any reader type" do 26 | reader = described_class.new(Avro::Schema.real_parse('int'), reader_schema) 27 | expect { reader.read(decoder) }.to raise_error(Avro::IO::SchemaMatchException) 28 | end 29 | end 30 | 31 | it "raises an error when the writer's schema is unknown" do 32 | writer_schema = Avro::Schema.new('foobar') 33 | reader = described_class.new(writer_schema, Avro::Schema.real_parse('int')) 34 | expect { reader.read(decoder) }.to raise_error(Avro::AvroError) 35 | end 36 | end 37 | end 38 | 39 | context "primitive types" do 40 | let(:schema_name) { 'test.primitive_types' } 41 | let(:values) do 42 | { 43 | s: 'foo', 44 | b: '123', 45 | tf: true, 46 | i: rand(10), 47 | l: 123456789, 48 | f: 0.5, 49 | d: 1.0 / 3.0, 50 | n: nil, 51 | fx: '1234567', 52 | e: 'A' 53 | } 54 | end 55 | 56 | # This test case is primarily to provide coverage for DatumReader 57 | it "reads all primitive types" do 58 | expect(attributes).to eq(values.stringify_keys) 59 | end 60 | end 61 | 62 | context "a record with a union" do 63 | let(:schema_name) { 'test.real_union' } 64 | let(:values) do 65 | { 66 | header: 'has bar', 67 | message: { bar_message: "I'm a bar" } 68 | } 69 | end 70 | 71 | it "returns a UnionDatum" do 72 | union_datum = attributes['message'] 73 | expect(union_datum).to be_a_kind_of(Avromatic::IO::UnionDatum) 74 | expect(union_datum.member_index).to eq(1) 75 | expect(union_datum.datum).to eq(values[:message].stringify_keys) 76 | end 77 | 78 | it "can decode a message" do 79 | expect(test_class.avro_message_decode(avro_message_value)).to eq(instance) 80 | end 81 | 82 | context "wtih a false value" do 83 | let(:values) do 84 | { header: 'has bar', message: false } 85 | end 86 | 87 | it "can decode a message" do 88 | expect(test_class.avro_message_decode(avro_message_value)).to eq(instance) 89 | end 90 | end 91 | 92 | context "a record with an optional union" do 93 | let(:schema_name) { 'test.optional_union' } 94 | 95 | it "returns a UnionDatum" do 96 | union_datum = attributes['message'] 97 | expect(union_datum).to be_a_kind_of(Avromatic::IO::UnionDatum) 98 | expect(union_datum.member_index).to eq(1) 99 | expect(union_datum.datum).to eq(values[:message].stringify_keys) 100 | end 101 | 102 | it "can decode a message" do 103 | expect(test_class.avro_message_decode(avro_message_value)).to eq(instance) 104 | end 105 | end 106 | 107 | context "with an optional field" do 108 | let(:schema_name) { 'test.with_union' } 109 | let(:values) { { s: 'foo' } } 110 | 111 | it "does not return a UnionDatum" do 112 | expect(attributes['s']).not_to be_a_kind_of(Avromatic::IO::UnionDatum) 113 | end 114 | 115 | it "can decode the message" do 116 | expect(attributes).to eq(values.stringify_keys) 117 | end 118 | end 119 | 120 | context "with null in a union" do 121 | let(:schema_name) { 'test.null_in_union' } 122 | let(:values) do 123 | { 124 | values: [ 125 | { i: 123 }, 126 | nil, 127 | { s: 'abc' } 128 | ] 129 | } 130 | end 131 | 132 | it "does not support a null in the middle of a union" do 133 | expect do 134 | Avromatic::Model.model(schema_name: schema_name) 135 | end.to raise_error('a null type in a union must be the first member') 136 | end 137 | end 138 | 139 | context "when use_custom_datum_reader is false" do 140 | before do 141 | allow(Avromatic).to receive(:use_custom_datum_reader).and_return(false) 142 | end 143 | 144 | it "does not return a UnionDatum" do 145 | expect(attributes['message']).not_to be_a_kind_of(Avromatic::IO::UnionDatum) 146 | end 147 | end 148 | end 149 | end 150 | -------------------------------------------------------------------------------- /spec/avromatic/model_registry_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | describe Avromatic::ModelRegistry do 4 | let(:model) { Avromatic::Model.model(schema_name: 'test.nested_record') } 5 | let(:instance) { described_class.new } 6 | 7 | describe "#registered?" do 8 | context "for a model that has not been registered" do 9 | it "returns false" do 10 | expect(instance.registered?('test.nested_record')).to be(false) 11 | expect(instance.registered?(model)).to be(false) 12 | end 13 | end 14 | 15 | context "for model that has been registered" do 16 | before { instance.register(model) } 17 | 18 | it "returns true" do 19 | expect(instance.registered?('test.nested_record')).to be(true) 20 | expect(instance.registered?(model)).to be(true) 21 | end 22 | end 23 | end 24 | 25 | describe "#ensure_registered_model" do 26 | context "when the model is already registered" do 27 | before do 28 | instance.register(model) 29 | end 30 | 31 | it "does not raise an error" do 32 | expect { instance.ensure_registered_model(model) }.not_to raise_error 33 | end 34 | 35 | context "when a different copy of the model is registered" do 36 | 37 | it "raises an error" do 38 | expect do 39 | instance.ensure_registered_model(model.dup) 40 | end.to raise_error(including('Attempted to replace existing Avromatic model')) 41 | end 42 | end 43 | end 44 | 45 | context "when the model is not already registered" do 46 | before do 47 | instance.clear 48 | instance.ensure_registered_model(model) 49 | end 50 | 51 | it "registers the model" do 52 | expect(instance.registered?('test.nested_record')).to be(true) 53 | end 54 | end 55 | end 56 | 57 | describe "#model_fullname" do 58 | let(:instance) { described_class.new(remove_namespace_prefix: 'test') } 59 | 60 | it "returns the value schema fullname with the prefix removed" do 61 | expect(instance.model_fullname(model)).to eq('nested_record') 62 | end 63 | end 64 | 65 | context "without a namespace prefix to remove" do 66 | it "stores a model by its fullname" do 67 | instance.register(model) 68 | expect(instance['test.nested_record']).to equal(model) 69 | end 70 | 71 | context "with a previously registered model" do 72 | before { instance.register(model) } 73 | 74 | it "raises an error" do 75 | expect do 76 | instance.register(model) 77 | end.to raise_error("'test.nested_record' has already been registered") 78 | end 79 | end 80 | 81 | context "for a model with a key schema" do 82 | let(:model) do 83 | Avromatic::Model.model(key_schema_name: 'test.encode_key', 84 | schema_name: 'test.encode_value') 85 | end 86 | 87 | it "raises an error" do 88 | expect { instance.register(model) } 89 | .to raise_error('models with a key schema are not supported') 90 | end 91 | end 92 | 93 | context "when a model has not been registered" do 94 | it "raises an error" do 95 | expect { instance['test.fnord'] } 96 | .to raise_error(/key not found: "test.fnord"/) 97 | end 98 | end 99 | end 100 | 101 | context "with a namespace prefix to remove" do 102 | let(:multilevel_model) do 103 | schema = Avro::Builder.build_schema do 104 | record :rec, namespace: 'test.sub' do 105 | required :i, :int 106 | end 107 | end 108 | Avromatic::Model.model(schema: schema) 109 | end 110 | 111 | context "with a String prefix" do 112 | let(:instance) { described_class.new(remove_namespace_prefix: 'test') } 113 | 114 | it "stores a model by its fullname with prefix removed" do 115 | instance.register(model) 116 | expect(instance['nested_record']).to equal(model) 117 | end 118 | 119 | it "only removes the matching namespace prefix" do 120 | instance.register(multilevel_model) 121 | expect(instance['sub.rec']).to equal(multilevel_model) 122 | end 123 | 124 | context "for a model that does not match the prefix" do 125 | let(:instance) { described_class.new(remove_namespace_prefix: 'test.sub') } 126 | 127 | it "stores the model by its fullname" do 128 | instance.register(model) 129 | expect(instance['test.nested_record']).to equal(model) 130 | end 131 | end 132 | end 133 | 134 | context "with a Regexp prefix" do 135 | let(:instance) { described_class.new(remove_namespace_prefix: /(\w+\.)+/) } 136 | 137 | it "stores a model by its fullname with prefix removed" do 138 | instance.register(model) 139 | expect(instance['nested_record']).to equal(model) 140 | end 141 | 142 | it "only removes the matching namespace prefix" do 143 | instance.register(multilevel_model) 144 | expect(instance['rec']).to equal(multilevel_model) 145 | end 146 | end 147 | 148 | context "with an invalid prefix value" do 149 | let(:instance) { described_class.new(remove_namespace_prefix: 1) } 150 | 151 | it "raises an error" do 152 | expect { instance.register(model) } 153 | .to raise_error('unsupported `remove_namespace_prefix` value: 1') 154 | end 155 | end 156 | end 157 | end 158 | -------------------------------------------------------------------------------- /lib/avromatic/model/types/type_factory.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'avromatic/model/types/array_type' 4 | require 'avromatic/model/types/boolean_type' 5 | require 'avromatic/model/types/custom_type' 6 | require 'avromatic/model/types/date_type' 7 | require 'avromatic/model/types/decimal_type' 8 | require 'avromatic/model/types/enum_type' 9 | require 'avromatic/model/types/fixed_type' 10 | require 'avromatic/model/types/float_type' 11 | require 'avromatic/model/types/integer_type' 12 | require 'avromatic/model/types/map_type' 13 | require 'avromatic/model/types/null_type' 14 | require 'avromatic/model/types/record_type' 15 | require 'avromatic/model/types/string_type' 16 | require 'avromatic/model/types/timestamp_micros_type' 17 | require 'avromatic/model/types/timestamp_millis_type' 18 | require 'avromatic/model/types/union_type' 19 | 20 | module Avromatic 21 | module Model 22 | module Types 23 | module TypeFactory 24 | extend self 25 | 26 | SINGLETON_TYPES = { 27 | 'date' => Avromatic::Model::Types::DateType.new, 28 | 'timestamp-micros' => Avromatic::Model::Types::TimestampMicrosType.new, 29 | 'timestamp-millis' => Avromatic::Model::Types::TimestampMillisType.new, 30 | 'string' => Avromatic::Model::Types::StringType.new, 31 | 'bytes' => Avromatic::Model::Types::StringType.new, 32 | 'boolean' => Avromatic::Model::Types::BooleanType.new, 33 | 'int' => Avromatic::Model::Types::IntegerType.new, 34 | 'long' => Avromatic::Model::Types::IntegerType.new, 35 | 'float' => Avromatic::Model::Types::FloatType.new, 36 | 'double' => Avromatic::Model::Types::FloatType.new, 37 | 'null' => Avromatic::Model::Types::NullType.new 38 | }.deep_freeze 39 | 40 | def create(schema:, nested_models:, use_custom_types: true) 41 | if use_custom_types && Avromatic.custom_type_registry.registered?(schema) 42 | custom_type_configuration = Avromatic.custom_type_registry.fetch(schema) 43 | default_type = create( 44 | schema: schema, 45 | nested_models: nested_models, 46 | use_custom_types: false 47 | ) 48 | Avromatic::Model::Types::CustomType.new( 49 | custom_type_configuration: custom_type_configuration, 50 | default_type: default_type 51 | ) 52 | elsif schema.respond_to?(:logical_type) && SINGLETON_TYPES.include?(schema.logical_type) 53 | SINGLETON_TYPES.fetch(schema.logical_type) 54 | elsif schema.respond_to?(:logical_type) && schema.logical_type == 'decimal' 55 | Avromatic::Model::Types::DecimalType.new(precision: schema.precision, scale: schema.scale || 0) 56 | elsif SINGLETON_TYPES.include?(schema.type) 57 | SINGLETON_TYPES.fetch(schema.type) 58 | else 59 | case schema.type_sym 60 | when :fixed 61 | Avromatic::Model::Types::FixedType.new(schema.size) 62 | when :enum 63 | Avromatic::Model::Types::EnumType.new(schema.symbols) 64 | when :array 65 | value_type = create(schema: schema.items, nested_models: nested_models, 66 | use_custom_types: use_custom_types) 67 | Avromatic::Model::Types::ArrayType.new(value_type: value_type) 68 | when :map 69 | value_type = create(schema: schema.values, nested_models: nested_models, 70 | use_custom_types: use_custom_types) 71 | Avromatic::Model::Types::MapType.new( 72 | key_type: Avromatic::Model::Types::StringType.new, 73 | value_type: value_type 74 | ) 75 | when :union 76 | null_index = schema.schemas.index { |member_schema| member_schema.type_sym == :null } 77 | raise 'a null type in a union must be the first member' if null_index && null_index > 0 78 | 79 | member_schemas = schema.schemas.reject { |member_schema| member_schema.type_sym == :null } 80 | if member_schemas.size == 1 81 | create(schema: member_schemas.first, nested_models: nested_models) 82 | else 83 | member_types = member_schemas.map do |member_schema| 84 | create(schema: member_schema, nested_models: nested_models, use_custom_types: use_custom_types) 85 | end 86 | Avromatic::Model::Types::UnionType.new(member_types: member_types) 87 | end 88 | when :record 89 | record_class = build_nested_model(schema: schema, nested_models: nested_models) 90 | Avromatic::Model::Types::RecordType.new(record_class: record_class) 91 | else 92 | raise ArgumentError.new("Unsupported type #{schema.type_sym}") 93 | end 94 | end 95 | end 96 | 97 | private 98 | 99 | def build_nested_model(schema:, nested_models:) 100 | fullname = nested_models.remove_prefix(schema.fullname) 101 | 102 | if nested_models.registered?(fullname) 103 | nested_model = nested_models[fullname] 104 | unless schema_fingerprint(schema) == schema_fingerprint(nested_model.avro_schema) 105 | raise "The #{nested_model.name} model is already registered with an incompatible version of the " \ 106 | "#{schema.fullname} schema" 107 | end 108 | 109 | nested_model 110 | else 111 | Avromatic::Model.model(schema: schema, nested_models: nested_models) 112 | end 113 | end 114 | 115 | def schema_fingerprint(schema) 116 | if schema.respond_to?(:sha256_resolution_fingerprint) 117 | schema.sha256_resolution_fingerprint 118 | else 119 | schema.sha256_fingerprint 120 | end 121 | end 122 | end 123 | end 124 | end 125 | end 126 | -------------------------------------------------------------------------------- /spec/avromatic/model/message_decoder_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | describe Avromatic::Model::MessageDecoder do 6 | let(:instance) { described_class.new(*models) } 7 | let(:model1) do 8 | Avromatic::Model.model( 9 | value_schema_name: 'test.encode_value', 10 | key_schema_name: 'test.encode_key' 11 | ) 12 | end 13 | let(:model2) do 14 | Avromatic::Model.model(value_schema_name: 'test.value') 15 | end 16 | 17 | describe "#initialize" do 18 | context "when multiple models use the same schema" do 19 | let(:model3) do 20 | Avromatic::Model.model( 21 | value_schema_name: 'test.encode_value', 22 | key_schema_name: 'test.encode_key' 23 | ) 24 | end 25 | let(:models) { [model1, model3] } 26 | 27 | it "raises an error" do 28 | expect { instance } 29 | .to raise_error(described_class::DuplicateKeyError, 30 | /Multiple models \[.*\] have the same key .*/) 31 | end 32 | end 33 | 34 | context "when the same model is used multiple times" do 35 | let(:models) { Array.new(2) { model1 } } 36 | 37 | it "does not raise an error" do 38 | expect { instance }.not_to raise_error 39 | end 40 | end 41 | end 42 | 43 | shared_examples_for "decoding failure cases" do |method_name| 44 | context "when the message_value does not begin with the magic byte" do 45 | it "raises an error" do 46 | expect do 47 | instance.send(method_name, 'X') 48 | end.to raise_error(described_class::MagicByteError, 49 | "Expected data to begin with a magic byte, got 'X'") 50 | end 51 | end 52 | 53 | context "when the schema name is not known by the decoder" do 54 | let(:unknown_model) do 55 | Avromatic::Model.model( 56 | value_schema_name: 'test.defaults', 57 | key_schema_name: 'test.encode_key' 58 | ) 59 | end 60 | let(:key_value) do 61 | unknown_model.new(id: 0).avro_message_key 62 | end 63 | let(:message_value) do 64 | unknown_model.new(id: 0).avro_message_value 65 | end 66 | 67 | it "raises an UnexpectedKeyError when the unknown model only has a value schema" do 68 | expect do 69 | instance.send(method_name, message_value) 70 | end.to raise_error do |error| 71 | expect(error).to be_a(described_class::UnexpectedKeyError) 72 | expect(error.message).to eq('Unexpected schemas [nil, "test.defaults"]') 73 | expect(error.key_schema_name).to be_nil 74 | expect(error.value_schema_name).to eq('test.defaults') 75 | end 76 | end 77 | 78 | it "raises an UnexpectedKeyError when the unknown model has a key schema and a value schema" do 79 | expect do 80 | instance.send(method_name, key_value, message_value) 81 | end.to raise_error do |error| 82 | expect(error).to be_a(described_class::UnexpectedKeyError) 83 | expect(error.message).to eq('Unexpected schemas ["test.encode_key", "test.defaults"]') 84 | expect(error.key_schema_name).to eq('test.encode_key') 85 | expect(error.value_schema_name).to eq('test.defaults') 86 | end 87 | end 88 | end 89 | end 90 | 91 | describe "#model" do 92 | let(:models) { [model1, model2] } 93 | let(:model1_instance) { model1.new(str1: 'A', str2: 'B', id: 99) } 94 | let(:model1_value) do 95 | model1_instance.avro_message_value 96 | end 97 | let(:model1_key) do 98 | model1_instance.avro_message_key 99 | end 100 | 101 | it "returns the associated model for a message" do 102 | expect(instance.model(model1_key, model1_value)).to equal(model1) 103 | end 104 | 105 | it_behaves_like "decoding failure cases", :model 106 | end 107 | 108 | describe "#decode" do 109 | let(:models) { [model1, model2] } 110 | let(:model1_instance) { model1.new(str1: 'A', str2: 'B', id: 99) } 111 | let(:model1_value) do 112 | model1_instance.avro_message_value 113 | end 114 | let(:model1_key) do 115 | model1_instance.avro_message_key 116 | end 117 | let(:model2_value) { model2.new(id: 100, action: :CREATE).avro_message_value } 118 | 119 | it "decodes message value and key pairs to registered models" do 120 | expect(instance.decode(model1_key, model1_value)).to be_a(model1) 121 | expect(instance.decode(model2_value)).to be_a(model2) 122 | end 123 | 124 | it_behaves_like "decoding failure cases", :decode 125 | 126 | context "when the decoder is initialized with a schema registry" do 127 | let(:schema_registry) { Avromatic.build_schema_registry } 128 | let(:instance) { described_class.new(*models, schema_registry: schema_registry) } 129 | 130 | before do 131 | allow(schema_registry).to receive(:fetch).and_call_original 132 | instance.decode(model1_key, model1_value) 133 | end 134 | 135 | it "calls find on the provided schema registry" do 136 | expect(schema_registry).to have_received(:fetch).at_least(1).times 137 | end 138 | end 139 | end 140 | 141 | describe "#decode_hash" do 142 | let(:models) { [model1, model2] } 143 | let(:model1_attributes) { { str1: 'A', str2: 'B', id: 99 } } 144 | let(:model1_instance) { model1.new(model1_attributes) } 145 | let(:model1_value) do 146 | model1_instance.avro_message_value 147 | end 148 | let(:model1_key) do 149 | model1_instance.avro_message_key 150 | end 151 | let(:model2_attributes) { { id: 100, action: 'CREATE' } } 152 | let(:model2_value) { model2.new(model2_attributes).avro_message_value } 153 | 154 | it "decodes message value and key pairs to registered models" do 155 | expect(instance.decode_hash(model1_key, model1_value)).to eq(model1_attributes.stringify_keys) 156 | expect(instance.decode_hash(model2_value)).to eq(model2_attributes.stringify_keys) 157 | end 158 | 159 | it_behaves_like "decoding failure cases", :decode_hash 160 | end 161 | end 162 | -------------------------------------------------------------------------------- /lib/avromatic/model/raw_serialization.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'active_support/deprecation' 4 | 5 | module Avromatic 6 | module Model 7 | 8 | # This module provides serialization support for encoding directly to Avro 9 | # without dependency on a schema registry. 10 | module RawSerialization 11 | extend ActiveSupport::Concern 12 | 13 | module Encode 14 | extend ActiveSupport::Concern 15 | 16 | UNSPECIFIED = Object.new 17 | 18 | delegate :datum_writer, :datum_reader, to: :class 19 | private :datum_writer, :datum_reader 20 | 21 | def avro_raw_value(validate: UNSPECIFIED) 22 | unless validate == UNSPECIFIED 23 | ActiveSupport::Deprecation.warn("The 'validate' argument to #{__method__} is deprecated.") 24 | end 25 | 26 | if self.class.recursively_immutable? 27 | @avro_raw_value ||= avro_raw_encode(value_attributes_for_avro, :value) 28 | else 29 | avro_raw_encode(value_attributes_for_avro, :value) 30 | end 31 | end 32 | 33 | def avro_raw_key(validate: UNSPECIFIED) 34 | unless validate == UNSPECIFIED 35 | ActiveSupport::Deprecation.warn("The 'validate' argument to #{__method__} is deprecated.") 36 | end 37 | 38 | raise 'Model has no key schema' unless key_avro_schema 39 | 40 | avro_raw_encode(key_attributes_for_avro, :key) 41 | end 42 | 43 | def value_attributes_for_avro(validate: UNSPECIFIED) 44 | unless validate == UNSPECIFIED 45 | ActiveSupport::Deprecation.warn("The 'validate' argument to #{__method__} is deprecated.") 46 | end 47 | 48 | if self.class.recursively_immutable? 49 | @value_attributes_for_avro ||= avro_hash(value_avro_field_references) 50 | else 51 | avro_hash(value_avro_field_references) 52 | end 53 | end 54 | 55 | def key_attributes_for_avro(validate: UNSPECIFIED) 56 | unless validate == UNSPECIFIED 57 | ActiveSupport::Deprecation.warn("The 'validate' argument to #{__method__} is deprecated.") 58 | end 59 | 60 | avro_hash(key_avro_field_references) 61 | end 62 | 63 | def avro_value_datum(validate: UNSPECIFIED) 64 | unless validate == UNSPECIFIED 65 | ActiveSupport::Deprecation.warn("The 'validate' argument to #{__method__} is deprecated.") 66 | end 67 | 68 | if self.class.recursively_immutable? 69 | @avro_value_datum ||= avro_hash(value_avro_field_references, strict: true) 70 | else 71 | avro_hash(value_avro_field_references, strict: true) 72 | end 73 | end 74 | 75 | def avro_key_datum(validate: UNSPECIFIED) 76 | unless validate == UNSPECIFIED 77 | ActiveSupport::Deprecation.warn("The 'validate' argument to #{__method__} is deprecated.") 78 | end 79 | 80 | avro_hash(key_avro_field_references, strict: true) 81 | end 82 | 83 | private 84 | 85 | def avro_hash(field_references, strict: false) 86 | field_references.each_with_object(Hash.new) do |field_reference, result| 87 | attribute_definition = self.class.attribute_definitions[field_reference.name_sym] 88 | value = _attributes[field_reference.name_sym] 89 | 90 | if value.nil? && !attribute_definition.nullable? 91 | # We're missing a required attribute so perform an explicit validation to generate 92 | # a more complete error message 93 | avro_validate! 94 | elsif _attributes.include?(field_reference.name_sym) 95 | begin 96 | result[field_reference.name] = attribute_definition.serialize(value, strict) 97 | rescue Avromatic::Model::ValidationError 98 | # Perform an explicit validation to generate a more complete error message 99 | avro_validate! 100 | # We should never get here but just in case... 101 | raise 102 | end 103 | end 104 | end 105 | end 106 | 107 | def avro_raw_encode(data, key_or_value = :value) 108 | stream = StringIO.new 109 | encoder = Avro::IO::BinaryEncoder.new(stream) 110 | datum_writer[key_or_value].write(data, encoder) 111 | stream.string 112 | end 113 | end 114 | include Encode 115 | 116 | module Decode 117 | 118 | # Create a new instance based on an encoded value and optional encoded key. 119 | # If supplied then the key_schema and value_schema are used as the writer's 120 | # schema for the corresponding value. The model's schemas are always used 121 | # as the reader's schemas. 122 | def avro_raw_decode(value:, key: nil, key_schema: nil, value_schema: nil) 123 | key_attributes = key && decode_avro_datum(key, key_schema, :key) 124 | value_attributes = decode_avro_datum(value, value_schema, :value) 125 | value_attributes.merge!(key_attributes) if key_attributes 126 | new(value_attributes) 127 | end 128 | 129 | private 130 | 131 | def decode_avro_datum(data, schema = nil, key_or_value = :value) 132 | stream = StringIO.new(data) 133 | decoder = Avro::IO::BinaryDecoder.new(stream) 134 | reader = schema ? custom_datum_reader(schema, key_or_value) : datum_reader[key_or_value] 135 | reader.read(decoder) 136 | end 137 | 138 | def custom_datum_reader(schema, key_or_value) 139 | datum_reader_class.new(schema, send("#{key_or_value}_avro_schema")) 140 | end 141 | end 142 | 143 | module ClassMethods 144 | def datum_reader_class 145 | Avromatic.use_custom_datum_reader ? Avromatic::IO::DatumReader : Avro::IO::DatumReader 146 | end 147 | 148 | def datum_writer_class 149 | Avromatic.use_custom_datum_writer ? Avromatic::IO::DatumWriter : Avro::IO::DatumWriter 150 | end 151 | 152 | def datum_writer 153 | @datum_writer ||= begin 154 | hash = { value: datum_writer_class.new(value_avro_schema) } 155 | hash[:key] = datum_writer_class.new(key_avro_schema) if key_avro_schema 156 | hash 157 | end 158 | end 159 | 160 | def datum_reader 161 | @datum_reader ||= begin 162 | hash = { value: datum_reader_class.new(value_avro_schema) } 163 | hash[:key] = datum_reader_class.new(key_avro_schema) if key_avro_schema 164 | hash 165 | end 166 | end 167 | 168 | include Decode 169 | end 170 | end 171 | end 172 | end 173 | -------------------------------------------------------------------------------- /spec/avro/schema/test/nested_nested_record.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "nested_nested_record", 4 | "namespace": "test", 5 | "fields": [ 6 | { 7 | "name": "n", 8 | "type": "null" 9 | }, 10 | { 11 | "name": "b", 12 | "type": "boolean" 13 | }, 14 | { 15 | "name": "i", 16 | "type": "int" 17 | }, 18 | { 19 | "name": "l", 20 | "type": "long" 21 | }, 22 | { 23 | "name": "f", 24 | "type": "float" 25 | }, 26 | { 27 | "name": "d", 28 | "type": "double" 29 | }, 30 | { 31 | "name": "bs", 32 | "type": "bytes" 33 | }, 34 | { 35 | "name": "str", 36 | "type": "string" 37 | }, 38 | { 39 | "name": "date", 40 | "type": { 41 | "type": "int", 42 | "logicalType": "date" 43 | } 44 | }, 45 | { 46 | "name": "ts_msec", 47 | "type": { 48 | "type": "long", 49 | "logicalType": "timestamp-millis" 50 | } 51 | }, 52 | { 53 | "name": "ts_usec", 54 | "type": { 55 | "type": "long", 56 | "logicalType": "timestamp-micros" 57 | } 58 | }, 59 | { 60 | "name": "unknown", 61 | "type": { 62 | "type": "int", 63 | "logicalType": "foobar" 64 | } 65 | }, 66 | { 67 | "name": "e", 68 | "type": { 69 | "name": "__nested_nested_record_e_enum", 70 | "type": "enum", 71 | "namespace": "test", 72 | "symbols": [ 73 | "A", 74 | "B" 75 | ] 76 | } 77 | }, 78 | { 79 | "name": "a", 80 | "type": { 81 | "type": "array", 82 | "items": "int" 83 | } 84 | }, 85 | { 86 | "name": "m", 87 | "type": { 88 | "type": "map", 89 | "values": "int" 90 | } 91 | }, 92 | { 93 | "name": "u", 94 | "type": [ 95 | "string", 96 | "int" 97 | ] 98 | }, 99 | { 100 | "name": "fx", 101 | "type": { 102 | "name": "__nested_nested_record_fx_fixed", 103 | "type": "fixed", 104 | "namespace": "test", 105 | "size": 2 106 | } 107 | }, 108 | { 109 | "name": "sub", 110 | "type": { 111 | "type": "record", 112 | "name": "__nested_nested_record_sub_record", 113 | "namespace": "test", 114 | "fields": [ 115 | { 116 | "name": "n", 117 | "type": "null" 118 | }, 119 | { 120 | "name": "b", 121 | "type": "boolean" 122 | }, 123 | { 124 | "name": "i", 125 | "type": "int" 126 | }, 127 | { 128 | "name": "l", 129 | "type": "long" 130 | }, 131 | { 132 | "name": "f", 133 | "type": "float" 134 | }, 135 | { 136 | "name": "d", 137 | "type": "double" 138 | }, 139 | { 140 | "name": "bs", 141 | "type": "bytes" 142 | }, 143 | { 144 | "name": "str", 145 | "type": "string" 146 | }, 147 | { 148 | "name": "date", 149 | "type": { 150 | "type": "int", 151 | "logicalType": "date" 152 | } 153 | }, 154 | { 155 | "name": "ts_msec", 156 | "type": { 157 | "type": "long", 158 | "logicalType": "timestamp-millis" 159 | } 160 | }, 161 | { 162 | "name": "ts_usec", 163 | "type": { 164 | "type": "long", 165 | "logicalType": "timestamp-micros" 166 | } 167 | }, 168 | { 169 | "name": "unknown", 170 | "type": { 171 | "type": "int", 172 | "logicalType": "foobar" 173 | } 174 | }, 175 | { 176 | "name": "e", 177 | "type": { 178 | "name": "__nested_nested_record_sub_e_enum", 179 | "type": "enum", 180 | "namespace": "test", 181 | "symbols": [ 182 | "A", 183 | "B" 184 | ] 185 | } 186 | }, 187 | { 188 | "name": "a", 189 | "type": { 190 | "type": "array", 191 | "items": "int" 192 | } 193 | }, 194 | { 195 | "name": "m", 196 | "type": { 197 | "type": "map", 198 | "values": "int" 199 | } 200 | }, 201 | { 202 | "name": "u", 203 | "type": [ 204 | "string", 205 | "int" 206 | ] 207 | }, 208 | { 209 | "name": "fx", 210 | "type": { 211 | "name": "__nested_nested_record_sub_fx_fixed", 212 | "type": "fixed", 213 | "namespace": "test", 214 | "size": 2 215 | } 216 | }, 217 | { 218 | "name": "subsub", 219 | "type": { 220 | "type": "record", 221 | "name": "__nested_nested_record_sub_subsub_record", 222 | "namespace": "test", 223 | "fields": [ 224 | { 225 | "name": "n", 226 | "type": "null" 227 | }, 228 | { 229 | "name": "b", 230 | "type": "boolean" 231 | }, 232 | { 233 | "name": "i", 234 | "type": "int" 235 | }, 236 | { 237 | "name": "l", 238 | "type": "long" 239 | }, 240 | { 241 | "name": "f", 242 | "type": "float" 243 | }, 244 | { 245 | "name": "d", 246 | "type": "double" 247 | }, 248 | { 249 | "name": "bs", 250 | "type": "bytes" 251 | }, 252 | { 253 | "name": "str", 254 | "type": "string" 255 | }, 256 | { 257 | "name": "date", 258 | "type": { 259 | "type": "int", 260 | "logicalType": "date" 261 | } 262 | }, 263 | { 264 | "name": "ts_msec", 265 | "type": { 266 | "type": "long", 267 | "logicalType": "timestamp-millis" 268 | } 269 | }, 270 | { 271 | "name": "ts_usec", 272 | "type": { 273 | "type": "long", 274 | "logicalType": "timestamp-micros" 275 | } 276 | }, 277 | { 278 | "name": "unknown", 279 | "type": { 280 | "type": "int", 281 | "logicalType": "foobar" 282 | } 283 | }, 284 | { 285 | "name": "e", 286 | "type": { 287 | "name": "__nested_nested_record_sub_subsub_e_enum", 288 | "type": "enum", 289 | "namespace": "test", 290 | "symbols": [ 291 | "A", 292 | "B" 293 | ] 294 | } 295 | }, 296 | { 297 | "name": "a", 298 | "type": { 299 | "type": "array", 300 | "items": "int" 301 | } 302 | }, 303 | { 304 | "name": "m", 305 | "type": { 306 | "type": "map", 307 | "values": "int" 308 | } 309 | }, 310 | { 311 | "name": "u", 312 | "type": [ 313 | "string", 314 | "int" 315 | ] 316 | }, 317 | { 318 | "name": "fx", 319 | "type": { 320 | "name": "__nested_nested_record_sub_subsub_fx_fixed", 321 | "type": "fixed", 322 | "namespace": "test", 323 | "size": 2 324 | } 325 | } 326 | ] 327 | } 328 | } 329 | ] 330 | } 331 | } 332 | ] 333 | } 334 | -------------------------------------------------------------------------------- /lib/avromatic/model/attributes.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'active_support/core_ext/object/duplicable' 4 | require 'active_support/time' 5 | 6 | module Avromatic 7 | module Model 8 | 9 | # This module supports defining Virtus attributes for a model based on the 10 | # fields of Avro schemas. 11 | module Attributes 12 | extend ActiveSupport::Concern 13 | 14 | class OptionalFieldError < StandardError 15 | attr_reader :field 16 | 17 | def initialize(field) 18 | @field = field 19 | super("Optional field not allowed: #{field}") 20 | end 21 | end 22 | 23 | class AttributeDefinition 24 | attr_reader :name, :name_string, :setter_name, :type, :field, :default, :owner 25 | 26 | delegate :serialize, to: :type 27 | 28 | def initialize(owner:, field:, type:) 29 | @owner = owner 30 | @field = field 31 | @required = FieldHelper.required?(field) 32 | @nullable = FieldHelper.nullable?(field) 33 | @type = type 34 | @values_immutable = type.referenced_model_classes.all?(&:recursively_immutable?) 35 | @name = field.name.to_sym 36 | @name_string = field.name.to_s.dup.freeze 37 | @setter_name = "#{field.name}=".to_sym 38 | @default = if field.default == :no_default 39 | nil 40 | elsif field.default.duplicable? 41 | field.default.dup.deep_freeze 42 | else 43 | field.default 44 | end 45 | end 46 | 47 | def nullable? 48 | @nullable 49 | end 50 | 51 | def required? 52 | @required 53 | end 54 | 55 | def values_immutable? 56 | @values_immutable 57 | end 58 | 59 | def coerce(input) 60 | type.coerce(input) 61 | rescue Avromatic::Model::UnknownAttributeError => e 62 | raise Avromatic::Model::CoercionError.new( 63 | "Value for #{owner.name}##{name} could not be coerced to a #{type.name} " \ 64 | "because the following unexpected attributes were provided: #{e.unknown_attributes.join(', ')}. " \ 65 | "Only the following attributes are allowed: #{e.allowed_attributes.join(', ')}. " \ 66 | "Provided argument: #{input.inspect}" 67 | ) 68 | rescue StandardError 69 | if type.input_classes && type.input_classes.none? { |input_class| input.is_a?(input_class) } 70 | raise Avromatic::Model::CoercionError.new( 71 | "Value for #{owner.name}##{name} could not be coerced to a #{type.name} " \ 72 | "because a #{input.class.name} was provided but expected a #{type.input_classes.map(&:name).to_sentence( 73 | two_words_connector: ' or ', last_word_connector: ', or ' 74 | )}. " \ 75 | "Provided argument: #{input.inspect}" 76 | ) 77 | elsif input.is_a?(Hash) && type.is_a?(Avromatic::Model::Types::UnionType) 78 | raise Avromatic::Model::CoercionError.new( 79 | "Value for #{owner.name}##{name} could not be coerced to a #{type.name} " \ 80 | "because no union member type matches the provided attributes: #{input.inspect}" 81 | ) 82 | else 83 | raise Avromatic::Model::CoercionError.new( 84 | "Value for #{owner.name}##{name} could not be coerced to a #{type.name}. " \ 85 | "Provided argument: #{input.inspect}" 86 | ) 87 | end 88 | end 89 | end 90 | 91 | included do 92 | class_attribute :attribute_definitions, instance_writer: false 93 | self.attribute_definitions = {} 94 | 95 | delegate :recursively_immutable?, to: :class 96 | end 97 | 98 | def initialize(data = {}) 99 | super() 100 | 101 | num_valid_keys = 0 102 | attribute_definitions.each do |attribute_name, attribute_definition| 103 | if data.include?(attribute_name) 104 | num_valid_keys += 1 105 | value = data.fetch(attribute_name) 106 | send(attribute_definition.setter_name, value) 107 | elsif data.include?(attribute_definition.name_string) 108 | num_valid_keys += 1 109 | value = data[attribute_definition.name_string] 110 | send(attribute_definition.setter_name, value) 111 | elsif !_attributes.include?(attribute_name) 112 | send(attribute_definition.setter_name, attribute_definition.default) 113 | end 114 | end 115 | 116 | unless Avromatic.allow_unknown_attributes || num_valid_keys == data.size 117 | unknown_attributes = (data.keys.map(&:to_s) - _attributes.keys.map(&:to_s)).sort 118 | allowed_attributes = attribute_definitions.keys.map(&:to_s).sort 119 | message = "Unexpected arguments for #{self.class.name}#initialize: #{unknown_attributes.join(', ')}. " \ 120 | "Only the following arguments are allowed: #{allowed_attributes.join(', ')}. " \ 121 | "Provided arguments: #{data.inspect}" 122 | raise Avromatic::Model::UnknownAttributeError.new(message, unknown_attributes: unknown_attributes, 123 | allowed_attributes: allowed_attributes) 124 | end 125 | end 126 | 127 | def to_h 128 | _attributes.dup 129 | end 130 | 131 | alias_method :to_hash, :to_h 132 | alias_method :attributes, :to_h 133 | 134 | private 135 | 136 | def _attributes 137 | @attributes ||= {} 138 | end 139 | 140 | module ClassMethods 141 | def add_avro_fields(generated_methods_module) 142 | # models are registered in Avromatic.nested_models at this point to 143 | # ensure that they are available as fields for recursive models. 144 | register! 145 | 146 | if key_avro_schema 147 | check_for_field_conflicts! 148 | begin 149 | define_avro_attributes(key_avro_schema, generated_methods_module, 150 | allow_optional: config.allow_optional_key_fields) 151 | rescue OptionalFieldError => e 152 | raise "Optional field '#{e.field.name}' not allowed in key schema." 153 | end 154 | end 155 | define_avro_attributes(avro_schema, generated_methods_module) 156 | end 157 | 158 | def recursively_immutable? 159 | return @recursively_immutable if defined?(@recursively_immutable) 160 | 161 | @recursively_immutable = immutable? && attribute_definitions.each_value.all?(&:values_immutable?) 162 | end 163 | 164 | private 165 | 166 | def check_for_field_conflicts! 167 | conflicts = 168 | (key_avro_field_names & value_avro_field_names).each_with_object([]) do |name, msgs| 169 | next unless schema_fields_differ?(name) 170 | 171 | msgs << "Field '#{name}' has a different type in each schema: "\ 172 | "value #{value_avro_fields_by_name[name]}, "\ 173 | "key #{key_avro_fields_by_name[name]}" 174 | end 175 | 176 | raise conflicts.join("\n") if conflicts.any? 177 | 178 | conflicts 179 | end 180 | 181 | # The Avro::Schema::Field#== method is lame. It just compares 182 | # .type.type_sym. 183 | def schema_fields_differ?(name) 184 | key_avro_fields_by_name[name].to_avro != 185 | value_avro_fields_by_name[name].to_avro 186 | end 187 | 188 | def define_avro_attributes(schema, generated_methods_module, allow_optional: true) 189 | if schema.type_sym != :record 190 | raise "Unsupported schema type '#{schema.type_sym}', only 'record' schemas are supported." 191 | end 192 | 193 | schema.fields.each do |field| 194 | raise OptionalFieldError.new(field) if !allow_optional && FieldHelper.optional?(field) 195 | 196 | symbolized_field_name = field.name.to_sym 197 | attribute_definition = AttributeDefinition.new( 198 | owner: self, 199 | field: field, 200 | type: Avromatic::Model::Types::TypeFactory.create(schema: field.type, nested_models: nested_models) 201 | ) 202 | attribute_definitions[symbolized_field_name] = attribute_definition 203 | 204 | # Add all generated methods to a module so they can be overridden 205 | generated_methods_module.send(:define_method, field.name) { _attributes[symbolized_field_name] } 206 | if FieldHelper.boolean?(field) 207 | generated_methods_module.send(:define_method, "#{field.name}?") do 208 | !!_attributes[symbolized_field_name] 209 | end 210 | end 211 | 212 | generated_methods_module.send(:define_method, "#{field.name}=") do |value| 213 | _attributes[symbolized_field_name] = attribute_definition.coerce(value) 214 | end 215 | 216 | unless mutable? # rubocop:disable Style/Next 217 | generated_methods_module.send(:private, "#{field.name}=") 218 | generated_methods_module.send(:define_method, :clone) { self } 219 | generated_methods_module.send(:define_method, :dup) { self } 220 | end 221 | end 222 | end 223 | end 224 | 225 | end 226 | end 227 | end 228 | -------------------------------------------------------------------------------- /spec/avromatic/model/builder_validation_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | describe Avromatic::Model::Builder, 'validation' do 4 | let(:schema) { schema_store.find(schema_name) } 5 | let(:test_class) do 6 | described_class.model(schema_name: schema_name) 7 | end 8 | let(:attribute_names) do 9 | test_class.attribute_definitions.keys.map(&:to_s) 10 | end 11 | 12 | context "required" do 13 | context "primitive types" do 14 | let(:schema_name) { 'test.primitive_types' } 15 | let(:valid_attributes) do 16 | { 17 | s: 'foo', 18 | b: '123', 19 | tf: true, 20 | i: 777, 21 | l: 123456789, 22 | f: 0.5, 23 | d: 1.0 / 3.0, 24 | n: nil, 25 | fx: '1234567', 26 | e: 'A' 27 | } 28 | end 29 | 30 | it "validates that required fields must be present" do 31 | instance = test_class.new 32 | assert_model_invalid(instance, 's', 'tf') 33 | end 34 | 35 | it "passes validation if all required fields are present" do 36 | instance = test_class.new(valid_attributes) 37 | assert_model_valid(instance) 38 | end 39 | 40 | context "boolean" do 41 | it "allows a boolean field to be false" do 42 | instance = test_class.new(valid_attributes.merge(tf: false)) 43 | assert_model_valid(instance) 44 | end 45 | end 46 | end 47 | 48 | context "array" do 49 | let(:schema) do 50 | Avro::Builder.build_schema do 51 | record :has_array do 52 | required :a, :array, items: :int 53 | end 54 | end 55 | end 56 | let(:test_class) { described_class.model(schema: schema) } 57 | 58 | it "validates that a required array is not nil" do 59 | instance = test_class.new(a: nil) 60 | assert_model_invalid(instance, 'a') 61 | end 62 | 63 | it "allows a required array to be empty" do 64 | instance = test_class.new(a: []) 65 | assert_model_valid(instance) 66 | end 67 | 68 | it "passes validation if all required fields are present" do 69 | instance = test_class.new(a: [1]) 70 | assert_model_valid(instance) 71 | end 72 | end 73 | 74 | context "map" do 75 | let(:schema) do 76 | Avro::Builder.build_schema do 77 | record :has_map do 78 | required :m, :map, values: :int 79 | end 80 | end 81 | end 82 | let(:test_class) { described_class.model(schema: schema) } 83 | 84 | it "validates that a required map is not nil" do 85 | instance = test_class.new(m: nil) 86 | assert_model_invalid(instance, 'm') 87 | end 88 | 89 | it "allows a required map to be empty" do 90 | instance = test_class.new(m: {}) 91 | assert_model_valid(instance) 92 | end 93 | 94 | it "passes validation if all required fields are present" do 95 | instance = test_class.new(m: { 'foo' => 1 }) 96 | assert_model_valid(instance) 97 | end 98 | end 99 | 100 | context "nested records" do 101 | let(:schema) do 102 | Avro::Builder.build_schema do 103 | record :has_record do 104 | required :sub, :record, type_name: 'sub_type' do 105 | required :i, :int 106 | required :s, :string 107 | end 108 | end 109 | end 110 | end 111 | let(:test_class) { described_class.model(schema: schema) } 112 | 113 | it "validates nested records" do 114 | instance = test_class.new(sub: test_class.nested_models['sub_type'].new) 115 | assert_model_invalid(instance, 'sub.i', 'sub.s') 116 | end 117 | 118 | it "validates nested records initialized with a hash" do 119 | instance = test_class.new(sub: { i: 0 }) 120 | assert_model_invalid(instance, 'sub.s') 121 | end 122 | 123 | it "passes validation if all required fields are present" do 124 | instance = test_class.new(sub: { i: 0, s: 'f' }) 125 | assert_model_valid(instance) 126 | end 127 | 128 | context "doubly nested record" do 129 | let(:schema) do 130 | Avro::Builder.build_schema do 131 | record :outer do 132 | required :sub, :record, type_name: 'level1' do 133 | required :sub_sub, :record, type_name: 'level2' do 134 | required :l, :long 135 | end 136 | end 137 | end 138 | end 139 | end 140 | 141 | it "validates multiple levels of nesting" do 142 | level1 = test_class.nested_models['level1'] 143 | level2 = test_class.nested_models['level2'] 144 | instance = test_class.new(sub: level1.new(sub_sub: level2.new)) 145 | assert_model_invalid(instance, 'sub.sub_sub.l') 146 | end 147 | 148 | it "validates multiple levels of nesting initialized with a hash" do 149 | instance = test_class.new(sub: { sub_sub: {} }) 150 | assert_model_invalid(instance, 'sub.sub_sub.l') 151 | end 152 | 153 | it "passes validation if all required fields are present" do 154 | instance = test_class.new(sub: { sub_sub: { l: 1 } }) 155 | assert_model_valid(instance) 156 | end 157 | end 158 | 159 | context "array of records" do 160 | let(:schema) do 161 | Avro::Builder.build_schema do 162 | record :x_and_y do 163 | required :x, :int 164 | required :y, :int 165 | end 166 | 167 | record :array_of_records do 168 | required :ary, :array, items: :x_and_y 169 | end 170 | end 171 | end 172 | 173 | it "validates records in an array" do 174 | nested_model = test_class.nested_models['x_and_y'] 175 | data = [nested_model.new, 176 | nested_model.new(x: 1), 177 | nested_model.new(y: 2)] 178 | instance = test_class.new(ary: data) 179 | assert_model_invalid(instance, 'ary[0].x', 'ary[0].y', 'ary[1].y', 'ary[2].x') 180 | end 181 | 182 | it "validates records in an array initialized with hashes" do 183 | data = [{}, 184 | { x: 1 }, 185 | { y: 2 }] 186 | instance = test_class.new(ary: data) 187 | assert_model_invalid(instance, 'ary[0].x', 'ary[0].y', 'ary[1].y', 'ary[2].x') 188 | end 189 | 190 | it "passes validation if all required fields are present" do 191 | instance = test_class.new(ary: [{ x: 1, y: 2 }]) 192 | assert_model_valid(instance) 193 | end 194 | end 195 | 196 | context "nested arrays of records" do 197 | let(:schema) do 198 | Avro::Builder.build_schema do 199 | record :elt do 200 | required :s, :string 201 | end 202 | 203 | record :with_matrix do 204 | required :m, array(array(:elt)) 205 | end 206 | end 207 | end 208 | let(:nested_model) { test_class.nested_models['elt'] } 209 | 210 | it "validates deeply nested records" do 211 | data = [ 212 | [nested_model.new(s: 'a'), nested_model.new], 213 | [nested_model.new(s: 'c'), nested_model.new, nested_model.new(s: 'b')] 214 | ] 215 | instance = test_class.new(m: data) 216 | assert_model_invalid(instance, 'm[0][1].s', 'm[1][1].s') 217 | end 218 | 219 | it "passes validation if all required fields are present" do 220 | instance = test_class.new(m: [[{ s: 's' }]]) 221 | assert_model_valid(instance) 222 | end 223 | end 224 | 225 | context "map of records" do 226 | let(:schema) do 227 | Avro::Builder.build_schema do 228 | record :x_and_y do 229 | required :x, :int 230 | required :y, :int 231 | end 232 | 233 | record :array_of_records do 234 | required :map, :map, values: :x_and_y 235 | end 236 | end 237 | end 238 | 239 | it "validates records in a map" do 240 | nested_model = test_class.nested_models['x_and_y'] 241 | data = { 242 | a: nested_model.new(y: 3), 243 | b: nested_model.new, 244 | c: nested_model.new(x: 4) 245 | } 246 | instance = test_class.new(map: data) 247 | assert_model_invalid(instance, "map['a'].x", "map['b'].x", "map['b'].y", "map['c'].y") 248 | end 249 | 250 | it "validates records in a map initialized with hashes" do 251 | data = { 252 | a: { y: 3 }, 253 | b: {}, 254 | c: { x: 4 } 255 | } 256 | instance = test_class.new(map: data) 257 | assert_model_invalid(instance, "map['a'].x", "map['b'].x", "map['b'].y", "map['c'].y") 258 | end 259 | 260 | it "passes validation if all required fields are present" do 261 | instance = test_class.new(map: { a: { x: 0, y: 1 } }) 262 | assert_model_valid(instance) 263 | end 264 | end 265 | 266 | context "record in a union" do 267 | let(:schema) do 268 | Avro::Builder.build_schema do 269 | record :x_and_y do 270 | required :x, :int 271 | required :y, :int 272 | end 273 | 274 | record :with_union do 275 | required :u, :union, types: [:x_and_y, :string] 276 | end 277 | end 278 | end 279 | 280 | it "validates a record in a union" do 281 | expect(test_class.new(u: 'foo')).to be_valid 282 | instance = test_class.new(u: test_class.nested_models['x_and_y'].new) 283 | assert_model_invalid(instance, 'u.x', 'u.y') 284 | end 285 | end 286 | end 287 | end 288 | 289 | context "optional" do 290 | let(:schema_name) { 'test.with_union' } 291 | 292 | it "does not require optional fields to be present" do 293 | instance = test_class.new 294 | assert_model_valid(instance) 295 | end 296 | end 297 | 298 | # Tests both ActiveModel validation and validation during serialization 299 | def assert_model_valid(instance) 300 | instance.value_attributes_for_avro 301 | expect(instance).to be_valid 302 | end 303 | 304 | # Tests both ActiveModel validation and validation during serialization 305 | def assert_model_invalid(instance, *missing_attributes) 306 | expect do 307 | instance.value_attributes_for_avro 308 | end.to raise_error do |error| 309 | expect(error).to be_a_kind_of(Avromatic::Model::ValidationError) 310 | missing_attributes.each do |missing_attribute| 311 | expect(error.message).to include(missing_attribute) 312 | end 313 | end 314 | 315 | expect(instance).to be_invalid 316 | missing_attributes.each do |missing_attribute| 317 | expect(instance.errors[:base]).to include("#{missing_attribute} can't be nil") 318 | end 319 | end 320 | end 321 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # avromatic changelog 2 | 3 | ## 5.0.0 4 | - Add support for Rails 7.1 5 | - Drop support for Ruby < 3 6 | - Add support for Rails < 6.1 7 | - Drop support for Avro 1.10 8 | 9 | ## 4.3.0 10 | - Add support for decimal logical type 11 | 12 | ## 4.2.0 13 | - Add an `Avromatic.eager_load_models` attribute reader method. 14 | - Remove unnecessary files from the gem distribution. 15 | 16 | ## 4.1.1 17 | - Fix eager loading of nested models when using the Zeitwerk classloader with Rails. 18 | 19 | ## 4.1.0 20 | - Add support for specifying a subject for the avro schema when building an Avromatic model 21 | 22 | ## 4.0.0 23 | - Drop support for Ruby 2.6. 24 | - Drop support for Avro 1.9. 25 | - Add support for Avro 1.11. 26 | - Add support for Rails 7.0. 27 | 28 | ## 3.0.2 29 | - Reset the schema registry client between RSpec tests to ensure any cached values are 30 | consistent with the fake schema registry. 31 | 32 | ## 3.0.1 33 | - Raise an error when registering a nested model that has already been auto-generated. 34 | This avoids hard to troubleshoot coercion errors when instantiating models and fixes 35 | a regression introduced in Avromatic 2.2.2. 36 | 37 | ## 3.0.0 38 | - Drop support for Ruby 2.4. 39 | - Add support for Ruby 3.0. 40 | - Drop support for Avro < 1.9. 41 | - Drop support for Rails < 5.2. 42 | - Fix decoding of unions containing false boolean values. 43 | 44 | ## v2.4.0 45 | - Ignore the `validate` argument and always validate during serialization. This 46 | argument will be removed in Avromatic 3.0. 47 | - Optimize model validation during serialization. 48 | - Don't cache immutable model validation results or serialized Avro attributes if a model has mutable children. 49 | 50 | ## v2.3.0 51 | - Add support for Rails 6.1. 52 | - Optimize nested model serialization. 53 | 54 | ## v2.2.6 55 | - Optimize memory usage when serializing models. 56 | 57 | ## v2.2.5 58 | - Optimize memory usage when serializing, deserializing and instantiating models. 59 | 60 | ## v2.2.4 61 | - Compatibility with Avro v1.10.x. 62 | 63 | ## v2.2.3 64 | - Fix bug where method `#referenced_model_classes` was declared as private instead of public. 65 | 66 | ## v2.2.2 67 | - Fix missing models in the model registry when in development by loading the nested models of eager loaded models. 68 | - Fake schema registry support for stubbing URLs with usernames and passwords. 69 | 70 | ## v2.2.1 71 | - Avoid allocating default empty hash in `Avromatic::IO::DatumReader.read_data` 72 | 73 | ## v2.2.0 74 | - Add support for Rails 6.0. 75 | - Drop support for Ruby < 2.4. 76 | 77 | ## v2.1.0 78 | - Add `key_schema_name` and `value_schema_name` attributes to `UnexpectedKeyError`. 79 | 80 | ## v2.0.2 81 | - Optimize model initialization and decoding 82 | 83 | ## v2.0.1 84 | - Allow generated model attribute accessors to be overridden. This was a regression in Avromatic 2.0.0. 85 | - Ensure that timestamp-millis are coerced when the number of microseconds is divisible by 1,000 but the 86 | number of nanoseconds is not divisible by 1,000,000. 87 | 88 | ## v2.0.0 89 | - Remove [virtus](https://github.com/solnic/virtus) dependency resulting in a 3x performance improvement in model instantation and 1.4x - 2.0x performance improvement in Avro serialization and Avromatic code simplification. 90 | - Raise `Avromatic::Model::CoercionError` when attribute values can't be coerced to the target type in model constructors and attribute setters. Previously coercion errors weren't detected until Avro serialization or an explicit call to `valid?`. 91 | - Prevent model instances from being constructed with unknown attributes. Previously unknown attributes were ignored. 92 | This can be disabled by setting `Avromatic.allow_unknown_attributes` to `true`. 93 | WARNING: Setting `Avromatic.allow_unknown_attributes` to `true` will result in incorrect union member coercions 94 | if an earlier union member is satisfied by a subset of the latter union member's attributes. 95 | - Validate required attributes are present when serializing to Avro for better error messages. Explicit 96 | validation can still be done by calling the `valid?` or `invalid?` methods from the 97 | [ActiveModel::Validations](https://edgeapi.rubyonrails.org/classes/ActiveModel/Validations.html) interface 98 | but errors will now appear under the `:base` key. Previously these errors were detected late in the Avro serialization process resulting in hard to understand error messages. 99 | - Support for custom types in unions with more than one non-null type. 100 | - Drop support for Ruby < 2.3 and Rails < 5.0. 101 | - Call `super()` in model constructor making it easier to define class/module hierarchies for models. 102 | 103 | ## v1.0.0 104 | - No changes. 105 | 106 | ## v0.33.0 107 | - Fix compatibility with avro-patches v0.4.0. 108 | 109 | ## v0.32.0 110 | - Improve partial assignment using a hash for records outside of unions. 111 | - Prevent invalid types from being assigned to primitives in unions. 112 | - Add validation that primitive attributes have the expected type. 113 | 114 | ## v0.31.0 115 | - Add support for Rails 5.2. 116 | 117 | ## v0.30.0 118 | - Add `Avromatic::Model::MessageDecoder#model` method to return the Avromatic 119 | model class for a message. 120 | 121 | ## v0.29.1 122 | - Add `Avromatic.build_messaging!` to `avromatic/rspec`. 123 | 124 | ## v0.29.0 125 | - Add new public methods `#avro_key_datum` and `#avro_value_datum` on an 126 | Avromatic model instance that return the attributes of the model suitable for 127 | Avro encoding without any customizations. 128 | 129 | ## v0.28.1 130 | - Fix a bug that raised an error when encoding a cached model containing optional 131 | field(s). With this change, immutable model caching now enabled only when 132 | `avro-patches` is present. 133 | 134 | ## v0.28.0 135 | - Add support for caching avro encodings for immutable models 136 | 137 | ## v0.27.0 138 | - Patches avromatic model classes to cache `Virtus::ValueObject::AllowedWriterMethods#allowed_writer_methods` 139 | - Support Rails 5.1 140 | 141 | ## v0.26.0 142 | - Caches result of Avromatic::Model::RawSerialization#value_attributes_for_avro for immutable models 143 | 144 | ## v0.25.0 145 | - Disallow optional fields in schemas used for keys by default. 146 | 147 | ## v0.24.0 148 | - Add `Avromatic::IO::DatumWriter` to optimize the encoding of Avro unions 149 | from Avromatic models. 150 | - Expose the `#key_attributes_for_avro` method on models. 151 | 152 | ## v0.23.0 153 | - Add mutable option when defining a model. 154 | 155 | ## v0.22.0 156 | - Require `avro_schema_registry_client` v0.3.0 or later to avoid 157 | using `avro-salsify-fork`. 158 | 159 | ## v0.21.1 160 | - Fix a bug in the optimization of optional union decoding. 161 | 162 | ## v0.21.0 163 | - Remove monkey-patches for `AvroTurf::ConfluentSchemaRegistry` and 164 | `FakeConfluentSchemaRegistryServer` and depend on `avro_schema_registry-client` 165 | instead. 166 | - Rename the configuration option `use_cacheable_schema_registration` to 167 | `use_schema_fingerprint_lookup`. 168 | 169 | ## v0.20.0 170 | - Support schema stores with a `#clear_schemas` method. 171 | 172 | ## v0.19.0 173 | - Use a fingerprint based on `avro-resolution_canonical_form`. 174 | 175 | ## v0.18.1 176 | - Replace another reference to `avromatic/test/fake_schema_registry_server`. 177 | 178 | ## v0.18.0 179 | - Compatibility with `avro_turf` v0.8.0. `avromatic/test/fake_schema_registry_server` 180 | is now deprecated and will be removed in a future release. 181 | Use `avromatic/test/fake_confluent_schema_registry_server` instead. 182 | 183 | ## v0.17.1 184 | - Correctly namespace Avro errors raised by `Avromatic::IO::DatumReader`. 185 | 186 | ## v0.17.0 187 | - Add `.register_schemas!` method to generated models to register the associated 188 | schemas in a schema registry. 189 | 190 | ## v0.16.0 191 | - Add `#lookup_subject_schema` method to `AvroTurf::SchemaRegistry` patch to 192 | directly support looking up existing schema ids by fingerprint. 193 | 194 | ## v0.15.1 195 | - Add `Avromatic.use_cacheable_schema_registration` option to control the lookup 196 | of existing schema ids by fingerprint. 197 | 198 | ## v0.15.0 199 | - Add patch to `AvroTurf::SchemaRegistry` to lookup existing schema ids using 200 | `GET /subjects/:subject/fingerprints/:fingerprint` from `#register`. 201 | This endpoint is supported in the avro-schema-registry. 202 | - Add patch to the `FakeSchemaRegistryServer` from `AvroTurf` to support the 203 | fingerprint endpoint. 204 | 205 | ## v0.14.0 206 | - Add `Avromatic::Messaging` and `Avromatic::IO::DatumReader` classes to 207 | optimize the decoding of Avro unions to Avromatic models. 208 | 209 | ## v0.13.0 210 | - Add interfaces to deserialize as a hash of attributes instead of a model. 211 | 212 | ## v0.12.0 213 | - Clear the schema store, if it supports it, prior to code reloading in Rails 214 | applications. This allows schema changes to be picked up during code 215 | reloading. 216 | 217 | ## v0.11.2 218 | - Fix for models containing optional array and map fields. 219 | 220 | ## v0.11.1 221 | - Another fix for Rails initialization and reloading. Do not clear the nested 222 | models registry the first time that the `to_prepare` hook is called. 223 | 224 | ## v0.11.0 225 | - Replace `Avromatic.on_initialize` proc with `Avromatic.eager_load_models` 226 | array. The models listed by this configuration are added to the registry 227 | at the end of `.configure` and prior to code reloading in Rails applications. 228 | This is a compatibility breaking change. 229 | 230 | ## v0.10.0 231 | - Add `Avromatic.on_initialize` proc that is called at the end of `.configure` 232 | and on code reloading in Rails applications. 233 | 234 | ## v0.9.0 235 | - Experimental: Add support for more than one non-null type in a union. 236 | - Allow nested models to be referenced and reused. 237 | - Fix the serialization of nested complex types. 238 | - Add support for recursive models. 239 | - Allow required array and map fields to be empty. Only nil values for required 240 | array and map fields are now considered invalid. 241 | - Validate nested models. This includes models embedded within other complex 242 | types (array, map, and union). 243 | - Truncate values for timestamps to the precision supported by the logical type. 244 | 245 | ## v0.8.0 246 | - Add support for logical types. Currently this requires using the 247 | `avro-salsify-fork` gem for logical types support with Ruby. 248 | 249 | ## v0.7.1 250 | - Raise a more descriptive error when attempting to generate a model for a 251 | non-record Avro type. 252 | 253 | ## v0.7.0 254 | - Add RSpec `FakeSchemaRegistryServer` test helper. 255 | 256 | ## v0.6.2 257 | - Remove dependency on `Hash#transform_values` from `ActiveSupport` v4.2. 258 | 259 | ## v0.6.1 260 | - Fix serialization of array and map types that contain nested models. 261 | 262 | ## v0.6.0 263 | - Require `avro_turf` v0.7.0 or later. 264 | 265 | ## v0.5.0 266 | - Rename `Avromatic::Model::Decoder` to `MessageDecoder`. 267 | - Rename `.deserialize` on generated models to `.avro_message_decode`. 268 | - Add `#avro_raw_key`, `#avro_raw_value` and `.avro_raw_decode` methods to 269 | generated models to support encoding and decoding without a schema registry. 270 | 271 | ## v0.4.0 272 | - Allow the specification of a custom type, including conversion to/from Avro, 273 | for named types. 274 | 275 | ## v0.3.0 276 | - Remove dependency on the `private_attr` gem. 277 | 278 | ## v0.2.0 279 | - Allow a module level schema registry to be configured. 280 | 281 | ## v0.1.2 282 | - Do not build an `AvroTurf::Messaging` object for each model class. 283 | 284 | ## v0.1.1 285 | - Fix Railtie. 286 | 287 | ## v0.1.0 288 | - Initial release 289 | -------------------------------------------------------------------------------- /spec/avromatic/model/messaging_serialization_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'avro/builder' 4 | 5 | describe Avromatic::Model::MessagingSerialization do 6 | let(:values) { { id: rand(99) } } 7 | let(:instance) { test_class.new(values) } 8 | let(:avro_message_value) { instance.avro_message_value } 9 | let(:avro_message_key) { instance.avro_message_key } 10 | 11 | before do 12 | allow(Avromatic.schema_registry).to receive(:register).and_call_original 13 | end 14 | 15 | describe "#avro_message_value" do 16 | let(:test_class) do 17 | Avromatic::Model.model(value_schema_name: 'test.encode_value') 18 | end 19 | let(:values) { { str1: 'a', str2: 'b' } } 20 | 21 | it "encodes the value for the model" do 22 | message_value = instance.avro_message_value 23 | decoded = test_class.avro_message_decode(message_value) 24 | expect(decoded).to eq(instance) 25 | expect(Avromatic.schema_registry).to have_received(:register) 26 | .with('test.encode_value', instance_of(Avro::Schema::RecordSchema)) 27 | end 28 | 29 | context "with a specified value subject" do 30 | let(:test_class) do 31 | Avromatic::Model.model(value_schema_name: 'test.encode_value', 32 | value_schema_subject: 'test.encode_value-subject') 33 | end 34 | let(:values) { { str1: 'a', str2: 'b' } } 35 | 36 | it "encodes the value for the model" do 37 | message_value = instance.avro_message_value 38 | decoded = test_class.avro_message_decode(message_value) 39 | expect(decoded).to eq(instance) 40 | expect(Avromatic.schema_registry).to have_received(:register) 41 | .with('test.encode_value-subject', 42 | instance_of(Avro::Schema::RecordSchema)) 43 | end 44 | end 45 | 46 | context "with a nested record" do 47 | let(:test_class) do 48 | Avromatic::Model.model(value_schema_name: 'test.nested_record') 49 | end 50 | let(:values) { { str: 'a', sub: { str: 'b', i: 1 } } } 51 | 52 | it "encodes the value for the model" do 53 | expect(instance.sub.str).to eq('b') 54 | expect(instance.sub.i).to eq(1) 55 | message_value = instance.avro_message_value 56 | decoded = test_class.avro_message_decode(message_value) 57 | expect(decoded).to eq(instance) 58 | end 59 | end 60 | 61 | context "with an array of models" do 62 | let(:test_class) do 63 | schema = Avro::Builder.build_schema do 64 | record :key_value do 65 | required :key, :string 66 | required :value, :string 67 | end 68 | 69 | record :key_value_pairs do 70 | required :id, :int 71 | required :pairs, :array, items: :key_value 72 | end 73 | end 74 | Avromatic::Model.model(schema: schema) 75 | end 76 | let(:values) do 77 | { 78 | id: 1, 79 | pairs: [ 80 | { key: 'foo', value: 'A' }, 81 | { key: 'bar', value: 'B' } 82 | ] 83 | } 84 | end 85 | let(:instance) { test_class.new(values) } 86 | 87 | before do 88 | allow(Avromatic.schema_store) 89 | .to receive(:find).with('key_value_pairs', nil).and_return(test_class.value_avro_schema) 90 | end 91 | 92 | it "encodes the value for the model" do 93 | first_pair = instance.pairs.first 94 | expect(first_pair.key).to eq('foo') 95 | expect(first_pair.value).to eq('A') 96 | message_value = instance.avro_message_value 97 | decoded = test_class.avro_message_decode(message_value) 98 | expect(decoded).to eq(instance) 99 | end 100 | end 101 | 102 | context "with an optional record" do 103 | let(:test_class) do 104 | Avromatic::Model.model(value_schema_name: 'test.optional_record') 105 | end 106 | let(:values) { { id: 42, message: { body: 'foo' } } } 107 | 108 | it "encodes the value for the model" do 109 | message_value = instance.avro_message_value 110 | decoded = test_class.avro_message_decode(message_value) 111 | expect(decoded).to eq(instance) 112 | end 113 | end 114 | 115 | context "with an optional array" do 116 | let(:test_class) do 117 | Avromatic::Model.model(value_schema_name: 'test.optional_array') 118 | end 119 | let(:values) { { id: 42, messages: [{ body: 'foo' }] } } 120 | 121 | it "encodes the value for the model" do 122 | message_value = instance.avro_message_value 123 | decoded = test_class.avro_message_decode(message_value) 124 | expect(decoded).to eq(instance) 125 | end 126 | end 127 | 128 | context "with a map of models" do 129 | let(:test_class) do 130 | schema = Avro::Builder.build_schema do 131 | record :submodel do 132 | required :length, :int 133 | required :str, :string 134 | end 135 | 136 | record :with_embedded do 137 | required :map, :map, values: :submodel 138 | end 139 | end 140 | Avromatic::Model.model(schema: schema) 141 | end 142 | let(:values) do 143 | { 144 | map: { 145 | foo: { length: 3, str: 'bar' }, 146 | baz: { length: 6, str: 'foobar' } 147 | } 148 | } 149 | end 150 | let(:instance) { test_class.new(values) } 151 | 152 | before do 153 | allow(Avromatic.schema_store) 154 | .to receive(:find).with('with_embedded', nil).and_return(test_class.value_avro_schema) 155 | end 156 | 157 | it "encodes the value for the model" do 158 | first_value = instance.map['foo'] 159 | expect(first_value.length).to eq(3) 160 | expect(first_value.str).to eq('bar') 161 | message_value = instance.avro_message_value 162 | decoded = test_class.avro_message_decode(message_value) 163 | expect(decoded).to eq(instance) 164 | end 165 | end 166 | end 167 | 168 | describe "#avro_message_key" do 169 | let(:test_class) do 170 | Avromatic::Model.model( 171 | value_schema_name: 'test.encode_value', 172 | key_schema_name: 'test.encode_key' 173 | ) 174 | end 175 | let(:values) { super().merge!(str1: 'a', str2: 'b') } 176 | 177 | it "encodes the key for the model" do 178 | message_value = instance.avro_message_value 179 | message_key = instance.avro_message_key 180 | decoded = test_class.avro_message_decode(message_key, message_value) 181 | expect(decoded).to eq(instance) 182 | expect(Avromatic.schema_registry).to have_received(:register) 183 | .with('test.encode_key', instance_of(Avro::Schema::RecordSchema)) 184 | end 185 | 186 | context "with a specified key subject" do 187 | let(:test_class) do 188 | Avromatic::Model.model(value_schema_name: 'test.encode_value', 189 | key_schema_name: 'test.encode_key', 190 | key_schema_subject: 'test.encode_key-subject') 191 | end 192 | 193 | it "encodes the key for the model" do 194 | message_value = instance.avro_message_value 195 | message_key = instance.avro_message_key 196 | decoded = test_class.avro_message_decode(message_key, message_value) 197 | expect(decoded).to eq(instance) 198 | expect(Avromatic.schema_registry).to have_received(:register) 199 | .with('test.encode_key-subject', instance_of(Avro::Schema::RecordSchema)) 200 | end 201 | end 202 | 203 | context "when a model does not have a key schema" do 204 | let(:test_class) do 205 | Avromatic::Model.model(value_schema_name: 'test.encode_value') 206 | end 207 | let(:values) { { str1: 'a', str2: 'b' } } 208 | 209 | it "raises an error" do 210 | expect { instance.avro_message_key }.to raise_error('Model has no key schema') 211 | end 212 | end 213 | end 214 | 215 | describe ".avro_message_decode" do 216 | let(:test_class) do 217 | Avromatic::Model.model(value_schema_name: 'test.encode_value') 218 | end 219 | let(:values) { { str1: 'a', str2: 'b' } } 220 | 221 | it "deserializes a model" do 222 | decoded = test_class.avro_message_decode(avro_message_value) 223 | expect(decoded).to eq(instance) 224 | end 225 | 226 | context "when a value and a key are specified" do 227 | let(:test_class) do 228 | Avromatic::Model.model( 229 | value_schema_name: 'test.encode_value', 230 | key_schema_name: 'test.encode_key' 231 | ) 232 | end 233 | let(:values) { { id: rand(99), str1: 'a', str2: 'b' } } 234 | 235 | it "deserializes a model" do 236 | decoded = test_class.avro_message_decode(avro_message_key, avro_message_value) 237 | expect(decoded).to eq(instance) 238 | end 239 | end 240 | 241 | context "a model with a union" do 242 | let(:use_custom_datum_reader) { true } 243 | let(:schema_name) { 'test.real_union' } 244 | let(:test_class) do 245 | Avromatic::Model.model(schema_name: schema_name) 246 | end 247 | let(:values) do 248 | { 249 | header: 'has bar', 250 | # This value corresponds to the second union member 251 | message: { bar_message: "I'm a bar" } 252 | } 253 | end 254 | let(:first_union_member) do 255 | test_class.attribute_definitions[:message].type.value_classes.first 256 | end 257 | 258 | before do 259 | instance 260 | allow(Avromatic).to receive(:use_custom_datum_reader).and_return(use_custom_datum_reader) 261 | allow(first_union_member).to receive(:new).and_call_original 262 | end 263 | 264 | it "only coerces using the correct union member" do 265 | decoded = test_class.avro_message_decode(avro_message_value) 266 | expect(decoded).to eq(instance) 267 | expect(first_union_member).not_to have_received(:new) 268 | end 269 | 270 | context "when use_custom_datum_reader is false" do 271 | let(:use_custom_datum_reader) { false } 272 | 273 | it "attempts to coerce until a union member matches" do 274 | decoded = test_class.avro_message_decode(avro_message_value) 275 | expect(decoded).to eq(instance) 276 | expect(first_union_member).to have_received(:new) 277 | end 278 | end 279 | end 280 | end 281 | 282 | describe ".avro_message_attributes" do 283 | let(:test_class) do 284 | Avromatic::Model.model(value_schema_name: 'test.encode_value') 285 | end 286 | let(:values) { { str1: 'a', str2: 'b' } } 287 | let(:attributes) { values.stringify_keys } 288 | 289 | it "deserializes attributes for a model" do 290 | decoded = test_class.avro_message_attributes(avro_message_value) 291 | expect(decoded).to eq(attributes) 292 | end 293 | 294 | context "when a value and a key are specified" do 295 | let(:test_class) do 296 | Avromatic::Model.model( 297 | value_schema_name: 'test.encode_value', 298 | key_schema_name: 'test.encode_key' 299 | ) 300 | end 301 | let(:values) { { id: rand(99), str1: 'a', str2: 'b' } } 302 | 303 | it "deserializes attributes for a model" do 304 | decoded = test_class.avro_message_attributes(avro_message_key, avro_message_value) 305 | expect(decoded).to eq(attributes) 306 | end 307 | end 308 | end 309 | 310 | it_behaves_like "logical type encoding and decoding" do 311 | let(:encoded_value) { instance.avro_message_value } 312 | let(:decoded) { test_class.avro_message_decode(encoded_value) } 313 | end 314 | 315 | context "custom types" do 316 | let(:schema_name) { 'test.named_type' } 317 | let(:test_class) do 318 | Avromatic::Model.model(schema_name: schema_name) 319 | end 320 | let(:values) { { six_str: 'fOObAR' } } 321 | 322 | context "with a value class" do 323 | let(:value_class) do 324 | Class.new do 325 | attr_reader :value 326 | 327 | def initialize(value) 328 | @value = value 329 | end 330 | 331 | def self.from_avro(value) 332 | new(value.downcase) 333 | end 334 | 335 | def self.to_avro(value) 336 | value.value.capitalize 337 | end 338 | end 339 | end 340 | 341 | before do 342 | Avromatic.register_type('test.six', value_class) 343 | end 344 | 345 | it "stores the attribute in the model class" do 346 | expect(instance.six_str).to be_a(value_class) 347 | end 348 | 349 | it "converts when assigning to the model" do 350 | expect(instance.six_str.value).to eq('foobar') 351 | end 352 | 353 | it "converts when encoding the value" do 354 | decoded = Avromatic.messaging.decode(avro_message_value, schema_name: schema_name) 355 | expect(decoded['six_str']).to eq('Foobar') 356 | end 357 | end 358 | 359 | context "without a value class" do 360 | before do 361 | Avromatic.register_type('test.six') do |type| 362 | type.from_avro = ->(value) { value.downcase } 363 | type.to_avro = ->(value) { value.capitalize } 364 | end 365 | end 366 | 367 | it "converts when assigning to the model" do 368 | expect(instance.six_str).to eq('foobar') 369 | end 370 | 371 | it "converts when encoding the value" do 372 | decoded = Avromatic.messaging.decode(avro_message_value, schema_name: schema_name) 373 | expect(decoded['six_str']).to eq('Foobar') 374 | end 375 | end 376 | 377 | context "custom type in a union" do 378 | let(:values) { { optional_six: 'fOObAR' } } 379 | 380 | before do 381 | Avromatic.register_type('test.six') do |type| 382 | type.from_avro = ->(value) { value.downcase } 383 | type.to_avro = ->(value) { value.capitalize } 384 | end 385 | end 386 | 387 | it "converts when assigning to the model" do 388 | expect(instance.optional_six).to eq('foobar') 389 | end 390 | 391 | it "converts when encoding the value" do 392 | decoded = Avromatic.messaging.decode(avro_message_value, schema_name: schema_name) 393 | expect(decoded['optional_six']).to eq('Foobar') 394 | end 395 | end 396 | 397 | context "custom type for record" do 398 | let(:schema_name) { 'test.with_varchar' } 399 | let(:test_class) do 400 | Avromatic::Model.model(schema_name: schema_name) 401 | end 402 | let(:values) { { str: 'test' } } 403 | 404 | before do 405 | Avromatic.register_type('test.varchar', String) do |type| 406 | type.from_avro = ->(value) do 407 | value.is_a?(String) ? value : value['data'] 408 | end 409 | type.to_avro = ->(value) do 410 | { 'data' => value, 'length' => value.size } 411 | end 412 | end 413 | end 414 | 415 | it "stores the attribute" do 416 | expect(instance.str).to eq('test') 417 | end 418 | 419 | it "converts when encoding the value" do 420 | decoded = Avromatic.messaging.decode(avro_message_value, schema_name: schema_name) 421 | expect(decoded['str']).to eq('length' => 4, 'data' => 'test') 422 | end 423 | 424 | it "converts when assigning to the model" do 425 | decoded = test_class.avro_message_decode(avro_message_value) 426 | expect(decoded.str).to eq('test') 427 | end 428 | end 429 | end 430 | 431 | describe ".register_schemas!" do 432 | let(:registry) { Avromatic.build_schema_registry } 433 | 434 | shared_examples_for "value schema registration" do 435 | it "registers the value schema" do 436 | expect(test_class.register_schemas!).to be_nil 437 | registered = registry.subject_version(test_class.value_avro_schema.fullname) 438 | aggregate_failures do 439 | expect(registered['version']).to eq(1) 440 | expect(registered['schema']).to eq(test_class.value_avro_schema.to_s) 441 | end 442 | end 443 | end 444 | 445 | context "a model without a key" do 446 | let(:test_class) do 447 | Avromatic::Model.model(value_schema_name: 'test.encode_value') 448 | end 449 | 450 | it_behaves_like "value schema registration" 451 | end 452 | 453 | context "a model with a specified subject" do 454 | let(:test_class) do 455 | Avromatic::Model.model(value_schema_name: 'test.encode_value', 456 | value_schema_subject: 'test.encode_value-subject') 457 | end 458 | 459 | it "registers the value schema with the specified subject" do 460 | expect(test_class.register_schemas!).to be_nil 461 | registered = registry.subject_version('test.encode_value-subject') 462 | aggregate_failures do 463 | expect(registered['version']).to eq(1) 464 | expect(registered['schema']).to eq(test_class.value_avro_schema.to_s) 465 | end 466 | end 467 | end 468 | 469 | context "a model with a key and value" do 470 | let(:test_class) do 471 | Avromatic::Model.model(value_schema_name: 'test.encode_value', 472 | key_schema_name: 'test.encode_key') 473 | end 474 | 475 | it_behaves_like "value schema registration" 476 | 477 | it "registers the key schema" do 478 | expect(test_class.register_schemas!).to be_nil 479 | registered = registry.subject_version(test_class.key_avro_schema.fullname) 480 | aggregate_failures do 481 | expect(registered['version']).to eq(1) 482 | expect(registered['schema']).to eq(test_class.key_avro_schema.to_s) 483 | end 484 | end 485 | end 486 | end 487 | end 488 | --------------------------------------------------------------------------------