' },
24 | allow_nil: true
25 |
26 | def compatibility=(value)
27 | super(value.try(:upcase))
28 | end
29 |
30 | def self.global
31 | find_or_create_by!(id: 0) do |config|
32 | config.compatibility = Rails.application.config.x.default_compatibility
33 | end
34 | end
35 |
36 | def update_compatibility!(compatibility)
37 | update!(compatibility: compatibility)
38 | rescue ActiveRecord::RecordInvalid
39 | if errors.key?(:compatibility)
40 | raise Compatibility::InvalidCompatibilityLevelError.new(compatibility)
41 | else
42 | raise
43 | end
44 | end
45 | end
46 |
--------------------------------------------------------------------------------
/bin/docker_start:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env ruby
2 |
3 | require 'uri'
4 | require 'active_record'
5 |
6 | if ENV['AUTO_MIGRATE'] == '1'
7 | pg_uri = URI.parse(ENV['DATABASE_URL'])
8 | port = pg_uri.port || 5432
9 |
10 | # check that we can connect to Postgres
11 | connected = false
12 | exception = nil
13 | connection = nil
14 | 60.times do
15 | begin
16 | puts "Attempting DB connection at #{pg_uri.host}:#{port} with user #{pg_uri.user}"
17 | ActiveRecord::Base.establish_connection(ENV['DATABASE_URL'])
18 | connection = ActiveRecord::Base.connection
19 | connected = true
20 | break
21 | rescue ActiveRecord::NoDatabaseError
22 | # `NoDatabaseError` error is OK, we'll create the DB below
23 | connected = true
24 | break
25 | rescue StandardError => e
26 | exception = e
27 | sleep(2.second)
28 | ensure
29 | connection.close unless connection.nil?
30 | end
31 | end
32 | raise exception unless connected
33 |
34 | # the PG driver is slow to release its port binding: https://www.postgresql.org/message-id/30038.1494916166%40sss.pgh.pa.us
35 | sleep(2.second)
36 |
37 | # create and migrate DB
38 | abort('Unable to migrate DB') unless system('bundle exec rails db:create db:migrate')
39 | end
40 |
41 | exec('bundle exec puma -C config/puma.rb')
42 |
--------------------------------------------------------------------------------
/config/initializers/content_security_policy.rb:
--------------------------------------------------------------------------------
1 | # frozen_string_literal: true
2 |
3 | # Be sure to restart your server when you modify this file.
4 |
5 | # Define an application-wide content security policy
6 | # For further information see the following documentation
7 | # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy
8 |
9 | # Rails.application.config.content_security_policy do |policy|
10 | # policy.default_src :self, :https
11 | # policy.font_src :self, :https, :data
12 | # policy.img_src :self, :https, :data
13 | # policy.object_src :none
14 | # policy.script_src :self, :https
15 | # policy.style_src :self, :https
16 |
17 | # # Specify URI for violation reports
18 | # # policy.report_uri "/csp-violation-report-endpoint"
19 | # end
20 |
21 | # If you are using UJS then enable automatic nonce generation
22 | # Rails.application.config.content_security_policy_nonce_generator = -> request { SecureRandom.base64(16) }
23 |
24 | # Set the nonce only to specific directives
25 | # Rails.application.config.content_security_policy_nonce_directives = %w(script-src)
26 |
27 | # Report CSP violations to a specified URI
28 | # For further information see the following documentation:
29 | # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy-Report-Only
30 | # Rails.application.config.content_security_policy_report_only = true
31 |
--------------------------------------------------------------------------------
/.circleci/config.yml:
--------------------------------------------------------------------------------
1 | version: 2.1
2 | jobs:
3 | build:
4 | docker:
5 | - image: ruby:3.4.3
6 | - image: cimg/postgres:14.7
7 | environment:
8 | POSTGRES_USER: "ubuntu"
9 | POSTGRES_DB: "avro-schema-registry_test"
10 | POSTGRES_HOST_AUTH_METHOD: "trust"
11 | environment:
12 | RACK_ENV: "test"
13 | RAILS_ENV: "test"
14 | CIRCLE_TEST_REPORTS: "test-results"
15 | DB_HOST: "localhost"
16 | DB_USER: "ubuntu"
17 | working_directory: ~/avro-schema-registry
18 | steps:
19 | - checkout
20 | - restore_cache:
21 | keys:
22 | - v1-gems-ruby-3.4.3-{{ checksum "Gemfile.lock" }}
23 | - v1-gems-ruby-3.4.3-
24 | - run:
25 | name: Install Gems
26 | command: |
27 | if ! bundle check --path=vendor/bundle; then
28 | bundle install --path=vendor/bundle --jobs=4 --retry=3
29 | bundle clean
30 | fi
31 | - save_cache:
32 | key: v1-gems-ruby-3.4.3-{{ checksum "Gemfile.lock" }}
33 | paths:
34 | - "vendor/bundle"
35 | - run:
36 | name: Run Rubocop
37 | command: bundle exec rubocop --config .rubocop.yml
38 | - run:
39 | name: Run Tests
40 | command: |
41 | bundle exec rspec --format RspecJunitFormatter --out $CIRCLE_TEST_REPORTS/rspec/junit.xml --format progress spec
42 | - store_test_results:
43 | path: "test-results"
44 | - store_artifacts:
45 | path: "log"
46 | workflows:
47 | build:
48 | jobs:
49 | - build
50 |
--------------------------------------------------------------------------------
/app/api/compatibility_api.rb:
--------------------------------------------------------------------------------
1 | # frozen_string_literal: true
2 |
3 | class CompatibilityAPI < Grape::API
4 | include BaseAPI
5 |
6 | helpers ::Helpers::SchemaVersionHelper
7 |
8 | rescue_from SchemaRegistry::InvalidAvroSchemaError do
9 | invalid_avro_schema!
10 | end
11 |
12 | rescue_from Grape::Exceptions::ValidationErrors do |e|
13 | if e.errors.keys == [['with_compatibility']]
14 | invalid_compatibility_level!
15 | else
16 | error!({ message: e.message }, 422)
17 | end
18 | end
19 |
20 | rescue_from :all do
21 | server_error!
22 | end
23 |
24 | desc 'Test input schema against a particular version of a subject\’s schema '\
25 | 'for compatibility'
26 | params do
27 | requires :subject, type: String, desc: 'Subject name'
28 | requires :version_id, types: [Integer, String],
29 | desc: 'Version of the schema registered under the subject'
30 | requires :schema, type: String, desc: 'New Avro schema to compare against'
31 | optional :with_compatibility, type: String, desc: 'The compatibility level to test',
32 | values: Compatibility::Constants::VALUES
33 | end
34 | post '/subjects/:subject/versions/:version_id', requirements: { subject: Subject::NAME_REGEXP } do
35 | with_schema_version(params[:subject], params[:version_id]) do |schema_version|
36 | status 200
37 | {
38 | is_compatible: SchemaRegistry.compatible?(params[:schema],
39 | version: schema_version,
40 | compatibility: params[:with_compatibility])
41 | }
42 | end
43 | end
44 | end
45 |
--------------------------------------------------------------------------------
/spec/requests/schema_api_spec.rb:
--------------------------------------------------------------------------------
1 | # frozen_string_literal: true
2 |
3 | require 'rails_helper'
4 |
5 | describe SchemaAPI do
6 | describe "GET /schemas/ids/:id" do
7 | let(:schema) { create(:schema) }
8 |
9 | it_behaves_like "a secure endpoint" do
10 | let(:action) { unauthorized_get("/schemas/ids/#{schema.id}") }
11 | end
12 |
13 | context "content type" do
14 | include_examples "content type", :get do
15 | let(:path) { "/schemas/ids/#{schema.id}" }
16 | let(:expected) do
17 | { schema: schema.json }.to_json
18 | end
19 | end
20 | end
21 |
22 | context "when the schema is found" do
23 | let(:expected) do
24 | { schema: schema.json }.to_json
25 | end
26 |
27 | it "returns the schema" do
28 | get("/schemas/ids/#{schema.id}")
29 | expect(response).to be_ok
30 | expect(response.body).to be_json_eql(expected)
31 | end
32 |
33 | it_behaves_like "a cached endpoint" do
34 | let(:action) { get("/schemas/ids/#{schema.id}") }
35 | end
36 | end
37 |
38 | context "when the schema is not found" do
39 | let(:schema_id) { create(:schema).id + 1 }
40 | let(:expected) do
41 | {
42 | error_code: 40403,
43 | message: 'Schema not found'
44 | }.to_json
45 | end
46 | let(:action) { get("/schemas/ids/#{schema_id}") }
47 |
48 | it "returns a not found response" do
49 | action
50 | expect(response).to be_not_found
51 | expect(response.body).to be_json_eql(expected)
52 | end
53 |
54 | it_behaves_like "an error that cannot be cached"
55 | end
56 | end
57 | end
58 |
--------------------------------------------------------------------------------
/spec/support/contexts/content_type_context.rb:
--------------------------------------------------------------------------------
1 | # frozen_string_literal: true
2 |
3 | # This shared example expects the following variables to be defined
4 | # @param :expected [JSON] JSON response
5 | # @param :path [String] Request path
6 | # @param :params [Hash, NilClass] Optional params for request
7 | shared_examples_for "content type" do |verb|
8 | let(:_params) { defined?(params) ? params : nil }
9 |
10 | it "returns a schema registry v1 response" do
11 | send(verb, path, params: _params)
12 | expect(response.headers['Content-Type']).to eq(BaseAPI::SCHEMA_REGISTRY_V1_CONTENT_TYPE)
13 | end
14 |
15 | it "accepts schema registry v1 json requests" do
16 | send(verb, path, params: _params,
17 | headers: {
18 | 'ACCEPT' => BaseAPI::SCHEMA_REGISTRY_V1_CONTENT_TYPE,
19 | 'CONTENT_TYPE' => BaseAPI::SCHEMA_REGISTRY_V1_CONTENT_TYPE
20 | })
21 | expect(response).to be_ok
22 | expect(response.body).to be_json_eql(expected)
23 | end
24 |
25 | it "accepts schema registry json requests" do
26 | send(verb, path, params: _params,
27 | headers: {
28 | 'ACCEPT' => BaseAPI::SCHEMA_REGISTRY_CONTENT_TYPE,
29 | 'CONTENT_TYPE' => BaseAPI::SCHEMA_REGISTRY_CONTENT_TYPE
30 | })
31 | expect(response).to be_ok
32 | expect(response.body).to be_json_eql(expected)
33 | end
34 |
35 | it "accepts json requests" do
36 | send(verb, path, params: _params,
37 | headers: {
38 | 'ACCEPT' => 'application/json',
39 | 'CONTENT_TYPE' => 'application/json'
40 | })
41 | expect(response).to be_ok
42 | expect(response.body).to be_json_eql(expected)
43 | end
44 | end
45 |
--------------------------------------------------------------------------------
/public/500.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | We're sorry, but something went wrong (500)
5 |
6 |
55 |
56 |
57 |
58 |
59 |
60 |
61 |
We're sorry, but something went wrong.
62 |
63 |
If you are the application owner check the logs for more information.
64 |
60 |
61 |
The change you wanted was rejected.
62 |
Maybe you tried to change something you didn't have access to.
63 |
64 |
If you are the application owner check the logs for more information.
65 |
60 |
61 |
The page you were looking for doesn't exist.
62 |
You may have mistyped the address or the page may have moved.
63 |
64 |
If you are the application owner check the logs for more information.
65 |
avro-schema-registry
37 |
38 |
39 | A service for storing and retrieving versions of Avro schemas
40 |
41 |
42 |
43 | Description
44 |
45 | This service implements the API of the Confluent Schema Registry.
46 |
47 | Imitation is the sincerest form of flattery, right?
48 |
49 |
50 |
51 | This service is hosted by Salsify
52 | for anyone to use.
53 |
54 |
55 | Host your own copy of this service on Heroku:
56 |
57 |
58 | 
59 |
60 |
61 |
62 | Details on the implementation can be found here.
63 |
64 | Background on how we use this service is described on our blog.
65 |
66 | Security
67 | This service is public. Uploaded schemas are visible to anyone.
68 |
69 | The password for the service is 'avro'. The username for requests is not
70 | verified but you should choose something to identify yourself.
71 |
72 | Data stored by this service may be cleared at any time.
73 |
74 |
75 |
--------------------------------------------------------------------------------
/app/models/schema_registry.rb:
--------------------------------------------------------------------------------
1 | # frozen_string_literal: true
2 |
3 | module SchemaRegistry
4 | InvalidAvroSchemaError = Class.new(StandardError)
5 | IncompatibleAvroSchemaError = Class.new(StandardError)
6 |
7 | extend self
8 |
9 | def compatible?(new_json, version:, compatibility: nil)
10 | compatibility ||= version.subject.config.try(:compatibility) || Compatibility.global
11 |
12 | if Compatibility::Constants::TRANSITIVE_VALUES.include?(compatibility)
13 | check_all_versions(compatibility, new_json, version.subject)
14 | else
15 | check_single_version(compatibility, version.schema.json, new_json)
16 | end
17 | end
18 |
19 | def compatible!(new_json, version:, compatibility: nil)
20 | raise IncompatibleAvroSchemaError unless compatible?(new_json, version: version, compatibility: compatibility)
21 | end
22 |
23 | private
24 |
25 | # If a version/fork of avro that defines Avro::SchemaCompability is
26 | # present, use the full compatibility check, otherwise fall back to
27 | # match_schemas.
28 | def check(readers_schema, writers_schema)
29 | if defined?(Avro::SchemaCompatibility)
30 | Avro::SchemaCompatibility.can_read?(writers_schema, readers_schema)
31 | else
32 | Avro::IO::DatumReader.match_schemas(writers_schema, readers_schema)
33 | end
34 | end
35 |
36 | def check_single_version(compatibility, old_json, new_json)
37 | old_schema = Schemas::Parse.call(old_json)
38 | new_schema = Schemas::Parse.call(new_json)
39 |
40 | case compatibility
41 | when Compatibility::Constants::NONE
42 | true
43 | when Compatibility::Constants::BACKWARD
44 | check(new_schema, old_schema)
45 | when Compatibility::Constants::FORWARD
46 | check(old_schema, new_schema)
47 | when Compatibility::Constants::FULL, Compatibility::Constants::BOTH
48 | check(old_schema, new_schema) && check(new_schema, old_schema)
49 | end
50 | end
51 |
52 | def check_all_versions(compatibility, new_json, subject)
53 | new_schema = Schemas::Parse.call(new_json)
54 | json_schemas = subject.versions.order(version: :desc).joins(:schema).pluck('version', 'schemas.json').map(&:last)
55 |
56 | case compatibility
57 | when Compatibility::Constants::BACKWARD_TRANSITIVE
58 | json_schemas.all? { |json| check(new_schema, Schemas::Parse.call(json)) }
59 | when Compatibility::Constants::FORWARD_TRANSITIVE
60 | json_schemas.all? { |json| check(Schemas::Parse.call(json), new_schema) }
61 | when Compatibility::Constants::FULL_TRANSITIVE
62 | json_schemas.all? do |json|
63 | old_schema = Schemas::Parse.call(json)
64 | check(old_schema, new_schema) && check(new_schema, old_schema)
65 | end
66 | end
67 | end
68 | end
69 |
--------------------------------------------------------------------------------
/CHANGELOG.md:
--------------------------------------------------------------------------------
1 | # avro-schema-registry
2 |
3 | ## v0.13.4
4 | - Allow hyphens in subject names.
5 |
6 | ## v0.13.3
7 | - Upgrade to avro-patches 0.3.4
8 |
9 | ## v0.13.2
10 | - Missing database no longer causes `docker_start` script to fail in auto-migrate mode.
11 |
12 | ## v0.13.1
13 | - Missing database no longer causes `docker_start` script to fail in auto-migrate mode.
14 |
15 | ## v0.13.0
16 | - Add auto-migrate and waiting for Postgres to the Docker container.
17 |
18 | ## v0.12.1
19 | - Upgrade to Puma 3.11.3.
20 |
21 | ## v0.12.0
22 | - Upgrade to Ruby 2.4.2.
23 | - Upgrade to Rails 5.1.
24 | - Allow default compatibility level to be set via environment variable and
25 | change the default for non-production environments.
26 | - Include Dockerfile.
27 |
28 | ## v0.11.0
29 | - Change the default fingerprint version to '2'. Set `FINGERPRINT_VERSION=1`
30 | before upgrading if you have not migrated to fingerprint version 2.
31 |
32 | ## v0.10.0
33 | - Use `avro-patches` instead of `avro-salsify-fork`.
34 |
35 | ## v0.9.1
36 | - Support dual deploys.
37 |
38 | ## v0.9.0
39 | - Add read-only mode for the application.
40 |
41 | ## v0.8.1
42 | - Reverse the definition of BACKWARD and FORWARD compatibility levels.
43 | Previous releases had these backwards.
44 | Note: The compatibility level is NOT changed for existing configs in the
45 | database. Current compatibility levels should be reviewed to ensure that the
46 | expectation is consistent with the description here:
47 | http://docs.confluent.io/3.2.0/avro.html#schema-evolution.
48 |
49 | ## v0.8.0
50 | - Allow the compatibility level to use while registering a schema to be specified,
51 | and a compatibility level to set for the subject after registration.
52 |
53 | ## v0.7.0
54 | - Allow the compatibility level to be specified in the Compatibility API.
55 |
56 | ## v0.6.2
57 | - Update `cache_all_requests` task to use the resolution fingerprint.
58 |
59 | ## v0.6.1
60 | - Only define rake task to cache requests in the development environment.
61 |
62 | ## v0.6.0
63 | - Introduce fingerprint2 based on `avro-resolution_canonical_form`.
64 | This is a compatibility breaking change and requires a sequence of upgrade steps.
65 | - Fix fingerprint endpoint when using an integer fingerprint.
66 |
67 | ## v0.5.0
68 | - Fix Config API for subjects.
69 | - Add endpoint to get schema id by fingerprint.
70 | - Upgrade to Rails 5.0.
71 | - Add rake task to issue cacheable requests for all schemas.
72 |
73 | ## v0.4.0
74 | - Use `avro-salsify-fork` v1.9.0.3.
75 | - Implement full schema compatibility check and transitive options from
76 | Confluent Schema Registry API v3.1.0.
77 |
78 | ## v0.3.0
79 | - Use `heroku_rails_deploy` gem.
80 |
81 | ## v0.2.0
82 | - Update to Rails 4.2.7.
83 | - Use `avro-salsify-fork`.
84 | - Add `bin/deploy` script.
85 |
86 | ## v0.1.0
87 | - Initial release
88 |
--------------------------------------------------------------------------------
/public/success.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | avro-schema-registry Success!
5 |
34 |
52 |
53 |
54 | Congratulations!
55 |
56 |
57 | You've successfully deployed the avro-schema-registry!
58 |
59 |
60 |
61 | Next Steps
62 |
63 | - Check the Heroku config for your app to find the value set for the
64 | SCHEMA_REGISTRY_PASSWORD. You'll need this value to make requests.
65 |
66 | - Use a client to make some requests:
67 |
81 |
82 |
83 | - Learn about the API in the Confluent Schema Registry
84 | documentation.
85 |
86 |
87 | - See the implementation in the avro-schema-registry repo.
88 |
89 | - Learn more about why we built it on the Salsify blog.
90 |
91 |
92 |
93 |
--------------------------------------------------------------------------------
/config/environments/production.rb:
--------------------------------------------------------------------------------
1 | # frozen_string_literal: true
2 |
3 | Rails.application.configure do
4 | # Settings specified here will take precedence over those in config/application.rb.
5 |
6 | # Code is not reloaded between requests.
7 | config.cache_classes = true
8 |
9 | # Eager load code on boot. This eager loads most of Rails and
10 | # your application in memory, allowing both threaded web servers
11 | # and those relying on copy on write to perform better.
12 | # Rake tasks automatically ignore this option for performance.
13 | config.eager_load = true
14 |
15 | # Full error reports are disabled and caching is turned on.
16 | config.consider_all_requests_local = false
17 | config.action_controller.perform_caching = true
18 |
19 | # Attempt to read encrypted secrets from `config/secrets.yml.enc`.
20 | # Requires an encryption key in `ENV["RAILS_MASTER_KEY"]` or
21 | # `config/secrets.yml.key`.
22 | # config.read_encrypted_secrets = true
23 |
24 | # Disable serving static files from the `/public` folder by default since
25 | # Apache or NGINX already handles this.
26 | config.public_file_server.enabled = ENV['RAILS_SERVE_STATIC_FILES'].present?
27 |
28 | # Enable serving of images, stylesheets, and JavaScripts from an asset server.
29 | # config.action_controller.asset_host = 'http://assets.example.com'
30 |
31 | # Specifies the header that your server uses for sending files.
32 | # config.action_dispatch.x_sendfile_header = 'X-Sendfile' # for Apache
33 | # config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for NGINX
34 |
35 | # Mount Action Cable outside main process or domain
36 | # config.action_cable.mount_path = nil
37 | # config.action_cable.url = 'wss://example.com/cable'
38 | # config.action_cable.allowed_request_origins = [ 'http://example.com', /http:\/\/example.*/ ]
39 |
40 | # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies.
41 | config.force_ssl = ActiveRecord::Type::Boolean.new.cast(ENV.fetch('FORCE_SSL', true))
42 |
43 | # Use the lowest log level to ensure availability of diagnostic information
44 | # when problems arise.
45 | config.log_level = ENV.fetch('LOG_LEVEL', :info).to_sym
46 |
47 | # Prepend all log lines with the following tags.
48 | # config.log_tags = [:request_id]
49 |
50 | # Use a different cache store in production.
51 | # config.cache_store = :mem_cache_store
52 |
53 | # Use a real queuing backend for Active Job (and separate queues per environment)
54 | # config.active_job.queue_adapter = :resque
55 | # config.active_job.queue_name_prefix = "avro_schema_registry_#{Rails.env}"
56 |
57 | # Enable locale fallbacks for I18n (makes lookups for any locale fall back to
58 | # the I18n.default_locale when a translation cannot be found).
59 | config.i18n.fallbacks = true
60 |
61 | # Send deprecation notices to registered listeners.
62 | config.active_support.deprecation = :notify
63 |
64 | # Use default logging formatter so that PID and timestamp are not suppressed.
65 | config.log_formatter = ::Logger::Formatter.new
66 |
67 | # Use a different logger for distributed setups.
68 | # require 'syslog/logger'
69 | # config.logger = ActiveSupport::TaggedLogging.new(Syslog::Logger.new 'app-name')
70 |
71 | if ENV['RAILS_LOG_TO_STDOUT'].present?
72 | logger = ActiveSupport::Logger.new($stdout)
73 | logger.formatter = config.log_formatter
74 | config.logger = ActiveSupport::TaggedLogging.new(logger)
75 | end
76 |
77 | # Do not dump schema after migrations.
78 | config.active_record.dump_schema_after_migration = false
79 |
80 | config.x.app_password = ENV.fetch('SCHEMA_REGISTRY_PASSWORD') unless config.x.disable_password
81 |
82 | # This default differs from the Confluent default of BACKWARD
83 | config.x.default_compatibility = ENV.fetch('DEFAULT_COMPATIBILITY', 'FULL_TRANSITIVE')
84 | end
85 |
--------------------------------------------------------------------------------
/app/models/schemas/register_new_version.rb:
--------------------------------------------------------------------------------
1 | # frozen_string_literal: true
2 |
3 | module Schemas
4 | # This class is called to register a new version of a schema.
5 | # If the schema already exists, then the existing model object is returned.
6 | # Subjects and SchemaVersions are created as necessary.
7 | # Race conditions are protected against by relying on unique indexes on the
8 | # created models.
9 | # If a unique index error is raised, then the operation is retried once.
10 | class RegisterNewVersion
11 | attr_reader :subject_name, :json
12 | attr_accessor :schema
13 |
14 | private_attr_accessor :retried
15 | private_attr_reader :options
16 |
17 | def self.call(...)
18 | new(...).call
19 | end
20 |
21 | def initialize(subject_name, json, **options)
22 | @subject_name = subject_name
23 | @json = json
24 | @options = options
25 | end
26 |
27 | # Retry once to make it easier to handle race conditions on the client,
28 | # i.e. the client should not need to retry.
29 | def call
30 | register_new_version
31 | schema
32 | rescue ActiveRecord::RecordNotUnique
33 | if retried
34 | raise
35 | else
36 | self.retried = true
37 | retry
38 | end
39 | end
40 |
41 | private
42 |
43 | def register_new_version
44 | self.schema = Schema.existing_schema(json)
45 |
46 | if schema
47 | create_new_version
48 | else
49 | create_new_schema
50 | end
51 | end
52 |
53 | def create_new_version
54 | create_version_with_optional_new_subject unless version_exists_for_subject_schema?(schema.id)
55 | end
56 |
57 | def create_new_schema
58 | create_version_with_optional_new_subject do
59 | self.schema = Schema.create!(json: json)
60 | end
61 | end
62 |
63 | def create_version_with_optional_new_subject
64 | latest_version = latest_version_for_subject
65 |
66 | if latest_version.nil?
67 | # Create new subject and version
68 | Subject.transaction do
69 | yield if block_given?
70 | subject = new_subject!(schema.id)
71 | after_compatibility!(subject)
72 | end
73 | else
74 | # Create new schema version for subject
75 | SchemaVersion.transaction do
76 | SchemaRegistry.compatible!(json,
77 | version: latest_version,
78 | compatibility: options[:with_compatibility])
79 | yield if block_given?
80 | schema_version = new_schema_version_for_subject!(schema.id, latest_version)
81 | after_compatibility!(schema_version.subject)
82 | end
83 | end
84 | end
85 |
86 | def new_schema_version_for_subject!(schema_id, previous_version)
87 | SchemaVersion.create!(schema_id: schema_id,
88 | subject_id: previous_version.subject_id,
89 | version: previous_version.version + 1)
90 |
91 | end
92 |
93 | def new_subject!(schema_id)
94 | subject = Subject.create!(name: subject_name)
95 | subject.versions.create!(schema_id: schema_id)
96 | subject
97 | end
98 |
99 | def latest_version_for_subject
100 | SchemaVersion.eager_load(:schema, subject: [:config])
101 | .latest_for_subject_name(subject_name).first
102 | end
103 |
104 | def version_exists_for_subject_schema?(schema_id)
105 | SchemaVersion.for_schema(schema_id)
106 | .for_subject_name(subject_name).first.present?
107 |
108 | end
109 |
110 | def after_compatibility!(subject)
111 | compatibility = options[:after_compatibility]
112 | if compatibility
113 | subject.create_config! unless subject.config
114 | subject.config.update_compatibility!(compatibility)
115 | end
116 | end
117 | end
118 | end
119 |
--------------------------------------------------------------------------------
/spec/models/schema_spec.rb:
--------------------------------------------------------------------------------
1 | # frozen_string_literal: true
2 |
3 | describe Schema do
4 | before do
5 | allow(Rails.configuration.x).to receive(:fingerprint_version).and_return(fingerprint_version)
6 | end
7 |
8 | describe '#save' do
9 | let(:schema) { build(:schema) }
10 |
11 | before do
12 | schema.save!
13 | end
14 |
15 | context "when fingerprint_version is '1'" do
16 | let(:fingerprint_version) { '1' }
17 |
18 | it "sets fingerprint" do
19 | expect(schema.fingerprint).to be_present
20 | end
21 |
22 | it "does not set fingerprint2" do
23 | expect(schema.fingerprint2).to be_nil
24 | end
25 | end
26 |
27 | shared_examples_for "it sets both fingerprints" do
28 | it "sets fingerprint" do
29 | expect(schema.fingerprint).to be_present
30 | end
31 |
32 | it "sets fingerprint2" do
33 | expect(schema.fingerprint2).to be_present
34 | end
35 | end
36 |
37 | context "when fingerprint_version is '2'" do
38 | let(:fingerprint_version) { '2' }
39 |
40 | it_behaves_like "it sets both fingerprints"
41 | end
42 |
43 | context "when fingerprint_version is 'all'" do
44 | let(:fingerprint_version) { 'all' }
45 |
46 | it_behaves_like "it sets both fingerprints"
47 | end
48 | end
49 |
50 | describe ".existing_schema" do
51 | let(:schema1) { create(:schema) }
52 | let(:schema2) { create(:schema) }
53 | let(:json) { build(:schema).json }
54 | let(:fingerprint_v1) { Schemas::FingerprintGenerator.generate_v1(json) }
55 | let(:fingerprint_v2) { Schemas::FingerprintGenerator.generate_v2(json) }
56 |
57 | # `Schema` is an immutable model, so we need raw SQL
58 | def update_schema_fingerprints(schema:, fingerprint:, fingerprint2:)
59 | params = [
60 | 'UPDATE schemas SET fingerprint = ?, fingerprint2 = ? WHERE id = ?',
61 | fingerprint,
62 | fingerprint2,
63 | schema.id
64 | ]
65 | params = ActiveRecord::Base.sanitize_sql(params)
66 | ActiveRecord::Base.connection.execute(params)
67 | end
68 |
69 | before do
70 | update_schema_fingerprints(
71 | schema: schema1,
72 | fingerprint: fingerprint_v1,
73 | fingerprint2: Schemas::FingerprintGenerator.generate_v2(schema1.json)
74 | )
75 |
76 | update_schema_fingerprints(
77 | schema: schema2,
78 | fingerprint: Schemas::FingerprintGenerator.generate_v1(schema2.json),
79 | fingerprint2: fingerprint_v2
80 | )
81 | end
82 |
83 | context "when fingerprint_version is '1'" do
84 | let(:fingerprint_version) { '1' }
85 |
86 | it "finds the existing schema by fingerprint v1" do
87 | expect(Schema.existing_schema(json)).to eq(schema1)
88 | end
89 |
90 | context "when there is no schema matching fingerprint v1" do
91 | before do
92 | ActiveRecord::Base.connection.execute("DELETE FROM schemas where id = #{schema1.id}")
93 | end
94 |
95 | it "returns nil" do
96 | expect(Schema.existing_schema(json)).to be_nil
97 | end
98 | end
99 | end
100 |
101 | context "when fingerprint_version is '2'" do
102 | let(:fingerprint_version) { '2' }
103 |
104 | it "finds the existing schema by fingerprint v2" do
105 | expect(Schema.existing_schema(json)).to eq(schema2)
106 | end
107 |
108 | context "when there is no schema matching fingerprint v2" do
109 | before do
110 | ActiveRecord::Base.connection.execute("DELETE FROM schemas where id = #{schema2.id}")
111 | end
112 |
113 | it "returns nil" do
114 | expect(Schema.existing_schema(json)).to be_nil
115 | end
116 | end
117 | end
118 |
119 | context "when fingerprint_version is 'all'" do
120 | let(:fingerprint_version) { 'all' }
121 |
122 | it "finds the existing schema by fingerprint v2" do
123 | expect(Schema.existing_schema(json)).to eq(schema2)
124 | end
125 |
126 | context "when there is no schema matching fingerprint v2" do
127 | before do
128 | ActiveRecord::Base.connection.execute("DELETE FROM schemas where id = #{schema2.id}")
129 | end
130 |
131 | it "finds the existing schema by fingerprint v1" do
132 | expect(Schema.existing_schema(json)).to eq(schema1)
133 | end
134 | end
135 | end
136 | end
137 | end
138 |
--------------------------------------------------------------------------------
/app/api/subject_api.rb:
--------------------------------------------------------------------------------
1 | # frozen_string_literal: true
2 |
3 | class SubjectAPI < Grape::API
4 | include BaseAPI
5 |
6 | INTEGER_FINGERPRINT_REGEXP = /^[0-9]+$/
7 |
8 | rescue_from ActiveRecord::RecordNotFound do
9 | subject_not_found!
10 | end
11 |
12 | rescue_from SchemaRegistry::InvalidAvroSchemaError do
13 | invalid_avro_schema!
14 | end
15 |
16 | rescue_from SchemaRegistry::IncompatibleAvroSchemaError do
17 | incompatible_avro_schema!
18 | end
19 |
20 | rescue_from :all do
21 | server_error!
22 | end
23 |
24 | helpers ::Helpers::SchemaVersionHelper
25 | helpers ::Helpers::CacheHelper
26 |
27 | desc 'Get a list of registered subjects'
28 | get '/' do
29 | Subject.order(:name).pluck(:name)
30 | end
31 |
32 | params { requires :name, type: String, desc: 'Subject name' }
33 | segment ':name', requirements: { name: Subject::NAME_REGEXP } do
34 | desc 'Get a list of versions registered under the specified subject.'
35 | get :versions do
36 | SchemaVersion.for_subject_name(params[:name])
37 | .order(:version)
38 | .pluck(:version)
39 | .tap do |result|
40 | raise ActiveRecord::RecordNotFound if result.empty?
41 | end
42 | end
43 |
44 | params do
45 | requires :version_id, types: [Integer, String],
46 | desc: 'version of the schema registered under the subject'
47 | end
48 | namespace '/versions/:version_id' do
49 | desc 'Get a specific version of the schema registered under this subject'
50 | get do
51 | with_schema_version(params[:name], params[:version_id]) do |schema_version|
52 | {
53 | id: schema_version.schema_id,
54 | name: schema_version.subject.name,
55 | version: schema_version.version,
56 | schema: schema_version.schema.json
57 | }
58 | end
59 | end
60 |
61 | desc 'Get the Avro schema for the specified version of this subject. Only the unescaped schema is returned.'
62 | get '/schema' do
63 | with_schema_version(params[:name], params[:version_id]) do |schema_version|
64 | JSON.parse(schema_version.schema.json)
65 | end
66 | end
67 | end
68 |
69 | desc 'Get the id of a specific version of the schema registered under a subject'
70 | params do
71 | requires :fingerprint, types: [String, Integer], desc: 'SHA256 fingerprint'
72 | end
73 | get '/fingerprints/:fingerprint' do
74 | fingerprint = if INTEGER_FINGERPRINT_REGEXP.match?(params[:fingerprint])
75 | params[:fingerprint].to_i.to_s(16)
76 | else
77 | params[:fingerprint]
78 | end
79 |
80 | schema_version = SchemaVersion.select(:schema_id)
81 | .for_subject_name(params[:name])
82 | .for_schema_fingerprint(fingerprint).first
83 |
84 | if schema_version
85 | cache_response!
86 | { id: schema_version.schema_id }
87 | else
88 | schema_not_found!
89 | end
90 | end
91 |
92 | desc 'Register a new schema under the specified subject'
93 | params do
94 | requires :schema, type: String, desc: 'The Avro schema string'
95 | optional :with_compatibility, type: String,
96 | desc: 'The compatibility level to use while registering the schema',
97 | values: Compatibility::Constants::VALUES
98 | optional :after_compatibility, type: String,
99 | desc: 'The compatibility level to set after registering the schema',
100 | values: Compatibility::Constants::VALUES
101 | end
102 | post '/versions' do
103 | read_only_mode! if Rails.configuration.x.read_only_mode
104 |
105 | error!({ message: 'Schema registration is disabled' }, 503) if Rails.configuration.x.disable_schema_registration
106 |
107 | new_schema_options = declared(params).slice(:with_compatibility, :after_compatibility).symbolize_keys
108 | schema = Schemas::RegisterNewVersion.call(params[:name], params[:schema], **new_schema_options)
109 | status 200
110 | { id: schema.id }
111 | end
112 |
113 | desc 'Check if a schema has been registered under the specified subject'
114 | params do
115 | requires :schema, type: String, desc: 'The Avro schema string'
116 | end
117 | post '/' do
118 | schema_version = SchemaVersion.eager_load(:schema, :subject)
119 | .for_subject_name(params[:name])
120 | .for_schema_json(params[:schema]).first
121 | if schema_version
122 | status 200
123 | {
124 | subject: schema_version.subject.name,
125 | id: schema_version.schema_id,
126 | version: schema_version.version,
127 | schema: schema_version.schema.json
128 | }
129 | elsif Subject.where(name: params[:name]).exists?
130 | schema_not_found!
131 | else
132 | subject_not_found!
133 | end
134 | end
135 |
136 | end
137 | end
138 |
--------------------------------------------------------------------------------
/spec/requests/compatibility_api_spec.rb:
--------------------------------------------------------------------------------
1 | # frozen_string_literal: true
2 |
3 | describe CompatibilityAPI do
4 |
5 | describe "POST /compatibility/subjects/:subject/versions/:version" do
6 | let(:version) { create(:schema_version) }
7 | let(:subject_name) { version.subject.name }
8 | let(:schema) do
9 | JSON.parse(version.schema.json).tap do |avro|
10 | avro['fields'] << { name: :new, type: :string, default: '' }
11 | end.to_json
12 | end
13 | let(:compatibility) { nil }
14 |
15 | it "tests compatibility of the schema with the version of the subject's schema" do
16 | allow(SchemaRegistry).to receive(:compatible?).with(schema, version: version, compatibility: nil)
17 | .and_return(true)
18 | post("/compatibility/subjects/#{subject_name}/versions/#{version.version}", params: { schema: schema })
19 | expect(response).to be_ok
20 | expect(response.body).to be_json_eql({ is_compatible: true }.to_json)
21 | end
22 |
23 | context "when compatibility is set for the subject" do
24 | let(:compatibility) { 'FORWARD' }
25 |
26 | before { version.subject.create_config!(compatibility: compatibility) }
27 |
28 | it "tests compatibility of the schema with the version of the subject's schema" do
29 | allow(SchemaRegistry).to receive(:compatible?).with(schema, version: version, compatibility: nil)
30 | .and_return(true)
31 | post("/compatibility/subjects/#{subject_name}/versions/#{version.version}", params: { schema: schema })
32 | expect(response).to be_ok
33 | expect(response.body).to be_json_eql({ is_compatible: true }.to_json)
34 | end
35 | end
36 |
37 | context "when compatibility is specified in the reqest" do
38 | let(:compatibility) { 'BACKWARD' }
39 |
40 | it "tests compatibility of the schema using the specified compatibility level" do
41 | allow(SchemaRegistry).to receive(:compatible?).with(schema, version: version, compatibility: compatibility)
42 | .and_return(true)
43 | post("/compatibility/subjects/#{subject_name}/versions/#{version.version}",
44 | params: { schema: schema, with_compatibility: compatibility })
45 | expect(response).to be_ok
46 | expect(response.body).to be_json_eql({ is_compatible: true }.to_json)
47 | end
48 | end
49 |
50 | context "when the version is specified as latest" do
51 | let(:second_version) { create(:schema_version, subject: version.subject, version: 2) }
52 |
53 | it "tests compatibility of the schema with the latest version of the subject's schema" do
54 | allow(SchemaRegistry).to receive(:compatible?).with(schema, version: second_version, compatibility: nil)
55 | .and_return(true)
56 | post("/compatibility/subjects/#{subject_name}/versions/latest", params: { schema: schema })
57 | expect(response).to be_ok
58 | expect(response.body).to be_json_eql({ is_compatible: true }.to_json)
59 | end
60 | end
61 |
62 | it_behaves_like "a secure endpoint" do
63 | let(:action) do
64 | unauthorized_post("/compatibility/subjects/#{subject_name}/versions/#{version.version}",
65 | params: { schema: schema })
66 | end
67 | end
68 |
69 | context "when the schema is invalid" do
70 | let(:schema) do
71 | # invalid due to missing record name
72 | { type: :record, fields: [{ name: :i, type: :int }] }.to_json
73 | end
74 |
75 | it "returns an invalid schema error" do
76 | post("/compatibility/subjects/#{subject_name}/versions/latest", params: { schema: schema })
77 | expect(status).to eq(422)
78 | expect(response.body).to be_json_eql(SchemaRegistry::Errors::INVALID_AVRO_SCHEMA.to_json)
79 | end
80 | end
81 |
82 | context "when the subject is not found" do
83 | it "returns a subject not found error" do
84 | post('/compatibility/subjects/example.not_found/versions/latest', params: { schema: schema })
85 | expect(response).to be_not_found
86 | expect(response.body).to be_json_eql(SchemaRegistry::Errors::SUBJECT_NOT_FOUND.to_json)
87 | end
88 | end
89 |
90 | context "when the version is not found" do
91 | it "returns a version not found error" do
92 | post("/compatibility/subjects/#{subject_name}/versions/2", params: { schema: schema })
93 | expect(response).to be_not_found
94 | expect(response.body).to be_json_eql(SchemaRegistry::Errors::VERSION_NOT_FOUND.to_json)
95 | end
96 | end
97 |
98 | context "when the compatibility level is invalid" do
99 | it "returns an invalid compatibility level error" do
100 | post("/compatibility/subjects/#{subject_name}/versions/latest",
101 | params: { schema: schema, with_compatibility: 'SAME' })
102 | expect(status).to eq(422)
103 | expect(response.body).to be_json_eql(SchemaRegistry::Errors::INVALID_COMPATIBILITY_LEVEL.to_json)
104 | end
105 | end
106 |
107 | context "when the schema is not specified" do
108 | it "returns an invalid compatibility level error" do
109 | post("/compatibility/subjects/#{subject_name}/versions/latest",
110 | params: { with_compatibility: 'BOTH' })
111 | expect(status).to eq(422)
112 | expect(response.body).to be_json_eql({ message: 'schema is missing' }.to_json)
113 | end
114 | end
115 | end
116 | end
117 |
--------------------------------------------------------------------------------
/spec/requests/config_api_spec.rb:
--------------------------------------------------------------------------------
1 | # frozen_string_literal: true
2 |
3 | describe ConfigAPI do
4 | let(:expected) do
5 | { compatibility: compatibility }.to_json
6 | end
7 |
8 | describe "GET /config" do
9 | let(:compatibility) { Compatibility.global }
10 |
11 | it "returns the global compatibility level" do
12 | get('/config')
13 | expect(response).to be_ok
14 | expect(response.body).to be_json_eql(expected)
15 | end
16 |
17 | it_behaves_like "a secure endpoint" do
18 | let(:action) { unauthorized_get('/config') }
19 | end
20 | end
21 |
22 | describe "PUT /config" do
23 | let(:compatibility) { 'FORWARD' }
24 |
25 | it "changes the global compatibility level" do
26 | put('/config', params: { compatibility: compatibility })
27 | expect(response).to be_ok
28 | expect(response.body).to be_json_eql({ compatibility: compatibility }.to_json)
29 | expect(Config.global.compatibility).to eq(compatibility)
30 | end
31 |
32 | context "when the app is in read-only mode" do
33 | before do
34 | allow(Rails.configuration.x).to receive(:read_only_mode).and_return(true)
35 | end
36 |
37 | it "returns an error" do
38 | put('/config', params: { compatibility: compatibility })
39 | expect(response.status).to eq(403)
40 | expect(response.body).to be_json_eql({ message: 'Running in read-only mode' }.to_json)
41 | end
42 | end
43 |
44 | it_behaves_like "a secure endpoint" do
45 | let(:action) { unauthorized_put('/config', params: { compatibility: compatibility }) }
46 | end
47 |
48 | context "when the compatibility value is not uppercase" do
49 | it "changes the global compatibility level" do
50 | put('/config', params: { compatibility: compatibility.downcase })
51 | expect(response).to be_ok
52 | expect(response.body).to be_json_eql({ compatibility: compatibility }.to_json)
53 | expect(Config.global.compatibility).to eq(compatibility)
54 | end
55 | end
56 |
57 | context "when the compatibility level is invalid" do
58 | let(:compatibility) { 'BACK' }
59 |
60 | it "returns an unprocessable entity error" do
61 | put('/config', params: { compatibility: compatibility })
62 | expect(status).to eq(422)
63 | expect(response.body)
64 | .to be_json_eql(SchemaRegistry::Errors::INVALID_COMPATIBILITY_LEVEL.to_json)
65 | end
66 | end
67 | end
68 |
69 | describe "GET /config/:subject" do
70 | let(:schema_subject) { create(:subject) }
71 | let(:compatibility) { Compatibility.global }
72 |
73 | it_behaves_like "a secure endpoint" do
74 | let(:action) { unauthorized_get("/config/#{schema_subject.name}") }
75 | end
76 |
77 | context "when compatibility is set for the subject" do
78 | before { schema_subject.create_config.update_compatibility!(compatibility) }
79 |
80 | it "returns the compatibility level for the subject" do
81 | get("/config/#{schema_subject.name}")
82 | expect(response).to be_ok
83 | expect(response.body).to be_json_eql(expected)
84 | end
85 | end
86 |
87 | context "when compatibility has not been set for the subject" do
88 | let(:compatibility) { nil }
89 |
90 | it "returns null" do
91 | get("/config/#{schema_subject.name}")
92 | expect(response).to be_ok
93 | expect(response.body).to be_json_eql(expected)
94 | end
95 | end
96 |
97 | context "when the subject does not exist" do
98 | let(:name) { 'example.does_not_exist' }
99 |
100 | it "returns a not found error" do
101 | get("/config/#{name}")
102 | expect(response).to be_not_found
103 | expect(response.body).to be_json_eql(SchemaRegistry::Errors::SUBJECT_NOT_FOUND.to_json)
104 | end
105 | end
106 | end
107 |
108 | describe "PUT /config/:subject" do
109 | let(:schema_subject) { create(:subject) }
110 | let(:compatibility) { 'BACKWARD' }
111 |
112 | it "updates the compatibility level on the subject" do
113 | put("/config/#{schema_subject.name}", params: { compatibility: compatibility })
114 | expect(response).to be_ok
115 | expect(response.body).to be_json_eql(expected)
116 | expect(schema_subject.config.compatibility).to eq(compatibility)
117 | end
118 |
119 | context "when the app is in read-only mode" do
120 | before do
121 | allow(Rails.configuration.x).to receive(:read_only_mode).and_return(true)
122 | end
123 |
124 | it "returns an error" do
125 | put("/config/#{schema_subject.name}", params: { compatibility: compatibility })
126 | expect(response.status).to eq(403)
127 | expect(response.body).to be_json_eql({ message: 'Running in read-only mode' }.to_json)
128 | end
129 | end
130 |
131 | context "when the subject already has a compatibility level set" do
132 | let(:original_compatibility) { 'FORWARD' }
133 |
134 | before { schema_subject.create_config!(compatibility: original_compatibility) }
135 |
136 | it "updates the compatibility level on the subject" do
137 | put("/config/#{schema_subject.name}", params: { compatibility: compatibility })
138 | expect(response).to be_ok
139 | expect(response.body).to be_json_eql(expected)
140 | expect(schema_subject.config.reload.compatibility).to eq(compatibility)
141 | end
142 | end
143 |
144 | context "when the compatibility level is not uppercase" do
145 | it "updates the compatibility level on the subject" do
146 | put("/config/#{schema_subject.name}", params: { compatibility: compatibility.downcase })
147 | expect(response).to be_ok
148 | expect(response.body).to be_json_eql(expected)
149 | expect(schema_subject.config.compatibility).to eq(compatibility)
150 | end
151 | end
152 |
153 | it_behaves_like "a secure endpoint" do
154 | let(:action) do
155 | unauthorized_put("/config/#{schema_subject.name}", params: { compatibility: compatibility })
156 | end
157 | end
158 |
159 | context "when the subject does not exist" do
160 | let(:name) { 'example.does_not_exist' }
161 |
162 | it "returns a not found error" do
163 | put("/config/#{name}", params: { compatibility: compatibility })
164 | expect(response).to be_not_found
165 | expect(response.body).to be_json_eql(SchemaRegistry::Errors::SUBJECT_NOT_FOUND.to_json)
166 | end
167 | end
168 |
169 | context "when the compatibility level is invalid" do
170 | let(:compatibility) { 'FOO' }
171 |
172 | it "returns an unprocessable entity error" do
173 | put("/config/#{schema_subject.name}", params: { compatibility: compatibility })
174 | expect(status).to eq(422)
175 | expect(response.body)
176 | .to be_json_eql(SchemaRegistry::Errors::INVALID_COMPATIBILITY_LEVEL.to_json)
177 | end
178 | end
179 | end
180 | end
181 |
--------------------------------------------------------------------------------
/spec/schema_registry_spec.rb:
--------------------------------------------------------------------------------
1 | # frozen_string_literal: true
2 |
3 | describe SchemaRegistry do
4 | let(:json_hash) do
5 | {
6 | type: :record,
7 | name: :rec,
8 | fields: [
9 | { name: :field1, type: :string, default: '' },
10 | { name: :field2, type: :string }
11 | ]
12 | }
13 | end
14 | let(:old_json) { json_hash.to_json }
15 | let(:schema) { create(:schema, json: old_json) }
16 | let(:version) { create(:schema_version, schema: schema) }
17 | let(:new_json) { build(:schema).json }
18 | let(:backward_json) do
19 | # BACKWARD compatible - can read old schema
20 | # It ignores the removed field in the old schema
21 | hash = JSON.parse(old_json)
22 | hash['fields'].pop
23 | hash.to_json
24 | end
25 | let(:forward_json) do
26 | # FORWARD compatible - can be read by old schema
27 | # The old schema ignores the new field.
28 | hash = JSON.parse(old_json)
29 | hash['fields'] << { name: :extra, type: :string }
30 | hash.to_json
31 | end
32 | let(:full_json) do
33 | # FULL compatible
34 | hash = JSON.parse(old_json)
35 | # removed required field with default
36 | hash['fields'].shift
37 | # add required field with default
38 | hash['fields'] << { name: :extra, type: :string, default: '' }
39 | hash.to_json
40 | end
41 | let(:old_schema) { Avro::Schema.parse(old_json) }
42 | let(:new_schema) { Avro::Schema.parse(new_json) }
43 |
44 | before do
45 | create(:config, subject_id: version.subject_id, compatibility: compatibility) if compatibility
46 | end
47 |
48 | describe ".compatible?" do
49 | let(:compatibility) { 'FULL_TRANSITIVE' }
50 |
51 | subject(:check) { described_class.compatible?(new_json, version: version) }
52 |
53 | before do
54 | allow(Avro::SchemaCompatibility).to receive(:can_read?).and_call_original
55 | end
56 |
57 | it "allows compatibility to be specified" do
58 | described_class.compatible?(new_json, version: version, compatibility: 'BACKWARD')
59 | expect(Avro::SchemaCompatibility).to have_received(:can_read?).with(new_schema, old_schema)
60 | end
61 |
62 | context "when compatibility is nil" do
63 | let(:compatibility) { nil }
64 |
65 | it "uses the global Compatibility level" do
66 | allow(Compatibility).to receive(:global).and_call_original
67 | check
68 | expect(Compatibility).to have_received(:global)
69 | end
70 | end
71 |
72 | context "when compatibility is NONE" do
73 | let(:compatibility) { 'NONE' }
74 |
75 | it "does not perform any check" do
76 | expect(check).to eq(true)
77 | expect(Avro::SchemaCompatibility).not_to have_received(:can_read?)
78 | end
79 | end
80 |
81 | context "when compatibility is BACKWARD" do
82 | let(:compatibility) { 'BACKWARD' }
83 |
84 | it "performs a check with the old schema as the readers schema" do
85 | check
86 | expect(Avro::SchemaCompatibility).to have_received(:can_read?).with(new_schema, old_schema)
87 | end
88 |
89 | it "returns false for a forward compatible schema" do
90 | expect(described_class.compatible?(forward_json, version: version)).to eq(false)
91 | end
92 |
93 | it "returns true for a backward compatible schema" do
94 | expect(described_class.compatible?(backward_json, version: version)).to eq(true)
95 | end
96 | end
97 |
98 | context "when compatibility is FORWARD" do
99 | let(:compatibility) { 'FORWARD' }
100 |
101 | it "performs a check with the new schema as the readers schema" do
102 | check
103 | expect(Avro::SchemaCompatibility).to have_received(:can_read?).with(old_schema, new_schema)
104 | end
105 |
106 | it "returns true for a forward compatible schema" do
107 | expect(described_class.compatible?(forward_json, version: version)).to eq(true)
108 | end
109 |
110 | it "returns false for a backward compatible schema" do
111 | expect(described_class.compatible?(backward_json, version: version)).to eq(false)
112 | end
113 | end
114 |
115 | context "when compatibility is BOTH (deprecated)" do
116 | let(:compatibility) { 'BOTH' }
117 |
118 | it "performs a check with each schema as the readers schema" do
119 | check
120 | expect(Avro::SchemaCompatibility).to have_received(:can_read?).with(new_schema, old_schema).once
121 | expect(Avro::SchemaCompatibility).to have_received(:can_read?).with(old_schema, new_schema).once
122 | end
123 |
124 | it "returns false" do
125 | expect(check).to eq(false)
126 | end
127 | end
128 |
129 | context "when compatibility is FULL" do
130 | let(:compatibility) { 'FULL' }
131 |
132 | it "performs a check with each schema as the readers schema" do
133 | check
134 | expect(Avro::SchemaCompatibility).to have_received(:can_read?).with(new_schema, old_schema)
135 | expect(Avro::SchemaCompatibility).to have_received(:can_read?).with(old_schema, new_schema)
136 | end
137 |
138 | it "returns false for a forward compatible schema" do
139 | expect(described_class.compatible?(forward_json, version: version)).to eq(false)
140 | end
141 |
142 | it "returns false for a backward compatible schema" do
143 | expect(described_class.compatible?(backward_json, version: version)).to eq(false)
144 | end
145 |
146 | it "returns true for a fully compatible schema" do
147 | expect(described_class.compatible?(full_json, version: version)).to eq(true)
148 | end
149 | end
150 |
151 | context "transitive checks" do
152 | let(:registry_subject) { version.subject }
153 | let!(:second_version) { create(:schema_version, subject: registry_subject, version: 2) }
154 | let(:second_schema) { Avro::Schema.parse(second_version.schema.json) }
155 | let(:can_read_args) { [] }
156 |
157 | before do
158 | # rspec checks for ordered calls with have_received were not working so
159 | # expectations are checked directly based on captured args
160 | allow(Avro::SchemaCompatibility).to receive(:can_read?) do |*args|
161 | can_read_args << args
162 | true
163 | end
164 | end
165 |
166 | context "when compatibility is BACKWARD_TRANSITIVE" do
167 | let(:compatibility) { 'BACKWARD_TRANSITIVE' }
168 |
169 | it "performs a check with all schemas as the readers schema" do
170 | check
171 | expect(can_read_args.first).to eq([new_schema, second_schema])
172 | expect(can_read_args.second).to eq([new_schema, old_schema])
173 | end
174 | end
175 |
176 | context "when compatibility is FORWARD_TRANSITIVE" do
177 | let(:compatibility) { 'FORWARD_TRANSITIVE' }
178 |
179 | it "performs a check with all schemas as the writers schema" do
180 | check
181 | expect(can_read_args.first).to eq([second_schema, new_schema])
182 | expect(can_read_args.second).to eq([old_schema, new_schema])
183 | end
184 | end
185 |
186 | context "when compatibility is FULL_TRANSITIVE" do
187 | let(:compatibility) { 'FULL_TRANSITIVE' }
188 |
189 | it "performs a check with all schemas as the writers schema" do
190 | check
191 | expect(can_read_args.first).to eq([new_schema, second_schema])
192 | expect(can_read_args.second).to eq([second_schema, new_schema])
193 | expect(can_read_args.third).to eq([new_schema, old_schema])
194 | expect(can_read_args.fourth).to eq([old_schema, new_schema])
195 | end
196 | end
197 | end
198 |
199 | context "server error" do
200 | let(:compatibility) { 'BOTH' }
201 |
202 | let(:old_json) do
203 | {
204 | type: 'record',
205 | name: 'event',
206 | fields: [
207 | {
208 | name: 'attribute',
209 | type: {
210 | type: 'record',
211 | name: 'reference',
212 | fields: [
213 | {
214 | name: 'id',
215 | type: 'string'
216 | }
217 | ]
218 | }
219 | }
220 | ]
221 | }.to_json
222 | end
223 |
224 | let(:new_json) do
225 | {
226 | type: 'record',
227 | name: 'event',
228 | fields: [
229 | {
230 | name: 'attribute',
231 | type: [
232 | 'null',
233 | {
234 | type: 'record',
235 | name: 'reference',
236 | fields: [
237 | {
238 | name: 'id',
239 | type: 'string'
240 | }
241 | ]
242 | }
243 | ],
244 | default: nil
245 | }
246 | ]
247 | }.to_json
248 | end
249 |
250 | it "returns false" do
251 | expect(check).to eq(false)
252 | end
253 |
254 | end
255 | end
256 |
257 | describe ".compatible!" do
258 | subject(:check) { described_class.compatible!(new_json, version: version) }
259 |
260 | before do
261 | allow(described_class).to receive(:compatible?)
262 | .with(new_json, version: version, compatibility: compatibility).and_return(compatible)
263 | end
264 |
265 | context "when the compatibility level is specified" do
266 | let(:compatibility) { 'FORWARD' }
267 | let(:compatible) { true }
268 |
269 | it "checks compatibility using the specified level" do
270 | expect do
271 | described_class.compatible!(new_json, version: version, compatibility: compatibility)
272 | end.not_to raise_error
273 | end
274 | end
275 |
276 | context "when schemas are compatible" do
277 | let(:compatibility) { nil }
278 | let(:compatible) { true }
279 |
280 | it "does not raise an error" do
281 | expect { check }.not_to raise_error
282 | end
283 | end
284 |
285 | context "when schemas are incompatible" do
286 | let(:compatibility) { nil }
287 | let(:compatible) { false }
288 |
289 | it "raises IncompatibleAvroSchemaError" do
290 | expect { check }.to raise_error(SchemaRegistry::IncompatibleAvroSchemaError)
291 | end
292 | end
293 | end
294 | end
295 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # avro-schema-registry
2 |
3 | [](https://heroku.com/deploy)
4 |
5 | [][circleci]
6 |
7 | [circleci]: https://circleci.com/gh/salsify/avro-schema-registry
8 |
9 | A service for storing and retrieving versions of Avro schemas.
10 |
11 | Schema versions stored by the service are assigned an id. These ids can be
12 | embedded in messages published to Kafka avoiding the need to send the full
13 | schema with each message.
14 |
15 | ## Upgrading to v0.11.0
16 |
17 | v0.11.0 changes the default fingerprint version to 2. Set `FINGERPRINT_VERSION=1`
18 | before upgrading if you have not migrated to fingerprint version 2.
19 |
20 | ## Upgrading to v0.6.0
21 |
22 | There is a compatibility break when upgrading to v0.6.0 due to the way that
23 | fingerprints are generated. Prior to v0.6.0 fingerprints were generated based
24 | on the Parsing Canonical Form for Avro schemas. This does not take into account
25 | attributes such as `default` that are used during schema resolution and for
26 | compatibility checking. The new fingerprint is based on [avro-resolution_canonical_form](https://github.com/salsify/avro-resolution_canonical_form).
27 |
28 | To upgrade:
29 |
30 | 1. Set `FINGERPRINT_VERSION=1` and `DISABLE_SCHEMA_REGISTRATION=true` in the
31 | environment for the application, and restart the application.
32 | 2. Deploy v0.6.0 and run migrations to create and populate the new `fingerprint2`
33 | column.
34 | 3. If NOT using the fingerprint endpoint, move to the final step.
35 | 4. Set `FINGERPRINT_VERSION=all`, unset `DISABLE_SCHEMA_REGISTRATION`, and restart the application.
36 | 5. Update all clients to use the v2 fingerprint.
37 | 6. Set `FINGERPRINT_VERSION=2` and unset `DISABLE_SCHEMA_REGISTRATION` (if still set) and
38 | restart the application.
39 |
40 | At some point in the future the original `fingerprint` column will be removed.
41 |
42 | ## Overview
43 |
44 | This application provides the same API as the Confluent
45 | [Schema Registry](http://docs.confluent.io/3.1.0/schema-registry/docs/api.html).
46 |
47 | The service is implemented as a Rails 7.2 application and stores Avro schemas in
48 | Postgres. The API is implemented using [Grape](https://github.com/ruby-grape/grape).
49 |
50 | ### Why?
51 |
52 | The Confluent Schema Registry has been reimplemented because the original
53 | implementation uses Kafka to store schemas. We view the messages that pass
54 | through Kafka as more ephemeral and want the flexibility to change how we host Kafka.
55 | In the future we may also apply per-subject permissions to the Avro schemas that
56 | are stored by the registry.
57 |
58 | ## Extensions
59 |
60 | In addition to the Confluent Schema Registry API, this application provides some
61 | extensions.
62 |
63 | ### Schema ID by Fingerprint
64 |
65 | There is an endpoint that can be used to determine by
66 | fingerprint if a schema is already registered for a subject.
67 |
68 | This endpoint provides a success response that can be cached indefinitely since
69 | the id for a schema will not change once it is registered for a subject.
70 |
71 | `GET /subjects/(string: subject)/fingerprints/(:fingerprint)`
72 |
73 | Get the id of the schema registered for the subject by fingerprint. The
74 | fingerprint may either be the hex string or the integer value produced by the
75 | [SHA256 fingerprint](http://avro.apache.org/docs/1.8.1/spec.html#Schema+Fingerprints).
76 |
77 | **Parameters:**
78 | - **subject** (_string_) - Name of the subject that the schema is registered under
79 | - **fingerprint** (_string_ or _integer_) - SHA256 fingerprint for the schema
80 |
81 | **Response JSON Object:**
82 | - **id** (_int_) - Globally unique identifier of the schema
83 |
84 | **Status Codes:**
85 | - 404 Not Found - Error Code 40403 - Schema not found
86 | - 500 Internal Server Error - Error code 50001 - Error in the backend datastore
87 |
88 | **Example Request:**
89 | ```
90 | GET /subjects/test/fingerprints/90479eea876f5d6c8482b5b9e3e865ff1c0931c1bfe0adb44c41d628fd20989c HTTP/1.1
91 | Host: schemaregistry.example.com
92 | Accept: application/vnd.schemaregistry.v1+json, application/vnd.schemaregistry+json, application/json
93 | ```
94 |
95 | **Example response:**
96 | ```
97 | HTTP/1.1 200 OK
98 | Content-Type: application/vnd.schemaregistry.v1+json
99 |
100 | {"id":1}
101 | ```
102 |
103 | ### Test Compatibility with a Specified Level
104 |
105 | The Compatibility API is extended to support a `with_compatibility` parameter
106 | that controls the level used for the compatibility check against the specified
107 | schema version.
108 |
109 | When `with_compatibility` is specified, it overrides any configuration for the
110 | subject and the global configuration.
111 |
112 | **Example Request:**
113 | ```
114 | POST /subjects/test/versions/latest HTTP/1.1
115 | Host: schemaregistry.example.com
116 | Accept: application/vnd.schemaregistry.v1+json, application/vnd.schemaregistry+json, application/json
117 |
118 | {
119 | "schema": "{ ... }",
120 | "with_compatibility": "BACKWARD"
121 | }
122 | ```
123 |
124 | ### Register A New Schema With Specified Compatibility Levels
125 |
126 | The Subject API is extended to support a `with_compatibility` parameter that
127 | controls the level used for the compatibility check when registering a new
128 | schema for a subject.
129 |
130 | An `after_compatibility` parameter is also supported to set a new compatibility
131 | level for the subject after a new schema version is registered. This option is
132 | ignored if no new version is created.
133 |
134 | **Example Request:**
135 | ```
136 | POST /subjects/test/versions HTTP/1.1
137 | Host: schemaregistry.example.com
138 | Accept: application/vnd.schemaregistry.v1+json, application/vnd.schemaregistry+json, application/json
139 |
140 | {
141 | "schema": "{ ... }",
142 | "with_compatibility": "NONE",
143 | "after_compatibility": "BACKwARD"
144 | }
145 | ```
146 |
147 | ## Setup
148 |
149 | The application is written using Ruby 3.4.3. Start the service using the following
150 | steps:
151 |
152 | ```bash
153 | git clone git@github.com:salsify/avro-schema-registry.git
154 | cd avro-schema-registry
155 | bundle install
156 | bin/rake db:setup
157 | bin/rails s
158 | ```
159 |
160 | By default the service runs on port 21000.
161 |
162 | ## Deployment
163 |
164 | Salsify hosts a public instance of this application at
165 | [avro-schema-registry.salsify.com](https://avro-schema-registry.salsify.com) that
166 | anyone can experiment with, just please don't rely on it for production!
167 |
168 | There is also a button above to easily deploy your own copy of the application to Heroku.
169 |
170 | ### Docker
171 |
172 | A Dockerfile is provided to run the application within a container. To
173 | build the image, navigate to the root of the repo and run `docker build`:
174 |
175 | ```bash
176 | docker build . -t avro-schema-registry
177 | ```
178 |
179 | The container is built for the `production` environment, so you'll
180 | need to pass in a few environment flags to run it:
181 |
182 | ```bash
183 | docker run -p 5000:5000 -d \
184 | -e DATABASE_URL=postgresql://user:pass@host/dbname \
185 | -e FORCE_SSL=false \
186 | -e SECRET_KEY_BASE=supersecret \
187 | -e SCHEMA_REGISTRY_PASSWORD=avro \
188 | avro-schema-registry
189 | ```
190 |
191 | If you also want to run PostgreSQL in a container, you can link the two containers:
192 |
193 | ```bash
194 | docker run --name avro-postgres -d \
195 | -e POSTGRES_PASSWORD=avro \
196 | -e POSTGRES_USER=avro \
197 | postgres:9.6
198 |
199 | docker run --name avro-schema-registry --link avro-postgres:postgres -p 5000:5000 -d \
200 | -e DATABASE_URL=postgresql://avro:avro@postgres/avro \
201 | -e FORCE_SSL=false \
202 | -e SECRET_KEY_BASE=supersecret \
203 | -e SCHEMA_REGISTRY_PASSWORD=avro \
204 | avro-schema-registry
205 | ```
206 |
207 | To setup the database the first time you run the app, you can call
208 | `rails db:setup` from within the container:
209 |
210 | ```bash
211 | docker exec avro-schema-registry bundle exec rails db:setup
212 | ```
213 |
214 | Alternatively, your can pass `-e AUTO_MIGRATE=1` to your `docker run` command to have the
215 | container automatically create and migrate the database schema.
216 |
217 | ## Security
218 |
219 | The service is secured using HTTP Basic authentication and should be used with
220 | SSL. The default password for the service is 'avro' but it can be set via
221 | the environment as `SCHEMA_REGISTRY_PASSWORD`.
222 |
223 | Authentication can be disabled by setting `DISABLE_PASSWORD` to 'true' in the
224 | environment.
225 |
226 | ## Caching
227 |
228 | When the environment variable `ALLOW_RESPONSE_CACHING` is set to `true` then the
229 | service sets headers to allow responses from the following endpoints to be cached:
230 |
231 | - `GET /schemas/(int: id)`
232 | - `GET /subjects/(string: subject)/fingerprints/(:fingerprint)`
233 |
234 | By default, responses for these endpoints are allowed to be cached for 30 days.
235 | This max age can be configured by setting `CACHE_MAX_AGE` to a number of seconds.
236 |
237 | To populate a cache of the responses from these endpoints, the application
238 | contains a rake task that can be run in a development environment to iterate
239 | through all registered schemas and issue the cacheable requests:
240 |
241 | ```bash
242 | rake cache_all_requests registry_url=https://anything:avro@registry.example.com
243 | ```
244 |
245 | ## Usage
246 |
247 | For more details on the REST API see the Confluent
248 | [documentation](http://docs.confluent.io/3.1.0/schema-registry/docs/api.html).
249 |
250 | A [client](https://github.com/dasch/avro_turf/blob/master/lib/avro_turf/confluent_schema_registry.rb)
251 | (see [AvroTurf](https://github.com/dasch/avro_turf)) can be used to
252 | communicate with the service:
253 |
254 | ```ruby
255 | require 'avro_turf'
256 | require 'avro_turf/confluent_schema_registry'
257 |
258 | url = 'https://anything:avro@registry.example.com'
259 | client = AvroTurf::ConfluentSchemaRegistry.new(url)
260 |
261 | # registering a new schema returns an id
262 | id = client.register('test_subject', avro_json_schema)
263 | # => 99
264 |
265 | # attempting to register the same schema for a subject returns the existing id
266 | id = client.register('test_subject', avro_json_schema)
267 | # => 99
268 |
269 | # the JSON for an Avro schema can be fetched by id
270 | client.fetch(id)
271 | # => avro_json_schema
272 | ```
273 |
274 | ## Tests
275 |
276 | Tests for the application can be run using:
277 |
278 | ```
279 | bundle exec rspec
280 | ```
281 |
282 | ## License
283 |
284 | This code is available as open source under the terms of the
285 | [MIT License](http://opensource.org/licenses/MIT).
286 |
287 | ## Contributing
288 |
289 | Bug reports and pull requests are welcome on GitHub at
290 | https://github.com/salsify/avro-schema-registry.
291 |
--------------------------------------------------------------------------------
/Gemfile.lock:
--------------------------------------------------------------------------------
1 | GEM
2 | remote: https://rubygems.org/
3 | specs:
4 | actioncable (7.2.2.2)
5 | actionpack (= 7.2.2.2)
6 | activesupport (= 7.2.2.2)
7 | nio4r (~> 2.0)
8 | websocket-driver (>= 0.6.1)
9 | zeitwerk (~> 2.6)
10 | actionmailbox (7.2.2.2)
11 | actionpack (= 7.2.2.2)
12 | activejob (= 7.2.2.2)
13 | activerecord (= 7.2.2.2)
14 | activestorage (= 7.2.2.2)
15 | activesupport (= 7.2.2.2)
16 | mail (>= 2.8.0)
17 | actionmailer (7.2.2.2)
18 | actionpack (= 7.2.2.2)
19 | actionview (= 7.2.2.2)
20 | activejob (= 7.2.2.2)
21 | activesupport (= 7.2.2.2)
22 | mail (>= 2.8.0)
23 | rails-dom-testing (~> 2.2)
24 | actionpack (7.2.2.2)
25 | actionview (= 7.2.2.2)
26 | activesupport (= 7.2.2.2)
27 | nokogiri (>= 1.8.5)
28 | racc
29 | rack (>= 2.2.4, < 3.2)
30 | rack-session (>= 1.0.1)
31 | rack-test (>= 0.6.3)
32 | rails-dom-testing (~> 2.2)
33 | rails-html-sanitizer (~> 1.6)
34 | useragent (~> 0.16)
35 | actiontext (7.2.2.2)
36 | actionpack (= 7.2.2.2)
37 | activerecord (= 7.2.2.2)
38 | activestorage (= 7.2.2.2)
39 | activesupport (= 7.2.2.2)
40 | globalid (>= 0.6.0)
41 | nokogiri (>= 1.8.5)
42 | actionview (7.2.2.2)
43 | activesupport (= 7.2.2.2)
44 | builder (~> 3.1)
45 | erubi (~> 1.11)
46 | rails-dom-testing (~> 2.2)
47 | rails-html-sanitizer (~> 1.6)
48 | activejob (7.2.2.2)
49 | activesupport (= 7.2.2.2)
50 | globalid (>= 0.3.6)
51 | activemodel (7.2.2.2)
52 | activesupport (= 7.2.2.2)
53 | activerecord (7.2.2.2)
54 | activemodel (= 7.2.2.2)
55 | activesupport (= 7.2.2.2)
56 | timeout (>= 0.4.0)
57 | activestorage (7.2.2.2)
58 | actionpack (= 7.2.2.2)
59 | activejob (= 7.2.2.2)
60 | activerecord (= 7.2.2.2)
61 | activesupport (= 7.2.2.2)
62 | marcel (~> 1.0)
63 | activesupport (7.2.2.2)
64 | base64
65 | benchmark (>= 0.3)
66 | bigdecimal
67 | concurrent-ruby (~> 1.0, >= 1.3.1)
68 | connection_pool (>= 2.2.5)
69 | drb
70 | i18n (>= 1.6, < 2)
71 | logger (>= 1.4.2)
72 | minitest (>= 5.1)
73 | securerandom (>= 0.3)
74 | tzinfo (~> 2.0, >= 2.0.5)
75 | annotate (3.2.0)
76 | activerecord (>= 3.2, < 8.0)
77 | rake (>= 10.4, < 14.0)
78 | ast (2.4.3)
79 | avro (1.10.2)
80 | multi_json (~> 1)
81 | avro-resolution_canonical_form (0.4.0)
82 | avro (~> 1.10.0)
83 | avro_turf (1.14.0)
84 | avro (>= 1.8.0, < 1.12)
85 | excon (~> 0.104)
86 | base64 (0.3.0)
87 | benchmark (0.4.1)
88 | bigdecimal (3.2.2)
89 | bootsnap (1.18.6)
90 | msgpack (~> 1.2)
91 | bugsnag (6.28.0)
92 | concurrent-ruby (~> 1.0)
93 | builder (3.3.0)
94 | childprocess (5.1.0)
95 | logger (~> 1.5)
96 | concurrent-ruby (1.3.5)
97 | connection_pool (2.5.3)
98 | crass (1.0.6)
99 | date (3.4.1)
100 | diff-lcs (1.6.2)
101 | docile (1.4.1)
102 | dotenv (3.1.8)
103 | dotenv-rails (3.1.8)
104 | dotenv (= 3.1.8)
105 | railties (>= 6.1)
106 | drb (2.2.3)
107 | dry-core (1.1.0)
108 | concurrent-ruby (~> 1.0)
109 | logger
110 | zeitwerk (~> 2.6)
111 | dry-inflector (1.2.0)
112 | dry-logic (1.6.0)
113 | bigdecimal
114 | concurrent-ruby (~> 1.0)
115 | dry-core (~> 1.1)
116 | zeitwerk (~> 2.6)
117 | dry-types (1.8.3)
118 | bigdecimal (~> 3.0)
119 | concurrent-ruby (~> 1.0)
120 | dry-core (~> 1.0)
121 | dry-inflector (~> 1.0)
122 | dry-logic (~> 1.4)
123 | zeitwerk (~> 2.6)
124 | erb (5.0.2)
125 | erubi (1.13.1)
126 | excon (0.112.0)
127 | factory_bot (6.5.4)
128 | activesupport (>= 6.1.0)
129 | factory_bot_rails (6.5.0)
130 | factory_bot (~> 6.5)
131 | railties (>= 6.1.0)
132 | globalid (1.2.1)
133 | activesupport (>= 6.1)
134 | grape (2.2.0)
135 | activesupport (>= 6)
136 | dry-types (>= 1.1)
137 | mustermann-grape (~> 1.1.0)
138 | rack (>= 2)
139 | zeitwerk
140 | heroku_rails_deploy (0.4.5)
141 | private_attr
142 | rails
143 | i18n (1.14.7)
144 | concurrent-ruby (~> 1.0)
145 | iniparse (1.5.0)
146 | io-console (0.8.1)
147 | irb (1.15.2)
148 | pp (>= 0.6.0)
149 | rdoc (>= 4.0.0)
150 | reline (>= 0.4.2)
151 | json (2.13.2)
152 | json_spec (1.1.5)
153 | multi_json (~> 1.0)
154 | rspec (>= 2.0, < 4.0)
155 | language_server-protocol (3.17.0.5)
156 | lint_roller (1.1.0)
157 | logger (1.7.0)
158 | loofah (2.24.1)
159 | crass (~> 1.0.2)
160 | nokogiri (>= 1.12.0)
161 | mail (2.8.1)
162 | mini_mime (>= 0.1.1)
163 | net-imap
164 | net-pop
165 | net-smtp
166 | marcel (1.0.4)
167 | mini_mime (1.1.5)
168 | mini_portile2 (2.8.9)
169 | minitest (5.25.5)
170 | msgpack (1.8.0)
171 | multi_json (1.17.0)
172 | mustermann (3.0.4)
173 | ruby2_keywords (~> 0.0.1)
174 | mustermann-grape (1.1.0)
175 | mustermann (>= 1.0.0)
176 | net-imap (0.5.9)
177 | date
178 | net-protocol
179 | net-pop (0.1.2)
180 | net-protocol
181 | net-protocol (0.2.2)
182 | timeout
183 | net-smtp (0.5.1)
184 | net-protocol
185 | newrelic_rpm (9.20.0)
186 | nio4r (2.7.4)
187 | nokogiri (1.18.9)
188 | mini_portile2 (~> 2.8.2)
189 | racc (~> 1.4)
190 | overcommit (0.68.0)
191 | childprocess (>= 0.6.3, < 6)
192 | iniparse (~> 1.4)
193 | rexml (>= 3.3.9)
194 | parallel (1.27.0)
195 | parser (3.3.9.0)
196 | ast (~> 2.4.1)
197 | racc
198 | pg (1.6.1)
199 | pp (0.6.2)
200 | prettyprint
201 | prettyprint (0.2.0)
202 | prism (1.4.0)
203 | private_attr (2.0.0)
204 | psych (5.2.6)
205 | date
206 | stringio
207 | puma (6.6.1)
208 | nio4r (~> 2.0)
209 | racc (1.8.1)
210 | rack (3.1.16)
211 | rack-session (2.1.1)
212 | base64 (>= 0.1.0)
213 | rack (>= 3.0.0)
214 | rack-test (2.2.0)
215 | rack (>= 1.3)
216 | rackup (2.2.1)
217 | rack (>= 3)
218 | rails (7.2.2.2)
219 | actioncable (= 7.2.2.2)
220 | actionmailbox (= 7.2.2.2)
221 | actionmailer (= 7.2.2.2)
222 | actionpack (= 7.2.2.2)
223 | actiontext (= 7.2.2.2)
224 | actionview (= 7.2.2.2)
225 | activejob (= 7.2.2.2)
226 | activemodel (= 7.2.2.2)
227 | activerecord (= 7.2.2.2)
228 | activestorage (= 7.2.2.2)
229 | activesupport (= 7.2.2.2)
230 | bundler (>= 1.15.0)
231 | railties (= 7.2.2.2)
232 | rails-controller-testing (1.0.5)
233 | actionpack (>= 5.0.1.rc1)
234 | actionview (>= 5.0.1.rc1)
235 | activesupport (>= 5.0.1.rc1)
236 | rails-dom-testing (2.3.0)
237 | activesupport (>= 5.0.0)
238 | minitest
239 | nokogiri (>= 1.6)
240 | rails-html-sanitizer (1.6.2)
241 | loofah (~> 2.21)
242 | nokogiri (>= 1.15.7, != 1.16.7, != 1.16.6, != 1.16.5, != 1.16.4, != 1.16.3, != 1.16.2, != 1.16.1, != 1.16.0.rc1, != 1.16.0)
243 | railties (7.2.2.2)
244 | actionpack (= 7.2.2.2)
245 | activesupport (= 7.2.2.2)
246 | irb (~> 1.13)
247 | rackup (>= 1.0.0)
248 | rake (>= 12.2)
249 | thor (~> 1.0, >= 1.2.2)
250 | zeitwerk (~> 2.6)
251 | rainbow (3.1.1)
252 | rake (13.3.0)
253 | rdoc (6.14.2)
254 | erb
255 | psych (>= 4.0.0)
256 | regexp_parser (2.11.0)
257 | reline (0.6.2)
258 | io-console (~> 0.5)
259 | rexml (3.4.1)
260 | rspec (3.13.1)
261 | rspec-core (~> 3.13.0)
262 | rspec-expectations (~> 3.13.0)
263 | rspec-mocks (~> 3.13.0)
264 | rspec-core (3.13.5)
265 | rspec-support (~> 3.13.0)
266 | rspec-expectations (3.13.5)
267 | diff-lcs (>= 1.2.0, < 2.0)
268 | rspec-support (~> 3.13.0)
269 | rspec-mocks (3.13.5)
270 | diff-lcs (>= 1.2.0, < 2.0)
271 | rspec-support (~> 3.13.0)
272 | rspec-rails (8.0.1)
273 | actionpack (>= 7.2)
274 | activesupport (>= 7.2)
275 | railties (>= 7.2)
276 | rspec-core (~> 3.13)
277 | rspec-expectations (~> 3.13)
278 | rspec-mocks (~> 3.13)
279 | rspec-support (~> 3.13)
280 | rspec-support (3.13.4)
281 | rspec_junit_formatter (0.6.0)
282 | rspec-core (>= 2, < 4, != 2.12.0)
283 | rubocop (1.74.0)
284 | json (~> 2.3)
285 | language_server-protocol (~> 3.17.0.2)
286 | lint_roller (~> 1.1.0)
287 | parallel (~> 1.10)
288 | parser (>= 3.3.0.2)
289 | rainbow (>= 2.2.2, < 4.0)
290 | regexp_parser (>= 2.9.3, < 3.0)
291 | rubocop-ast (>= 1.38.0, < 2.0)
292 | ruby-progressbar (~> 1.7)
293 | unicode-display_width (>= 2.4.0, < 4.0)
294 | rubocop-ast (1.46.0)
295 | parser (>= 3.3.7.2)
296 | prism (~> 1.4)
297 | rubocop-performance (1.15.2)
298 | rubocop (>= 1.7.0, < 2.0)
299 | rubocop-ast (>= 0.4.0)
300 | rubocop-rails (2.17.4)
301 | activesupport (>= 4.2.0)
302 | rack (>= 1.1)
303 | rubocop (>= 1.33.0, < 2.0)
304 | rubocop-rake (0.6.0)
305 | rubocop (~> 1.0)
306 | rubocop-rspec (2.16.0)
307 | rubocop (~> 1.33)
308 | ruby-progressbar (1.13.0)
309 | ruby2_keywords (0.0.5)
310 | salsify_rubocop (1.74.0)
311 | rubocop (~> 1.74.0)
312 | rubocop-performance (~> 1.15.2)
313 | rubocop-rails (~> 2.17.4)
314 | rubocop-rake (~> 0.6.0)
315 | rubocop-rspec (~> 2.16.0)
316 | securerandom (0.4.1)
317 | simplecov (0.22.0)
318 | docile (~> 1.1)
319 | simplecov-html (~> 0.11)
320 | simplecov_json_formatter (~> 0.1)
321 | simplecov-html (0.13.2)
322 | simplecov_json_formatter (0.1.4)
323 | stringio (3.1.7)
324 | thor (1.4.0)
325 | timeout (0.4.3)
326 | tzinfo (2.0.6)
327 | concurrent-ruby (~> 1.0)
328 | unicode-display_width (3.1.4)
329 | unicode-emoji (~> 4.0, >= 4.0.4)
330 | unicode-emoji (4.0.4)
331 | useragent (0.16.11)
332 | websocket-driver (0.8.0)
333 | base64
334 | websocket-extensions (>= 0.1.0)
335 | websocket-extensions (0.1.5)
336 | zeitwerk (2.7.3)
337 |
338 | PLATFORMS
339 | ruby
340 |
341 | DEPENDENCIES
342 | annotate
343 | avro (~> 1.10.0)
344 | avro-resolution_canonical_form (>= 0.2.0)
345 | avro_turf (>= 0.8.0)
346 | bootsnap
347 | bugsnag
348 | dotenv-rails
349 | factory_bot_rails
350 | grape
351 | heroku_rails_deploy (>= 0.4.1)
352 | json_spec
353 | newrelic_rpm
354 | overcommit
355 | pg
356 | private_attr
357 | puma (>= 5.6.7)
358 | rails (~> 7.2.2)
359 | rails-controller-testing
360 | rspec-rails
361 | rspec_junit_formatter
362 | salsify_rubocop
363 | simplecov
364 |
365 | RUBY VERSION
366 | ruby 3.4.3p32
367 |
368 | BUNDLED WITH
369 | 2.7.1
370 |
--------------------------------------------------------------------------------
/spec/requests/subject_api_spec.rb:
--------------------------------------------------------------------------------
1 | # frozen_string_literal: true
2 |
3 | describe SubjectAPI do
4 | let(:invalid_json) do
5 | # invalid due to missing record name
6 | { type: :record, fields: [{ name: :i, type: :int }] }.to_json
7 | end
8 |
9 | context "content type" do
10 | # content type is configured at the API level so only one endpoint is
11 | # tested here.
12 | include_examples "content type", :get do
13 | let(:path) { '/subjects' }
14 | let(:expected) { [].to_json }
15 | end
16 |
17 | context "with JSON body" do
18 | include_examples "content type", :post do
19 | let!(:version) { create(:schema_version) }
20 | let(:subject_name) { version.subject.name }
21 | let(:expected) do
22 | {
23 | subject: version.subject.name,
24 | id: version.schema_id,
25 | version: version.version,
26 | schema: version.schema.json
27 | }.to_json
28 | end
29 |
30 | let(:path) { "/subjects/#{subject_name}" }
31 | let(:params) { { schema: version.schema.json } }
32 | end
33 | end
34 | end
35 |
36 | describe "GET /subjects" do
37 | let!(:subjects) { create_list(:subject, 3) }
38 | let(:expected) do
39 | subjects.map(&:name).sort.to_json
40 | end
41 |
42 | it "returns a list of subject names" do
43 | get('/subjects')
44 | expect(response).to be_ok
45 | expect(response.body).to be_json_eql(expected)
46 | end
47 |
48 | it_behaves_like "a secure endpoint" do
49 | let(:action) { unauthorized_get('/subjects') }
50 | end
51 | end
52 |
53 | describe "GET /subjects/:name/versions" do
54 |
55 | context "supported subject names" do
56 | # This is only being tested for one representative route under
57 | # /subjects/:name
58 | shared_examples_for "a supported subject name" do |desc, name|
59 | let(:subject) { create(:subject, name: name) }
60 | let!(:schema_version) { create(:version, subject: subject) }
61 |
62 | it "supports #{desc}" do
63 | get("/subjects/#{subject.name}/versions")
64 | expect(response).to be_ok
65 | expect(response.body).to eq([schema_version.version].to_json)
66 | end
67 | end
68 |
69 | shared_examples_for "an unsupported subject name" do |desc, name|
70 | it "does not support #{desc}" do
71 | expect do
72 | get("/subjects/#{name}/versions")
73 | end.to raise_error(ActionController::RoutingError)
74 | end
75 | end
76 |
77 | it_behaves_like "a supported subject name",
78 | 'a name containing a period',
79 | 'com.example.foo'
80 |
81 | it_behaves_like "a supported subject name",
82 | 'a name beginning with an underscore',
83 | '_underscore'
84 |
85 | it_behaves_like "a supported subject name",
86 | 'a name containing a digit',
87 | 'number5'
88 |
89 | it_behaves_like "a supported subject name",
90 | 'a name containing mixed case',
91 | 'UPPER_lower_0123456789'
92 |
93 | it_behaves_like "a supported subject name",
94 | 'a name containing a hyphen',
95 | 'topic-value'
96 |
97 | it_behaves_like "an unsupported subject name",
98 | 'a name beginning with a digit',
99 | '5alive'
100 |
101 | it_behaves_like "an unsupported subject name",
102 | 'a name beginning with a hyphen',
103 | '-foobar'
104 |
105 | it_behaves_like "an unsupported subject name",
106 | 'a name beginning with a period',
107 | '.com'
108 | end
109 |
110 | context "when the subject exists" do
111 | let(:schema_subject) { create(:subject) }
112 | let!(:schema_versions) { Array.new(1) { create(:version, subject: schema_subject) } }
113 | let(:expected) do
114 | schema_versions.map(&:version).sort.to_json
115 | end
116 |
117 | it "returns a list of the versions" do
118 | get("/subjects/#{schema_subject.name}/versions")
119 | expect(response).to be_ok
120 | expect(response.body).to eq(expected)
121 | end
122 | end
123 |
124 | it_behaves_like "a secure endpoint" do
125 | let(:version) { create(:version) }
126 | let(:action) { unauthorized_get("/subjects/#{version.subject.name}/versions") }
127 | end
128 |
129 | context "when the subject does not exist" do
130 | let(:name) { 'fnord' }
131 |
132 | it "returns a not found error" do
133 | get("/subjects/#{name}/versions")
134 | expect(response).to be_not_found
135 | expect(response.body).to be_json_eql(SchemaRegistry::Errors::SUBJECT_NOT_FOUND.to_json)
136 | end
137 | end
138 | end
139 |
140 | describe "GET /subjects/:name/versions/:version_id" do
141 | it_behaves_like "a secure endpoint" do
142 | let(:version) { create(:schema_version) }
143 | let(:action) do
144 | unauthorized_get("/subjects/#{version.subject.name}/versions/#{version.version}")
145 | end
146 | end
147 |
148 | context "when the subject and version exists" do
149 | let!(:other_schema_version) { create(:schema_version) }
150 | let(:version) { create(:schema_version) }
151 | let(:subject_name) { version.subject.name }
152 | let(:schema) { version.schema }
153 |
154 | let(:expected) do
155 | {
156 | id: version.schema_id,
157 | name: subject_name,
158 | version: version.version,
159 | schema: schema.json
160 | }.to_json
161 | end
162 |
163 | it "returns the schema" do
164 | get("/subjects/#{subject_name}/versions/#{version.version}")
165 | expect(response).to be_ok
166 | expect(JSON.parse(response.body)['id']).to eq(version.schema_id)
167 | expect(response.body).to be_json_eql(expected)
168 | end
169 |
170 | context "when the version is specified as 'latest'" do
171 | it "returns the schema" do
172 | get("/subjects/#{subject_name}/versions/latest")
173 | expect(response).to be_ok
174 | expect(response.body).to be_json_eql(expected)
175 | end
176 | end
177 |
178 | context "when the version is an invalid string" do
179 | it "returns a not found error" do
180 | get("/subjects/#{subject_name}/versions/invalid")
181 | expect(response).to be_not_found
182 | expect(response.body).to be_json_eql(SchemaRegistry::Errors::VERSION_NOT_FOUND.to_json)
183 | end
184 | end
185 | end
186 |
187 | context "when the subject does not exist" do
188 | it "returns a not found error" do
189 | get('/subjects/fnord/versions/latest')
190 | expect(response).to be_not_found
191 | expect(response.body).to be_json_eql(SchemaRegistry::Errors::SUBJECT_NOT_FOUND.to_json)
192 | end
193 | end
194 |
195 | context "when the version does not exist" do
196 | let!(:version) { create(:schema_version) }
197 |
198 | it "returns a not found error" do
199 | get("/subjects/#{version.subject.name}/versions/2")
200 | expect(response).to be_not_found
201 | expect(response.body).to be_json_eql(SchemaRegistry::Errors::VERSION_NOT_FOUND.to_json)
202 | end
203 | end
204 | end
205 |
206 | describe "GET /subjects/:name/versions/:version_id/schema" do
207 | it_behaves_like "a secure endpoint" do
208 | let(:version) { create(:schema_version) }
209 | let(:action) do
210 | unauthorized_get("/subjects/#{version.subject.name}/versions/#{version.version}/schema")
211 | end
212 | end
213 |
214 | context "when the subject and version exists" do
215 | let!(:other_schema_version) { create(:schema_version) }
216 | let(:version) { create(:schema_version) }
217 | let(:subject_name) { version.subject.name }
218 | let(:schema) { version.schema }
219 | let(:expected) { schema.json }
220 |
221 | it "returns the schema" do
222 | get("/subjects/#{subject_name}/versions/#{version.version}/schema")
223 | expect(response).to be_ok
224 | expect(response.body).to be_json_eql(expected)
225 | end
226 |
227 | context "when the version is specified as 'latest'" do
228 | it "returns the schema" do
229 | get("/subjects/#{subject_name}/versions/latest/schema")
230 | expect(response).to be_ok
231 | expect(response.body).to be_json_eql(expected)
232 | end
233 | end
234 |
235 | context "when the version is an invalid string" do
236 | it "returns a not found error" do
237 | get("/subjects/#{subject_name}/versions/invalid/schema")
238 | expect(response).to be_not_found
239 | expect(response.body).to be_json_eql(SchemaRegistry::Errors::VERSION_NOT_FOUND.to_json)
240 | end
241 | end
242 | end
243 |
244 | context "when the subject does not exist" do
245 | it "returns a not found error" do
246 | get('/subjects/fnord/versions/latest/schema')
247 | expect(response).to be_not_found
248 | expect(response.body).to be_json_eql(SchemaRegistry::Errors::SUBJECT_NOT_FOUND.to_json)
249 | end
250 | end
251 |
252 | context "when the version does not exist" do
253 | let!(:version) { create(:schema_version) }
254 |
255 | it "returns a not found error" do
256 | get("/subjects/#{version.subject.name}/versions/2/schema")
257 | expect(response).to be_not_found
258 | expect(response.body).to be_json_eql(SchemaRegistry::Errors::VERSION_NOT_FOUND.to_json)
259 | end
260 | end
261 | end
262 |
263 | describe "GET /subjects/:name/fingerprints/:fingerprint" do
264 | let(:version) { create(:schema_version) }
265 | let(:schema) { version.schema }
266 | let(:other_schema) { build(:schema) }
267 | let(:other_version) { create(:schema_version) }
268 | let(:fingerprint) do
269 | Schemas::FingerprintGenerator.generate_v2(schema.json)
270 | end
271 | let(:other_fingerprint) do
272 | Schemas::FingerprintGenerator.generate_v2(other_schema.json)
273 | end
274 | let(:existing_fingerprint) do
275 | Schemas::FingerprintGenerator.generate_v2(other_version.schema.json)
276 | end
277 |
278 | it_behaves_like "a secure endpoint" do
279 | let(:action) { unauthorized_get("/subjects/#{version.subject.name}/fingerprints/#{fingerprint}") }
280 | end
281 |
282 | context "content type" do
283 | include_examples "content type", :get do
284 | let(:path) { "/subjects/#{version.subject.name}/fingerprints/#{fingerprint}" }
285 | let(:expected) do
286 | { id: schema.id }.to_json
287 | end
288 | end
289 | end
290 |
291 | shared_examples_for "identifying a schema by fingerprint" do
292 | context "when the schema is found" do
293 | let(:expected) do
294 | { id: schema.id }.to_json
295 | end
296 |
297 | it "returns the schema" do
298 | get("/subjects/#{version.subject.name}/fingerprints/#{fingerprint}")
299 | expect(response).to be_ok
300 | expect(response.body).to be_json_eql(expected).including(:id)
301 | end
302 |
303 | it_behaves_like "a cached endpoint" do
304 | let(:action) { get("/subjects/#{version.subject.name}/fingerprints/#{fingerprint}") }
305 | end
306 | end
307 |
308 | context "error cases" do
309 | context "when the schema is not found" do
310 | let(:expected) do
311 | {
312 | error_code: 40403,
313 | message: 'Schema not found'
314 | }.to_json
315 | end
316 | let(:action) { get("/subjects/#{version.subject.name}/fingerprints/#{other_fingerprint}") }
317 |
318 | it "returns a not found response" do
319 | action
320 | expect(response).to be_not_found
321 | expect(response.body).to be_json_eql(expected)
322 | end
323 |
324 | it_behaves_like "an error that cannot be cached"
325 | end
326 |
327 | context "when the schema exists for a different subject" do
328 | let(:expected) do
329 | {
330 | error_code: 40403,
331 | message: 'Schema not found'
332 | }.to_json
333 | end
334 | let(:action) { get("/subjects/#{version.subject.name}/fingerprints/#{existing_fingerprint}") }
335 |
336 | it "returns a not found response" do
337 | action
338 | expect(response).to be_not_found
339 | expect(response.body).to be_json_eql(expected)
340 | end
341 |
342 | it_behaves_like "an error that cannot be cached"
343 | end
344 | end
345 | end
346 |
347 | context "when the fingerprint version is '1'" do
348 | let(:fingerprint) { schema.fingerprint }
349 | let(:other_fingerprint) do
350 | Schemas::FingerprintGenerator.generate_v1(other_schema.json)
351 | end
352 | let(:existing_fingerprint) { other_version.schema.fingerprint }
353 |
354 | before do
355 | allow(Rails.configuration.x).to receive(:fingerprint_version).and_return('1')
356 | end
357 |
358 | context "using a string fingerprint" do
359 | it_behaves_like "identifying a schema by fingerprint"
360 |
361 | context "using a v2 fingerprint" do
362 | let(:fingerprint) { Schemas::FingerprintGenerator.generate_v2(schema.json) }
363 |
364 | it "does not return the schema" do
365 | get("/subjects/#{version.subject.name}/fingerprints/#{fingerprint}")
366 | expect(response).to be_not_found
367 | end
368 | end
369 | end
370 |
371 | context "using an integer fingerprint" do
372 | let(:fingerprint) { super().to_i(16) }
373 | let(:other_fingerprint) { super().to_i(16) }
374 | let(:existing_fingerprint) { super().to_i(16) }
375 |
376 | it_behaves_like "identifying a schema by fingerprint"
377 | end
378 | end
379 |
380 | context "when the fingerprint version is '2'" do
381 | before do
382 | allow(Rails.configuration.x).to receive(:fingerprint_version).and_return('2')
383 | end
384 |
385 | context "using a v1 fingerprint" do
386 | it "does not return the schema" do
387 | get("/subjects/#{version.subject.name}/fingerprints/#{schema.fingerprint}")
388 | expect(response).to be_not_found
389 | end
390 | end
391 |
392 | context "using a v2 fingerprint" do
393 | let(:fingerprint) { schema.fingerprint2 }
394 | let(:other_fingerprint) do
395 | Schemas::FingerprintGenerator.generate_v2(other_schema.json)
396 | end
397 | let(:existing_fingerprint) { other_version.schema.fingerprint2 }
398 |
399 | it_behaves_like "identifying a schema by fingerprint"
400 | end
401 | end
402 |
403 | context "when the fingerprint version is 'all'" do
404 | before do
405 | allow(Rails.configuration.x).to receive(:fingerprint_version).and_return('all')
406 | end
407 |
408 | it_behaves_like "identifying a schema by fingerprint"
409 |
410 | context "using a v2 fingerprint" do
411 | let(:fingerprint) { schema.fingerprint2 }
412 | let(:other_fingerprint) do
413 | Schemas::FingerprintGenerator.generate_v2(other_schema.json)
414 | end
415 | let(:existing_fingerprint) { other_version.schema.fingerprint2 }
416 |
417 | it_behaves_like "identifying a schema by fingerprint"
418 | end
419 | end
420 | end
421 |
422 | describe "POST /subjects/:name/versions" do
423 | it_behaves_like "a secure endpoint" do
424 | let(:version) { create(:version) }
425 | let(:action) do
426 | unauthorized_post("/subjects/#{version.subject.name}/versions",
427 | params: { schema: version.schema.json })
428 | end
429 | end
430 |
431 | context "with an invalid avro schema" do
432 | let(:subject_name) { 'invalid' }
433 |
434 | it "returns an unprocessable entity error" do
435 | post("/subjects/#{subject_name}/versions", params: { schema: invalid_json })
436 | expect(response.status).to eq(422)
437 | expect(response.body).to be_json_eql(SchemaRegistry::Errors::INVALID_AVRO_SCHEMA.to_json)
438 | end
439 | end
440 |
441 | context "when DISABLE_SCHEMA_REGISTRATION is set to 'true'" do
442 | let(:json) { build(:schema).json }
443 | let(:subject_name) { 'new_subject' }
444 |
445 | before do
446 | allow(Rails.configuration.x).to receive(:disable_schema_registration).and_return(true)
447 | end
448 |
449 | it "returns an error" do
450 | post("/subjects/#{subject_name}/versions", params: { schema: json })
451 | expect(response.status).to eq(503)
452 | expect(response.body).to be_json_eql({ message: 'Schema registration is disabled' }.to_json)
453 | end
454 | end
455 |
456 | context "when the app is in read-only mode" do
457 | let(:json) { build(:schema).json }
458 | let(:subject_name) { 'new_subject' }
459 |
460 | before do
461 | allow(Rails.configuration.x).to receive(:read_only_mode).and_return(true)
462 | end
463 |
464 | it "returns an error" do
465 | post("/subjects/#{subject_name}/versions", params: { schema: json })
466 | expect(response.status).to eq(403)
467 | expect(response.body).to be_json_eql({ message: 'Running in read-only mode' }.to_json)
468 | end
469 | end
470 |
471 | context "when the schema and subject do not exist" do
472 | let(:json) { build(:schema).json }
473 | let(:subject_name) { 'new_subject' }
474 |
475 | it "returns the id of the new schema", :aggregate_failures do
476 | expect do
477 | post("/subjects/#{subject_name}/versions", params: { schema: json })
478 | end.to change(Schema, :count).by(1)
479 | expect(response).to be_ok
480 | schema_id = SchemaVersion.latest_for_subject_name(subject_name).first.schema_id
481 | expect(response.body).to be_json_eql({ id: schema_id }.to_json).including(:id)
482 | end
483 |
484 | it "creates a new subject and version" do
485 | expect do
486 | expect do
487 | post("/subjects/#{subject_name}/versions", params: { schema: json })
488 | end.to change(Subject, :count).by(1)
489 | end.to change(SchemaVersion, :count).by(1)
490 | expect(Subject.find_by(name: subject_name)).to be_present
491 | expect(SchemaVersion.latest_for_subject_name(subject_name).first).to be_present
492 | end
493 |
494 | context "when the compatibility level to use after registration is specified" do
495 | let(:after_compatibility) { 'FULL' }
496 |
497 | it "creates the config for the subject", :aggregate_failures do
498 | expect do
499 | post("/subjects/#{subject_name}/versions",
500 | params: { schema: json, after_compatibility: after_compatibility })
501 | end.to change(Config, :count).by(1)
502 | expect(Subject.find_by(name: subject_name).config.compatibility).to eq(after_compatibility)
503 | end
504 | end
505 | end
506 |
507 | context "when the schema is already registered under the subject" do
508 | let!(:version) { create(:schema_version) }
509 | let(:schema_subject) { version.subject }
510 |
511 | it "returns the id of the schema" do
512 | post("/subjects/#{schema_subject.name}/versions", params: { schema: version.schema.json })
513 | expect(response).to be_ok
514 | expect(response.body).to be_json_eql({ id: version.schema_id }.to_json).including(:id)
515 | end
516 |
517 | it "does not create a new version" do
518 | expect do
519 | post("/subjects/#{schema_subject.name}/versions", params: { schema: version.schema.json })
520 | end.not_to change(SchemaVersion, :count)
521 | end
522 |
523 | it "ignores setting a new compatibility level" do
524 | expect do
525 | post("/subjects/#{schema_subject.name}/versions",
526 | params: { schema: version.schema.json, after_compatibility: 'BACKWARD' })
527 | end.not_to change(Config, :count)
528 | end
529 | end
530 |
531 | context "when a previous version of the schema is registered under the subject" do
532 | before do
533 | allow(Rails.application.config.x).to receive(:default_compatibility)
534 | .and_return(Compatibility::Constants::FULL_TRANSITIVE)
535 | end
536 |
537 | let!(:version) { create(:schema_version) }
538 | let(:schema_subject) { version.subject }
539 | let(:json) do
540 | JSON.parse(version.schema.json).tap do |h|
541 | h['fields'] << { type: :string, name: :additional_field, default: '' }
542 | end.to_json
543 | end
544 |
545 | it "returns the id of a new schema" do
546 | expect do
547 | expect do
548 | post("/subjects/#{schema_subject.name}/versions", params: { schema: json })
549 | end.to change(Schema, :count).by(1)
550 | end.to change(SchemaVersion, :count).by(1)
551 |
552 | expect(response).to be_ok
553 | expect(JSON.parse(response.body)['id']).not_to eq(version.schema.id)
554 | end
555 |
556 | context "when the new version of the schema adds a default" do
557 | let(:schema) { create(:schema_without_default) }
558 | let!(:version) { create(:schema_version, schema_id: schema.id) }
559 | let(:json) do
560 | JSON.parse(schema.json).tap do |h|
561 | h['fields'].first.merge!(default: 0)
562 | end.to_json
563 | end
564 |
565 | before do
566 | # only fingerprint version '2' recognizes a default as a change
567 | allow(Rails.configuration.x).to receive(:fingerprint_version).and_return('2')
568 | end
569 |
570 | it "returns the id of a new schema" do
571 | expect do
572 | expect do
573 | post("/subjects/#{schema_subject.name}/versions", params: { schema: json })
574 | end.to change(Schema, :count).by(1)
575 | end.to change(SchemaVersion, :count).by(1)
576 |
577 | expect(response).to be_ok
578 | expect(JSON.parse(response.body)['id']).not_to eq(schema.id)
579 | end
580 | end
581 |
582 | context "when the new version of the schema is incompatible" do
583 | let(:json) { build(:schema).json }
584 |
585 | it "returns an incompatible schema error" do
586 | post("/subjects/#{schema_subject.name}/versions", params: { schema: json })
587 | expect(status).to eq(409)
588 | expect(response.body).to be_json_eql(SchemaRegistry::Errors::INCOMPATIBLE_AVRO_SCHEMA.to_json)
589 | end
590 |
591 | context "when the compatibility level to use during registration is specified" do
592 | it "returns the id of a new schema" do
593 | expect do
594 | expect do
595 | expect do
596 | post("/subjects/#{schema_subject.name}/versions",
597 | params: { schema: json, with_compatibility: 'NONE' })
598 | end.to change(Schema, :count).by(1)
599 | end.to change(SchemaVersion, :count).by(1)
600 | end.not_to change(Config, :count)
601 | end
602 |
603 | context "when a compatibility level to use after registration is specified" do
604 | let(:after_compatibility) { 'FORWARD' }
605 |
606 | it "returns the id of a new schema and updates the config for the subject", :aggregate_failures do
607 | expect do
608 | expect do
609 | expect do
610 | post("/subjects/#{schema_subject.name}/versions",
611 | params: { schema: json, with_compatibility: 'NONE', after_compatibility: after_compatibility })
612 | end.to change(Schema, :count).by(1)
613 | end.to change(SchemaVersion, :count).by(1)
614 | end.to change(Config, :count).by(1)
615 |
616 | expect(Config.find_by(subject_id: schema_subject.id).compatibility).to eq(after_compatibility)
617 | end
618 |
619 | context "when config already exists for the subject" do
620 | let!(:config) { create(:config, subject_id: schema_subject.id, compatibility: 'FULL') }
621 |
622 | it "updates the config for the subject" do
623 | expect do
624 | post("/subjects/#{schema_subject.name}/versions",
625 | params: { schema: json, with_compatibility: 'NONE', after_compatibility: after_compatibility })
626 | end.not_to change(Config, :count)
627 | expect(config.reload.compatibility).to eq(after_compatibility)
628 | end
629 | end
630 | end
631 | end
632 | end
633 | end
634 |
635 | context "when the schema is already registered under a different subject" do
636 | let!(:version) { create(:schema_version) }
637 | let(:original_subject) { version.subject }
638 |
639 | context "when the subject does not exist" do
640 | let(:subject_name) { 'new_subject' }
641 |
642 | it "returns the id of the schema" do
643 | post("/subjects/#{subject_name}/versions", params: { schema: version.schema.json })
644 | expect(response).to be_ok
645 | expect(response.body).to be_json_eql({ id: version.schema_id }.to_json).including(:id)
646 | end
647 |
648 | it "creates a new subject and version" do
649 | expect do
650 | expect do
651 | post("/subjects/#{subject_name}/versions", params: { schema: version.schema.json })
652 | end.to change(Subject, :count).by(1)
653 | end.to change(SchemaVersion, :count).by(1)
654 | expect(Subject.find_by(name: subject_name)).to be_present
655 | expect(SchemaVersion.latest_for_subject_name(subject_name).first).to be_present
656 | end
657 | end
658 |
659 | context "when the subject exists" do
660 | let(:json) { version.schema.json }
661 | let(:new_subject) { create(:subject) }
662 | let(:subject_name) { new_subject.name }
663 | let(:new_json) do
664 | JSON.parse(json).tap do |avro|
665 | avro['fields'] << { name: :new, type: :string, default: '' }
666 | end.to_json
667 | end
668 | let(:new_schema) { create(:schema, json: new_json) }
669 | let!(:new_version) { create(:schema_version, subject: new_subject, schema: new_schema) }
670 |
671 | it "returns the id of the schema" do
672 | post("/subjects/#{subject_name}/versions", params: { schema: json })
673 | expect(response).to be_ok
674 | expect(response.body).to be_json_eql({ id: version.schema_id }.to_json).including(:id)
675 | end
676 |
677 | it "creates a new schema version" do
678 | expect do
679 | post("/subjects/#{subject_name}/versions", params: { schema: json })
680 | end.to change(SchemaVersion, :count).by(1)
681 | new_schema_version = SchemaVersion.latest_for_subject_name(subject_name).first
682 | expect(new_schema_version.version).to eq(2)
683 | expect(new_schema_version.schema_id).to eq(version.schema_id)
684 | end
685 | end
686 |
687 | context "when the subject and schema do not exist" do
688 | let(:json) { build(:schema).json }
689 | let(:subject_name) { 'new_subject' }
690 |
691 | it "returns the id of a new schema" do
692 | post("/subjects/#{subject_name}/versions", params: { schema: json })
693 | expect(response).to be_ok
694 | version = SchemaVersion.latest_for_subject_name(subject_name).first
695 | expect(response.body).to be_json_eql({ id: version.schema_id }.to_json).including(:id)
696 | end
697 |
698 | it "creates a new subject, schema, and schema version" do
699 | expect do
700 | expect do
701 | expect do
702 | post("/subjects/#{subject_name}/versions", params: { schema: json })
703 | end.to change(Schema, :count).by(1)
704 | end.to change(Subject, :count).by(1)
705 | end.to change(SchemaVersion, :count).by(1)
706 |
707 | version = SchemaVersion.latest_for_subject_name(subject_name).first
708 | expect(version.version).to eq(1)
709 | expect(version.subject.name).to eq(subject_name)
710 | expect(version.schema.json).to eq(json)
711 | end
712 | end
713 | end
714 |
715 | context "retry" do
716 | before do
717 | # after migrations on version fingerprint2 has a unique constraint
718 | allow(Rails.configuration.x).to receive(:fingerprint_version).and_return('2')
719 | end
720 |
721 | context "registering a new schema for a subject" do
722 | let!(:previous_version) { create(:schema_version) }
723 | let(:subject_name) { previous_version.subject.name }
724 | let(:json) do
725 | JSON.parse(previous_version.schema.json).tap do |avro|
726 | avro['fields'] << { name: :extra, type: :string, default: '' }
727 | end.to_json
728 | end
729 | let(:fingerprint) { Schemas::FingerprintGenerator.generate_v2(json) }
730 |
731 | before do
732 | first_time = true
733 | allow(Schema).to receive(:find_by).with(fingerprint2: fingerprint) do
734 | if first_time
735 | @schema = Schema.create!(json: json)
736 | first_time = false
737 | nil
738 | else
739 | @schema
740 | end
741 | end
742 | end
743 |
744 | it "retries once" do
745 | expect do
746 | post("/subjects/#{subject_name}/versions", params: { schema: json })
747 | end.to change(Schema, :count).by(1)
748 |
749 | expect(response).to be_ok
750 | expect(response.body).to be_json_eql({ id: @schema.id }.to_json).including(:id)
751 | end
752 | end
753 | end
754 | end
755 |
756 | describe "POST /subjects/:name" do
757 | it_behaves_like "a secure endpoint" do
758 | let(:version) { create(:version) }
759 | let(:action) do
760 | unauthorized_post("/subjects/#{version.subject.name}",
761 | params: { schema: version.schema.json })
762 | end
763 | end
764 |
765 | shared_examples_for "checking for schema existence" do
766 | before do
767 | allow(Rails.configuration.x).to receive(:fingerprint_version).and_return(fingerprint_version)
768 | end
769 |
770 | context "when the schema exists for the subject" do
771 | let!(:version) { create(:schema_version) }
772 | let(:subject_name) { version.subject.name }
773 | let(:expected) do
774 | {
775 | subject: version.subject.name,
776 | id: version.schema_id,
777 | version: version.version,
778 | schema: version.schema.json
779 | }.to_json
780 | end
781 |
782 | it "returns information about the schema" do
783 | post("/subjects/#{subject_name}", params: { schema: version.schema.json })
784 | expect(response).to be_ok
785 | expect(response.body).to be_json_eql(expected)
786 | end
787 | end
788 |
789 | context "when the subject does not exist" do
790 | let(:subject_name) { 'fnord' }
791 | let(:json) { build(:schema).json }
792 |
793 | it "returns a subject not found error" do
794 | post("/subjects/#{subject_name}", params: { schema: json })
795 | expect(response).to be_not_found
796 | expect(response.body).to be_json_eql(SchemaRegistry::Errors::SUBJECT_NOT_FOUND.to_json)
797 | end
798 | end
799 |
800 | context "when the schema does not exist for the subject" do
801 | let!(:version) { create(:schema_version) }
802 | let(:subject_name) { version.subject.name }
803 | let(:json) { build(:schema).json }
804 |
805 | it "return a schema not found error" do
806 | post("/subjects/#{subject_name}", params: { schema: json })
807 | expect(response).to be_not_found
808 | expect(response.body).to be_json_eql(SchemaRegistry::Errors::SCHEMA_NOT_FOUND.to_json)
809 | end
810 | end
811 | end
812 |
813 | context "when fingerprint_version is '1'" do
814 | let(:fingerprint_version) { '1' }
815 |
816 | it_behaves_like "checking for schema existence"
817 | end
818 |
819 | context "when fingerprint_version is '2'" do
820 | let(:fingerprint_version) { '2' }
821 |
822 | it_behaves_like "checking for schema existence"
823 | end
824 |
825 | context "when fingerprint_version is 'all'" do
826 | let(:fingerprint_version) { 'all' }
827 |
828 | it_behaves_like "checking for schema existence"
829 | end
830 |
831 | context "when the schema is invalid" do
832 | # Confluent schema registry does not specify anything in this case, for
833 | # this endpoint, but a 422 makes the most sense to me. Better than a 404.
834 | it "returns an unprocessable entity error" do
835 | post('/subjects/foo', params: { schema: invalid_json })
836 | expect(response.status).to eq(422)
837 | expect(response.body).to eq(SchemaRegistry::Errors::INVALID_AVRO_SCHEMA.to_json)
838 | end
839 | end
840 | end
841 | end
842 |
--------------------------------------------------------------------------------