├── .rspec ├── lib ├── clickhouse-rails.rb ├── clickhouse │ ├── rails │ │ ├── version.rb │ │ ├── config.rb │ │ └── migrations │ │ │ └── base.rb │ ├── rails.rb │ └── table.rb ├── generators │ └── clickhouse │ │ ├── migration │ │ ├── templates │ │ │ └── db │ │ │ │ └── migrate │ │ │ │ └── template.rb.tt │ │ └── migration_generator.rb │ │ └── install │ │ ├── templates │ │ ├── config │ │ │ └── clickhouse.yml │ │ └── db │ │ │ └── clickhouse │ │ │ └── migrate │ │ │ └── 001_init.rb │ │ └── install_generator.rb └── tasks │ └── clickhouse │ └── db │ └── migrate.rake ├── Gemfile ├── spec ├── fixtures │ └── files │ │ ├── clickhouse.yml │ │ └── 001_init.rb ├── support │ ├── migration.rb │ ├── init.rb │ └── generators.rb ├── spec_helper.rb ├── generators │ ├── migration │ │ └── migration_generator_spec.rb │ └── install │ │ └── install_generator_spec.rb ├── rails_helper.rb ├── config_spec.rb ├── table_spec.rb └── migrations │ └── base_spec.rb ├── Dockerfile.test ├── .travis.yml ├── .rubocop.yml ├── docker-compose.yml ├── clickhouse-rails.gemspec ├── LICENSE ├── .gitignore ├── README.md └── Gemfile.lock /.rspec: -------------------------------------------------------------------------------- 1 | --require spec_helper 2 | -------------------------------------------------------------------------------- /lib/clickhouse-rails.rb: -------------------------------------------------------------------------------- 1 | require 'clickhouse' 2 | require 'clickhouse/table' 3 | require 'clickhouse/rails' 4 | -------------------------------------------------------------------------------- /lib/clickhouse/rails/version.rb: -------------------------------------------------------------------------------- 1 | module Clickhouse 2 | module Rails 3 | VERSION = '0.1.5'.freeze 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gemspec 4 | 5 | gem 'codecov', require: false, group: :test 6 | gem 'rubocop', require: false 7 | -------------------------------------------------------------------------------- /spec/fixtures/files/clickhouse.yml: -------------------------------------------------------------------------------- 1 | test: 2 | hosts: <%= ENV.fetch('CLICKHOUSE_HOSTS', 'localhost:8123') %> 3 | username: <%= ENV.fetch('CLICKHOUSE_USER', 'default') %> 4 | -------------------------------------------------------------------------------- /lib/generators/clickhouse/migration/templates/db/migrate/template.rb.tt: -------------------------------------------------------------------------------- 1 | class <%= migration_class_name %> < Clickhouse::Rails::Migrations::Base 2 | def self.up 3 | end 4 | end 5 | -------------------------------------------------------------------------------- /lib/generators/clickhouse/install/templates/config/clickhouse.yml: -------------------------------------------------------------------------------- 1 | development: 2 | hosts: <%= ENV.fetch('CLICKHOUSE_HOSTS', 'localhost:8123') %> 3 | username: <%= ENV.fetch('CLICKHOUSE_USER', 'default') %> 4 | -------------------------------------------------------------------------------- /spec/support/migration.rb: -------------------------------------------------------------------------------- 1 | def with_table(table_name, &block) 2 | Clickhouse::Rails::Migrations::Base.soft_drop_table(table_name) 3 | Clickhouse::Rails::Migrations::Base.create_table(table_name, &block) 4 | end 5 | -------------------------------------------------------------------------------- /Dockerfile.test: -------------------------------------------------------------------------------- 1 | FROM ruby:2.5 2 | 3 | RUN mkdir /app 4 | WORKDIR /app 5 | ADD . /app 6 | COPY ./Gemfile* /app/ 7 | 8 | RUN gem install bundler &&\ 9 | bundle install 10 | 11 | CMD ["bundle exec rspec"] 12 | -------------------------------------------------------------------------------- /spec/fixtures/files/001_init.rb: -------------------------------------------------------------------------------- 1 | class Init < Clickhouse::Rails::Migrations::Base 2 | def self.up 3 | create_table MIGRATION_TABLE do |t| 4 | t.string :version 5 | 6 | t.engine 'File(TabSeparated)' 7 | end 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /lib/generators/clickhouse/install/templates/db/clickhouse/migrate/001_init.rb: -------------------------------------------------------------------------------- 1 | class Init < Clickhouse::Rails::Migrations::Base 2 | def self.up 3 | create_table MIGRATION_TABLE do |t| 4 | t.string :version 5 | 6 | t.engine 'File(TabSeparated)' 7 | end 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: ruby 2 | cache: bundler 3 | rvm: 4 | - 2.5.5 5 | services: 6 | - docker 7 | before_install: 8 | - bundle install 9 | - docker run -e "TZ=Europe/Moscow" -d -p 127.0.0.1:8123:8123 --name test-clickhouse-server --ulimit nofile=262144:262144 yandex/clickhouse-server 10 | script: 11 | - rubocop 12 | - rspec 13 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | RSpec.configure do |config| 2 | config.expect_with :rspec do |expectations| 3 | expectations.include_chain_clauses_in_custom_matcher_descriptions = true 4 | end 5 | 6 | config.mock_with :rspec do |mocks| 7 | mocks.verify_partial_doubles = true 8 | end 9 | 10 | config.shared_context_metadata_behavior = :apply_to_host_groups 11 | end 12 | -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | AllCops: 2 | Exclude: 3 | - 'vendor/bundle/**/*' 4 | - './tmp/**/*.rb' 5 | 6 | Metrics/BlockLength: 7 | Max: 25 8 | Exclude: 9 | - './spec/**/*_spec.rb' 10 | 11 | Naming/FileName: 12 | Exclude: 13 | - 'lib/clickhouse-rails.rb' 14 | 15 | Style/Documentation: 16 | Enabled: false 17 | 18 | Metrics/LineLength: 19 | Max: 97 20 | 21 | Style/FrozenStringLiteralComment: 22 | Enabled: false 23 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | ruby: 4 | build: 5 | dockerfile: ./Dockerfile.test 6 | context: . 7 | environment: 8 | - "CLICKHOUSE_HOSTS=clickhouse:8123" 9 | - "CLICKHOUSE_USER=default" 10 | - "CLICKHOUSE_ENV=test" 11 | command: bash -c "bundle install && rspec" 12 | volumes: 13 | - ./:/app 14 | links: 15 | - "clickhouse:clickhouse" 16 | clickhouse: 17 | image: yandex/clickhouse-server 18 | -------------------------------------------------------------------------------- /spec/generators/migration/migration_generator_spec.rb: -------------------------------------------------------------------------------- 1 | require_relative '../../rails_helper' 2 | require_relative '../../../lib/generators/clickhouse/migration/migration_generator' 3 | 4 | RSpec.describe Clickhouse::Generators::MigrationGenerator, type: :generator do 5 | setup_default_destination 6 | 7 | it 'generates db/clickhouse/migrate/temp.rb' do 8 | run_generator(['temp']) 9 | expect(file('db/clickhouse/migrate/001_temp.rb')).to exist 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /spec/support/init.rb: -------------------------------------------------------------------------------- 1 | RSpec.shared_context 'with init migration' do 2 | let(:config_file) { file_fixture('clickhouse.yml') } 3 | let(:file) { file_fixture('001_init.rb') } 4 | let(:migration_table) { described_class::MIGRATION_TABLE } 5 | 6 | before do 7 | require file.realpath 8 | 9 | Clickhouse::Rails.config(config_file.realpath.to_s) 10 | Clickhouse::Rails.init! 11 | 12 | described_class.soft_drop_table(migration_table) 13 | Init.up 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /lib/tasks/clickhouse/db/migrate.rake: -------------------------------------------------------------------------------- 1 | require_relative '../../../clickhouse/rails' 2 | require_relative '../../../clickhouse/rails/migrations/base' 3 | 4 | namespace :clickhouse do 5 | namespace :db do 6 | desc 'Migrate clickhouse migrations' 7 | task migrate: :environment do 8 | path_to_migrations = './db/clickhouse/migrate/*.rb' 9 | Dir[path_to_migrations].sort.each { |file| require file } 10 | 11 | Clickhouse::Rails::Migrations::Base.run_up 12 | end 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /spec/rails_helper.rb: -------------------------------------------------------------------------------- 1 | ENV['RAILS_ENV'] ||= 'test' 2 | 3 | require 'simplecov' 4 | SimpleCov.start 5 | 6 | require 'codecov' 7 | SimpleCov.formatter = SimpleCov::Formatter::Codecov 8 | require 'rails' 9 | require 'action_view/railtie' 10 | require 'action_controller/railtie' 11 | require 'rails/generators' 12 | 13 | require 'ammeter/init' 14 | 15 | require_relative '../lib/clickhouse-rails' 16 | require_relative 'support/generators' 17 | require_relative 'support/init' 18 | require_relative 'support/migration' 19 | 20 | Rails.logger = Logger.new('/dev/null') 21 | -------------------------------------------------------------------------------- /spec/generators/install/install_generator_spec.rb: -------------------------------------------------------------------------------- 1 | require_relative '../../rails_helper' 2 | require_relative '../../../lib/generators/clickhouse/install/install_generator' 3 | 4 | RSpec.describe Clickhouse::Generators::InstallGenerator, type: :generator do 5 | setup_default_destination 6 | 7 | it 'generates config/clickhouse.yml' do 8 | run_generator 9 | expect(file('config/clickhouse.yml')).to exist 10 | end 11 | 12 | it 'generates db/clickhouse/migrate/001_init.rb' do 13 | run_generator 14 | expect(file('db/clickhouse/migrate/001_init.rb')).to exist 15 | end 16 | end 17 | -------------------------------------------------------------------------------- /spec/support/generators.rb: -------------------------------------------------------------------------------- 1 | module Clickhouse 2 | module Rails 3 | module Generators 4 | module Macros 5 | def set_default_destination 6 | destination File.expand_path('../../tmp', __dir__) 7 | end 8 | 9 | def setup_default_destination 10 | set_default_destination 11 | before { prepare_destination } 12 | end 13 | end 14 | 15 | def self.included(klass) 16 | klass.extend(Macros) 17 | end 18 | end 19 | end 20 | end 21 | 22 | RSpec.configure do |config| 23 | config.include Clickhouse::Rails::Generators, type: :generator 24 | end 25 | -------------------------------------------------------------------------------- /lib/generators/clickhouse/install/install_generator.rb: -------------------------------------------------------------------------------- 1 | module Clickhouse 2 | module Generators 3 | class InstallGenerator < ::Rails::Generators::Base 4 | desc <<-DESC 5 | Description: 6 | Copy clickhouse file to your application 7 | DESC 8 | 9 | def self.source_root 10 | @source_root ||= File.expand_path(File.join(File.dirname(__FILE__), 'templates')) 11 | end 12 | 13 | def copy_install_files 14 | copy_file 'config/clickhouse.yml' 15 | directory db_migrate_path 16 | end 17 | 18 | def db_migrate_path 19 | 'db/clickhouse/migrate' 20 | end 21 | end 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /spec/config_spec.rb: -------------------------------------------------------------------------------- 1 | require_relative 'rails_helper' 2 | 3 | describe Clickhouse::Rails::Config do 4 | describe '.init' do 5 | subject(:init) { described_class.init } 6 | 7 | context 'when config file does not exists' do 8 | let(:error_class) { Clickhouse::Rails::Config::ConfigurationNotFound } 9 | it { expect { init }.to raise_error(error_class) } 10 | end 11 | 12 | context 'when config file exists' do 13 | let(:path) { File.dirname(__FILE__) + '/fixtures/files/clickhouse.yml' } 14 | 15 | before do 16 | stub_const('Clickhouse::Rails::Config::DEFAULT_CONFIG_PATH', path) 17 | end 18 | 19 | it { expect { init }.not_to raise_error } 20 | end 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /clickhouse-rails.gemspec: -------------------------------------------------------------------------------- 1 | require File.expand_path('lib/clickhouse/rails/version', __dir__) 2 | 3 | Gem::Specification.new do |gem| 4 | gem.authors = ['Vsevolod Avramov'] 5 | gem.email = ['gsevka@gmail.com'] 6 | gem.summary = 'A Rails database driver for ClickHouse' 7 | gem.description = 'A Rails database driver for ClickHouse' 8 | gem.homepage = 'https://github.com/vsevolod/clickhouse-rails' 9 | 10 | gem.files = `git ls-files`.split("\n") 11 | gem.test_files = `git ls-files -- {test,spec,features}/*`.split('\n') 12 | gem.name = 'clickhouse-rails' 13 | gem.require_paths = ['lib'] 14 | gem.version = Clickhouse::Rails::VERSION 15 | gem.licenses = ['MIT'] 16 | 17 | gem.add_dependency 'clickhouse' 18 | gem.add_dependency 'railties' 19 | 20 | gem.add_development_dependency 'ammeter', '~> 1.1.2' 21 | gem.add_development_dependency 'pry' 22 | gem.add_development_dependency 'rspec-rails' 23 | end 24 | -------------------------------------------------------------------------------- /lib/clickhouse/rails.rb: -------------------------------------------------------------------------------- 1 | require 'rails' 2 | require 'clickhouse/rails/config' 3 | 4 | module Clickhouse 5 | module Rails 6 | def self.init! 7 | config && Clickhouse.establish_connection(config) 8 | end 9 | 10 | def self.config(config_path = nil) 11 | @config ||= Clickhouse::Rails::Config.init(config_path) 12 | end 13 | 14 | class Railtie < ::Rails::Railtie 15 | generators = 16 | if config.respond_to?(:app_generators) 17 | config.app_generators 18 | else 19 | config.generators 20 | end 21 | generators.integration_tool :clickhouse 22 | generators.test_framework :clickhouse 23 | 24 | generators do 25 | require 'generators/clickhouse/install/install_generator' 26 | end 27 | 28 | rake_tasks do 29 | load 'tasks/clickhouse/db/migrate.rake' 30 | end 31 | 32 | config.to_prepare do 33 | Clickhouse::Rails.init! 34 | end 35 | end 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /lib/generators/clickhouse/migration/migration_generator.rb: -------------------------------------------------------------------------------- 1 | module Clickhouse 2 | module Generators 3 | class MigrationGenerator < ::Rails::Generators::NamedBase 4 | include ::Rails::Generators::Migration 5 | 6 | desc <<-DESC 7 | Description: 8 | Add migration for clickhouse database 9 | DESC 10 | 11 | def self.source_root 12 | @source_root ||= File.expand_path(File.join(File.dirname(__FILE__), 'templates')) 13 | end 14 | 15 | def self.next_migration_number(dirname) 16 | number = current_migration_number(dirname) + 1 17 | format('%03d', number) 18 | end 19 | 20 | def create_migration_file 21 | set_local_assign! 22 | migration_template @migration_template, File.join(db_migrate_path, "#{file_name}.rb") 23 | end 24 | 25 | private 26 | 27 | def set_local_assign! 28 | @migration_template = 'db/migrate/template.rb' 29 | end 30 | 31 | def db_migrate_path 32 | 'db/clickhouse/migrate' 33 | end 34 | end 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Avramov Vsevolod 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /lib/clickhouse/rails/config.rb: -------------------------------------------------------------------------------- 1 | require 'yaml' 2 | 3 | module Clickhouse 4 | module Rails 5 | module Config 6 | class ConfigurationNotFound < NameError; end 7 | 8 | CLICKHOUSE_ROOT = File.expand_path(::Rails.root.present? ? ::Rails.root : '.') 9 | DEFAULT_CONFIG_PATH = File.join(CLICKHOUSE_ROOT, 'config', 'clickhouse.yml') 10 | 11 | def self.init(config_path = nil) 12 | config_path ||= DEFAULT_CONFIG_PATH 13 | exists = config_path && File.exist?(config_path) 14 | unless exists 15 | raise ConfigurationNotFound, "could not find the \"#{config_path}\" configuration file" 16 | end 17 | 18 | content = File.read(config_path) 19 | data = defined?(ERB) ? ERB.new(content).result : content 20 | source = YAML.safe_load(data)[defined?(::Rails) ? ::Rails.env : ENV['CLICKHOUSE_ENV']] 21 | config_mapper(source) 22 | end 23 | 24 | def self.config_mapper(source) 25 | return nil if source.nil? 26 | return nil if source['hosts'].nil? 27 | 28 | { 29 | urls: source['hosts'].split(','), 30 | username: source['username'], 31 | password: source['password'] 32 | } 33 | end 34 | end 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | *.rbc 3 | /.config 4 | /coverage/ 5 | /InstalledFiles 6 | /pkg/ 7 | /spec/reports/ 8 | /spec/examples.txt 9 | /spec/tmp 10 | /test/tmp/ 11 | /test/version_tmp/ 12 | /tmp/ 13 | /tmp/db/clickhouse 14 | 15 | # Used by dotenv library to load environment variables. 16 | # .env 17 | 18 | ## Specific to RubyMotion: 19 | .dat* 20 | .repl_history 21 | build/ 22 | *.bridgesupport 23 | build-iPhoneOS/ 24 | build-iPhoneSimulator/ 25 | 26 | ## Specific to RubyMotion (use of CocoaPods): 27 | # 28 | # We recommend against adding the Pods directory to your .gitignore. However 29 | # you should judge for yourself, the pros and cons are mentioned at: 30 | # https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control 31 | # 32 | # vendor/Pods/ 33 | 34 | ## Documentation cache and generated files: 35 | /.yardoc/ 36 | /_yardoc/ 37 | /doc/ 38 | /rdoc/ 39 | 40 | ## Environment normalization: 41 | /.bundle/ 42 | /vendor/bundle 43 | /lib/bundler/man/ 44 | 45 | # for a library or gem, you might want to ignore these files since the code is 46 | # intended to run in multiple environments; otherwise, check them in: 47 | # Gemfile.lock 48 | # .ruby-version 49 | # .ruby-gemset 50 | 51 | # unless supporting rvm < 1.11.0 or doing something fancy, ignore this: 52 | .rvmrc 53 | -------------------------------------------------------------------------------- /spec/table_spec.rb: -------------------------------------------------------------------------------- 1 | require_relative 'rails_helper' 2 | 3 | class CustomTable 4 | include Clickhouse::Table 5 | end 6 | 7 | describe Clickhouse::Table do 8 | before do 9 | with_table 'custom_tables' do |t| 10 | t.string 'field' 11 | 12 | t.engine 'File(TabSeparated)' 13 | end 14 | end 15 | 16 | describe '#table_columns' do 17 | subject(:method) { CustomTable.table_columns } 18 | 19 | it 'returns all info' do 20 | is_expected.to eq('field' => 'String') 21 | end 22 | end 23 | 24 | describe '#insert_row' do 25 | let(:inserted_rows) { CustomTable.rows.to_a } 26 | let(:row) { { 'field' => 'a' } } 27 | 28 | it 'add the row' do 29 | CustomTable.insert_row(row) 30 | 31 | expect(inserted_rows).to eq([['a']]) 32 | end 33 | end 34 | 35 | describe '#insert_rows' do 36 | let(:inserted_rows) { CustomTable.rows.to_a } 37 | 38 | context 'when row is a hash' do 39 | let(:rows) { [{ 'field' => 'a' }, { 'field' => 2 }] } 40 | 41 | it 'adds two rows' do 42 | CustomTable.insert_rows(rows) 43 | 44 | expect(inserted_rows).to eq([['a'], ['2']]) 45 | end 46 | end 47 | 48 | context 'when row has empty attributes' do 49 | let(:rows) { [{}] } 50 | 51 | it 'adds empty row' do 52 | CustomTable.insert_rows(rows) 53 | 54 | expect(inserted_rows).to eq([['']]) 55 | end 56 | end 57 | end 58 | 59 | describe '#prepare_row' do 60 | subject(:method) { CustomTable.prepare_row(row) } 61 | 62 | context 'when row has wrong type' do 63 | # TODO: implement array logic 64 | let(:row) { ['a'] } 65 | 66 | it 'raises an error' do 67 | expect { method }.to raise_error(Clickhouse::Table::WrongTypeRowError) 68 | end 69 | end 70 | 71 | context 'when row has extra fields' do 72 | let(:row) { { 'field' => 'a', 'extra_field' => 'b' } } 73 | 74 | it 'leaves only existing fields' do 75 | is_expected.to eq('field' => 'a') 76 | end 77 | end 78 | end 79 | end 80 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Clickhouse rails 2 | 3 | [![Build Status](https://travis-ci.com/vsevolod/clickhouse-rails.svg?branch=master)](https://travis-ci.com/vsevolod/clickhouse-rails) 4 | [![codecov](https://codecov.io/gh/vsevolod/clickhouse-rails/branch/master/graph/badge.svg)](https://codecov.io/gh/vsevolod/clickhouse-rails) 5 | 6 | ## Install 7 | 8 | 1. Add to Gemfile 9 | ```ruby 10 | gem 'clickhouse-rails' 11 | ``` 12 | 13 | 2. Run bundle 14 | ```bash 15 | $ bundle install 16 | ``` 17 | 18 | 3. Init config files and folders 19 | ```bash 20 | $ rails g clickhouse:install 21 | ``` 22 | 23 | 4. Change clickhouse.yml at `config/clickhouse.yml` path 24 | 25 | ## Usage 26 | 27 | 1. Create migrations 28 | ```bash 29 | $ rails g clickhouse:migration add_tmp_table 30 | create db/clickhouse/migrate/002_add_tmp_table.rb 31 | ``` 32 | 33 | 2. Edit file like this: 34 | ```ruby 35 | # db/clickhouse/migrate/002_add_tmp_table.rb 36 | class AddTmpTable < Clickhouse::Rails::Migrations::Base 37 | def self.up 38 | create_table :tmp do |t| 39 | t.date :date 40 | t.uint16 :id 41 | 42 | t.engine "MergeTree(date, (date), 8192)" 43 | end 44 | end 45 | end 46 | ``` 47 | 48 | 3. Run migrations 49 | ```bash 50 | $ rake clickhouse:db:migrate 51 | ``` 52 | 53 | You can create class of clickhouse table: 54 | ```ruby 55 | # app/models/custom_table.rb 56 | class CustomTable 57 | include Clickhouse::Table 58 | end 59 | ``` 60 | 61 | And insert rows like this: 62 | ```ruby 63 | CustomTable.insert_rows([{a:1, b:2}]) 64 | CustomTable.insert_row({a:1, b:2}) 65 | ``` 66 | 67 | Alter table migration 68 | ```ruby 69 | alter_table "analytics_table" do 70 | fetch_column :new_ids, :uint8 # Adds column new_ids or skip it 71 | end 72 | ``` 73 | 74 | ## TODO: 75 | 76 | 1. Rollback migrations 77 | 2. Alter table 78 | 79 | ## Additional clickhouse links 80 | 81 | - [Base gem](https://github.com/archan937/clickhouse) 82 | - [List of data types](https://clickhouse.yandex/docs/en/data_types/) 83 | - [Table engines](https://clickhouse.yandex/docs/en/operations/table_engines/) 84 | -------------------------------------------------------------------------------- /spec/migrations/base_spec.rb: -------------------------------------------------------------------------------- 1 | require_relative '../rails_helper' 2 | require_relative '../../lib/clickhouse/rails/migrations/base' 3 | 4 | RSpec.describe Clickhouse::Rails::Migrations::Base do 5 | describe '.run_up' do 6 | subject(:run_up) { described_class.run_up } 7 | 8 | include_context 'with init migration' do 9 | it 'executes up and add version' do 10 | expect(Init).to receive(:up) 11 | expect(Init).to receive(:add_version) 12 | 13 | subject 14 | end 15 | 16 | context 'when two Init in @migrations_list' do 17 | before do 18 | described_class.migrations_list << Init 19 | end 20 | 21 | it 'skips the second time' do 22 | expect(Init).to receive(:up).once 23 | 24 | subject 25 | end 26 | end 27 | end 28 | end 29 | 30 | describe '.create_table' do 31 | subject(:create_table) { described_class.create_table(table_name, &block) } 32 | 33 | let(:config_file) { file_fixture('clickhouse.yml') } 34 | let(:table_name) { 'example' } 35 | let(:block) do 36 | lambda do |t| 37 | t.string 'field' 38 | 39 | t.engine 'File(TabSeparated)' 40 | end 41 | end 42 | 43 | before do 44 | Clickhouse::Rails.config(config_file.realpath.to_s) 45 | Clickhouse::Rails.init! 46 | described_class.soft_drop_table(table_name) 47 | end 48 | 49 | it 'adds table to clickhouse' do 50 | create_table 51 | 52 | expect(described_class).to be_table_exists(table_name) 53 | end 54 | end 55 | 56 | describe '.alter_table' do 57 | let(:table_name) { 'custom_table2' } 58 | subject(:alter_table) { described_class.alter_table(table_name, &block) } 59 | # let(:block) { lambda { fetch_column :new_ids, :uint8 } } 60 | let(:block) do 61 | proc do 62 | described_class.fetch_column :new_ids, :uint8 63 | end 64 | end 65 | 66 | before do 67 | with_table table_name do |t| 68 | t.date 'date' 69 | 70 | t.engine 'MergeTree(date, date, 8192)' 71 | end 72 | end 73 | 74 | it 'adds column to existing table' do 75 | alter_table 76 | 77 | columns = described_class.connection.describe_table(table_name) 78 | expect(columns.flatten).to include('date', 'new_ids') 79 | end 80 | end 81 | 82 | describe '.add_version' do 83 | include_context 'with init migration' 84 | 85 | it 'executes up and add version' do 86 | described_class.add_version 87 | 88 | expect(Clickhouse.connection.count(from: migration_table)).to eq(1) 89 | end 90 | end 91 | end 92 | -------------------------------------------------------------------------------- /lib/clickhouse/table.rb: -------------------------------------------------------------------------------- 1 | module Clickhouse 2 | module Table 3 | class WrongTypeRowError < StandardError; end 4 | 5 | def self.included(base) 6 | base.extend ClassMethods 7 | end 8 | 9 | module ClassMethods 10 | def table_name 11 | @table_name ||= to_s.tableize 12 | end 13 | 14 | def insert_row(row) 15 | return if row.nil? 16 | 17 | connection.insert_rows(table_name) do 18 | complete_row = prepare_row(block_given? ? yield(row) : row) 19 | 20 | [complete_row] 21 | end 22 | end 23 | alias create insert_row 24 | 25 | def insert_rows(batch_rows) 26 | connection.insert_rows(table_name) do |table_rows| 27 | batch_rows.each do |row| 28 | next if row.nil? 29 | 30 | complete_row = prepare_row(block_given? ? yield(row) : row) 31 | 32 | table_rows << complete_row 33 | end 34 | 35 | table_rows 36 | end 37 | end 38 | 39 | def table_columns 40 | @table_columns ||= 41 | connection.select_rows( 42 | select: 'name, type', 43 | from: 'system.columns', 44 | where: "table = '#{table_name}'" 45 | ).to_h 46 | end 47 | 48 | def rows(attributes = {}) 49 | connection.select_rows(attributes.merge(from: table_name)) 50 | end 51 | 52 | def empty_row 53 | @empty_row ||= table_columns.map do |k, v| 54 | value = 55 | case v 56 | when /UInt/ then 0 57 | when /Float/ then 0.0 58 | else 59 | '' 60 | end 61 | 62 | [k, value] 63 | end.to_h 64 | end 65 | 66 | def prepare_row(row) 67 | validate_row(row) 68 | 69 | row.stringify_keys! 70 | crop_row(row) 71 | 72 | empty_row.merge(row) 73 | end 74 | 75 | def connection 76 | ::Clickhouse.connection 77 | end 78 | 79 | def logger 80 | ::Rails.logger 81 | end 82 | 83 | private 84 | 85 | def validate_row(row) 86 | return if row.is_a?(Hash) 87 | 88 | raise WrongTypeRowError, "#{row.inspect} has wrong type" 89 | end 90 | 91 | def crop_row(row) 92 | undefined_columns = row.keys - empty_row.keys 93 | 94 | return if undefined_columns.empty? 95 | 96 | logger.warn("Clickhouse: Undefined columns for #{table_name}: #{undefined_columns}") 97 | row.except!(*undefined_columns) 98 | end 99 | end 100 | end 101 | end 102 | -------------------------------------------------------------------------------- /lib/clickhouse/rails/migrations/base.rb: -------------------------------------------------------------------------------- 1 | module Clickhouse 2 | module Rails 3 | module Migrations 4 | class Base 5 | MIGRATION_TABLE = 'schema_migrations'.freeze 6 | 7 | class << self 8 | attr_accessor :migrations_list 9 | 10 | def inherited(child) 11 | @migrations_list ||= [] 12 | @migrations_list.push(child) 13 | end 14 | 15 | def run_up 16 | return unless @migrations_list 17 | 18 | @migrations_list.each do |migration| 19 | next if migration.passed? 20 | 21 | run_migration(migration) 22 | end 23 | end 24 | 25 | def passed? 26 | return false unless table_exists?(MIGRATION_TABLE) 27 | 28 | @rows ||= connection.select_rows(select: 'version', from: MIGRATION_TABLE).flatten 29 | @rows.include?(name) 30 | rescue Clickhouse::QueryError 31 | false 32 | end 33 | 34 | def up; end 35 | 36 | def add_version 37 | connection.insert_rows(MIGRATION_TABLE, names: ['version']) do |row| 38 | row << [name] 39 | end 40 | end 41 | 42 | def table_exists?(table_name) 43 | connection.execute("EXISTS TABLE #{table_name}").strip == '1' 44 | end 45 | 46 | def soft_create_table(table_name, &block) 47 | return if table_exists?(table_name) 48 | 49 | create_table(table_name, &block) 50 | end 51 | 52 | def soft_drop_table(table_name) 53 | return unless table_exists?(table_name) 54 | 55 | drop_table(table_name) 56 | end 57 | 58 | def create_table(table_name, &block) 59 | logger.info "# >======= Create #{table_name} ========" 60 | connection.create_table(table_name, &block) 61 | end 62 | 63 | def drop_table(table_name, &block) 64 | logger.info "# >======= Drop #{table_name} ========" 65 | connection.drop_table(table_name, &block) 66 | end 67 | 68 | def alter_table(table_name) 69 | @table_info = connection.describe_table(table_name) 70 | @table_name = table_name 71 | 72 | yield(table_name) 73 | end 74 | 75 | def fetch_column(column, type) 76 | return if @table_info.find { |c_info| c_info.first == column.to_s } 77 | 78 | type = type.to_s.gsub(/(^.|_\w)/) do 79 | Regexp.last_match(1).upcase 80 | end 81 | type = type.gsub('Uint', 'UInt').delete('_') 82 | 83 | query = "ALTER TABLE #{@table_name} ADD COLUMN #{column} #{type}" 84 | logger.info(query) 85 | connection.execute(query) 86 | end 87 | 88 | def run_migration(migration) 89 | logger.info "# >========== #{migration.name} ===========" 90 | migration.up 91 | migration.add_version 92 | rescue Clickhouse::QueryError => e 93 | logger.info "# Error #{e.class}:" 94 | logger.info "# #{e.message}" 95 | ensure 96 | logger.info "# <========== #{migration.name} ===========\n\n" 97 | end 98 | 99 | def connection 100 | Clickhouse.connection 101 | end 102 | 103 | def logger 104 | Logger.new(STDOUT) 105 | end 106 | end 107 | end 108 | end 109 | end 110 | end 111 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | PATH 2 | remote: . 3 | specs: 4 | clickhouse-rails (0.1.5) 5 | clickhouse 6 | railties 7 | 8 | GEM 9 | remote: https://rubygems.org/ 10 | specs: 11 | actionpack (5.2.3) 12 | actionview (= 5.2.3) 13 | activesupport (= 5.2.3) 14 | rack (~> 2.0) 15 | rack-test (>= 0.6.3) 16 | rails-dom-testing (~> 2.0) 17 | rails-html-sanitizer (~> 1.0, >= 1.0.2) 18 | actionview (5.2.3) 19 | activesupport (= 5.2.3) 20 | builder (~> 3.1) 21 | erubi (~> 1.4) 22 | rails-dom-testing (~> 2.0) 23 | rails-html-sanitizer (~> 1.0, >= 1.0.3) 24 | activesupport (5.2.3) 25 | concurrent-ruby (~> 1.0, >= 1.0.2) 26 | i18n (>= 0.7, < 2) 27 | minitest (~> 5.1) 28 | tzinfo (~> 1.1) 29 | addressable (2.6.0) 30 | public_suffix (>= 2.0.2, < 4.0) 31 | ammeter (1.1.4) 32 | activesupport (>= 3.0) 33 | railties (>= 3.0) 34 | rspec-rails (>= 2.2) 35 | ast (2.4.0) 36 | builder (3.2.3) 37 | clickhouse (0.1.10) 38 | activesupport (>= 4.1.8) 39 | bundler (>= 1.13.4) 40 | erubis 41 | faraday 42 | launchy 43 | pond 44 | sinatra 45 | thor 46 | codecov (0.1.14) 47 | json 48 | simplecov 49 | url 50 | coderay (1.1.2) 51 | concurrent-ruby (1.1.5) 52 | crass (1.0.5) 53 | diff-lcs (1.3) 54 | docile (1.3.1) 55 | erubi (1.8.0) 56 | erubis (2.7.0) 57 | faraday (0.15.4) 58 | multipart-post (>= 1.2, < 3) 59 | i18n (1.6.0) 60 | concurrent-ruby (~> 1.0) 61 | jaro_winkler (1.5.2) 62 | json (2.3.1) 63 | launchy (2.4.3) 64 | addressable (~> 2.3) 65 | loofah (2.3.1) 66 | crass (~> 1.0.2) 67 | nokogiri (>= 1.5.9) 68 | method_source (0.9.2) 69 | mini_portile2 (2.4.0) 70 | minitest (5.11.3) 71 | multipart-post (2.1.1) 72 | mustermann (1.0.3) 73 | nokogiri (1.10.5) 74 | mini_portile2 (~> 2.4.0) 75 | parallel (1.17.0) 76 | parser (2.6.2.1) 77 | ast (~> 2.4.0) 78 | pond (0.3.0) 79 | pry (0.12.2) 80 | coderay (~> 1.1.0) 81 | method_source (~> 0.9.0) 82 | psych (3.1.0) 83 | public_suffix (3.1.1) 84 | rack (2.0.8) 85 | rack-protection (2.0.5) 86 | rack 87 | rack-test (1.1.0) 88 | rack (>= 1.0, < 3) 89 | rails-dom-testing (2.0.3) 90 | activesupport (>= 4.2.0) 91 | nokogiri (>= 1.6) 92 | rails-html-sanitizer (1.0.4) 93 | loofah (~> 2.2, >= 2.2.2) 94 | railties (5.2.3) 95 | actionpack (= 5.2.3) 96 | activesupport (= 5.2.3) 97 | method_source 98 | rake (>= 0.8.7) 99 | thor (>= 0.19.0, < 2.0) 100 | rainbow (3.0.0) 101 | rake (12.3.0) 102 | rspec-core (3.8.0) 103 | rspec-support (~> 3.8.0) 104 | rspec-expectations (3.8.2) 105 | diff-lcs (>= 1.2.0, < 2.0) 106 | rspec-support (~> 3.8.0) 107 | rspec-mocks (3.8.0) 108 | diff-lcs (>= 1.2.0, < 2.0) 109 | rspec-support (~> 3.8.0) 110 | rspec-rails (3.8.2) 111 | actionpack (>= 3.0) 112 | activesupport (>= 3.0) 113 | railties (>= 3.0) 114 | rspec-core (~> 3.8.0) 115 | rspec-expectations (~> 3.8.0) 116 | rspec-mocks (~> 3.8.0) 117 | rspec-support (~> 3.8.0) 118 | rspec-support (3.8.0) 119 | rubocop (0.67.2) 120 | jaro_winkler (~> 1.5.1) 121 | parallel (~> 1.10) 122 | parser (>= 2.5, != 2.5.1.1) 123 | psych (>= 3.1.0) 124 | rainbow (>= 2.2.2, < 4.0) 125 | ruby-progressbar (~> 1.7) 126 | unicode-display_width (>= 1.4.0, < 1.6) 127 | ruby-progressbar (1.10.0) 128 | simplecov (0.16.1) 129 | docile (~> 1.1) 130 | json (>= 1.8, < 3) 131 | simplecov-html (~> 0.10.0) 132 | simplecov-html (0.10.2) 133 | sinatra (2.0.5) 134 | mustermann (~> 1.0) 135 | rack (~> 2.0) 136 | rack-protection (= 2.0.5) 137 | tilt (~> 2.0) 138 | thor (0.20.3) 139 | thread_safe (0.3.6) 140 | tilt (2.0.9) 141 | tzinfo (1.2.5) 142 | thread_safe (~> 0.1) 143 | unicode-display_width (1.5.0) 144 | url (0.3.2) 145 | 146 | PLATFORMS 147 | ruby 148 | 149 | DEPENDENCIES 150 | ammeter (~> 1.1.2) 151 | clickhouse-rails! 152 | codecov 153 | pry 154 | rspec-rails 155 | rubocop 156 | 157 | BUNDLED WITH 158 | 2.0.1 159 | --------------------------------------------------------------------------------