├── .gitignore ├── .rspec ├── .rubocop.yml ├── .travis.yml ├── Gemfile ├── Gemfile.lock ├── LICENSE.txt ├── README.md ├── Rakefile ├── _resource ├── schema.png └── schema.sketch ├── bin ├── console └── setup ├── lib ├── generators │ └── rails_async_migrations │ │ ├── install_generator.rb │ │ └── templates │ │ └── create_async_schema_migrations.rb ├── rails_async_migrations.rb ├── rails_async_migrations │ ├── class_mutators.rb │ ├── config.rb │ ├── connection │ │ └── active_record.rb │ ├── instance_mutators.rb │ ├── migration.rb │ ├── migration │ │ ├── check_queue.rb │ │ ├── fire_migration.rb │ │ ├── give.rb │ │ ├── method_added.rb │ │ ├── overwrite.rb │ │ ├── run.rb │ │ └── take.rb │ ├── models │ │ └── async_schema_migration.rb │ ├── mutators │ │ ├── base.rb │ │ ├── trigger_callback.rb │ │ └── turn_async.rb │ ├── railtie.rb │ ├── tracer.rb │ ├── version.rb │ ├── workers.rb │ └── workers │ │ └── sidekiq │ │ ├── check_queue_worker.rb │ │ └── fire_migration_worker.rb └── tasks │ └── rails_async_migrations.rake ├── rails_async_migrations.gemspec └── spec ├── rails_async_migrations ├── connection │ └── active_record_spec.rb ├── migration │ ├── check_queue_spec.rb │ ├── fire_migration_spec.rb │ ├── give_spec.rb │ ├── method_added_spec.rb │ ├── overwrite_spec.rb │ ├── run_spec.rb │ └── take_spec.rb ├── migration_spec.rb ├── models │ └── async_schema_migration_spec.rb ├── mutators │ ├── trigger_callback_spec.rb │ └── turn_async_spec.rb ├── workers │ └── sidekiq │ │ ├── check_queue_worker_spec.rb │ │ └── fire_migration_worker_spec.rb └── workers_spec.rb ├── rails_async_migrations_spec.rb ├── spec_helper.rb └── support ├── db ├── migrate │ └── 2010010101010_fake_migration.rb └── schema.rb └── utils_helpers.rb /.gitignore: -------------------------------------------------------------------------------- 1 | /.bundle/ 2 | /.yardoc 3 | /_yardoc/ 4 | /coverage/ 5 | /doc/ 6 | /pkg/ 7 | /spec/reports/ 8 | /tmp/ 9 | 10 | # rspec failure tracking 11 | .rspec_status 12 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --format documentation 2 | --color 3 | --require spec_helper -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | # require: rubocop-rspec 2 | 3 | AllCops: 4 | TargetRubyVersion: 2.5 5 | # RuboCop has a bunch of cops enabled by default. This setting tells RuboCop 6 | # to ignore them, so only the ones explicitly set in this file are enabled. 7 | DisabledByDefault: true 8 | Exclude: 9 | - '**/.DS_Store' 10 | - 'db/**/*' 11 | - 'tmp/**/*' 12 | - 'vendor/**/*' 13 | - 'bin/**/*' 14 | - 'log/**/*' 15 | - 'script/**/*' 16 | 17 | Include: 18 | - 'app/**/*' 19 | - 'spec/**/*' 20 | - 'lib/**/*' 21 | 22 | # Prefer &&/|| over and/or. 23 | Style/AndOr: 24 | Enabled: true 25 | 26 | # Do not use braces for hash literals when they are the last argument of a 27 | # method call. 28 | Style/BracesAroundHashParameters: 29 | Enabled: true 30 | 31 | # Align `when` with `case`. 32 | Layout/CaseIndentation: 33 | Enabled: true 34 | 35 | # Align comments with method definitions. 36 | Layout/CommentIndentation: 37 | Enabled: true 38 | 39 | # No extra empty lines. 40 | Layout/EmptyLines: 41 | Enabled: true 42 | 43 | # In a regular class definition, no empty lines around the body. 44 | Layout/EmptyLinesAroundClassBody: 45 | Enabled: true 46 | 47 | # In a regular method definition, no empty lines around the body. 48 | Layout/EmptyLinesAroundMethodBody: 49 | Enabled: true 50 | 51 | # In a regular module definition, no empty lines around the body. 52 | Layout/EmptyLinesAroundModuleBody: 53 | Enabled: true 54 | 55 | # Use Ruby >= 1.9 syntax for hashes. Prefer { a: :b } over { :a => :b }. 56 | Style/HashSyntax: 57 | Enabled: true 58 | 59 | # Two spaces, no tabs (for indentation). 60 | Layout/IndentationWidth: 61 | Enabled: true 62 | 63 | Layout/SpaceAfterColon: 64 | Enabled: true 65 | 66 | Layout/SpaceAfterComma: 67 | Enabled: true 68 | 69 | Layout/SpaceAroundEqualsInParameterDefault: 70 | Enabled: true 71 | 72 | Layout/SpaceAroundKeyword: 73 | Enabled: true 74 | 75 | Layout/SpaceAroundOperators: 76 | Enabled: true 77 | 78 | Layout/SpaceBeforeFirstArg: 79 | Enabled: true 80 | 81 | # Defining a method with parameters needs parentheses. 82 | Style/MethodDefParentheses: 83 | Enabled: true 84 | 85 | # Use `foo {}` not `foo{}`. 86 | Layout/SpaceBeforeBlockBraces: 87 | Enabled: true 88 | 89 | # Use `foo { bar }` not `foo {bar}`. 90 | Layout/SpaceInsideBlockBraces: 91 | Enabled: true 92 | 93 | # Use `{ a: 1 }` not `{a:1}`. 94 | Layout/SpaceInsideHashLiteralBraces: 95 | Enabled: true 96 | 97 | Layout/SpaceInsideParens: 98 | Enabled: true 99 | 100 | # Check quotes usage according to lint rule below. 101 | Style/StringLiterals: 102 | Enabled: true 103 | EnforcedStyle: single_quotes 104 | 105 | # Detect hard tabs, no hard tabs. 106 | Layout/Tab: 107 | Enabled: true 108 | 109 | # Blank lines should not have any spaces. 110 | Layout/TrailingBlankLines: 111 | Enabled: true 112 | 113 | # No trailing whitespace. 114 | Layout/TrailingWhitespace: 115 | Enabled: true 116 | 117 | # Use quotes for string literals when they are enough. 118 | Style/UnneededPercentQ: 119 | Enabled: true 120 | 121 | # Align `end` with the matching keyword or starting expression except for 122 | # assignments, where it should be aligned with the LHS. 123 | Layout/EndAlignment: 124 | Enabled: true 125 | EnforcedStyleAlignWith: variable 126 | 127 | # Use my_method(my_arg) not my_method( my_arg ) or my_method my_arg. 128 | Lint/RequireParentheses: 129 | Enabled: true 130 | 131 | Lint/Debugger: 132 | Enabled: true 133 | 134 | Metrics/LineLength: 135 | # This will disable the rule completely, regardless what other options you put 136 | Enabled: true 137 | # Change the default 80 chars limit value 138 | Max: 150 139 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | --- 2 | sudo: false 3 | language: ruby 4 | cache: bundler 5 | rvm: 6 | - 2.5.1 7 | before_install: gem install bundler -v 1.17.1 8 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gemspec 4 | 5 | gem 'activerecord', '~> 5.2.0' 6 | gem 'sidekiq', '~> 5.2.3' 7 | gem 'delayed_job_active_record', '~> 4.1.3' -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | PATH 2 | remote: . 3 | specs: 4 | rails_async_migrations (1.0.5) 5 | 6 | GEM 7 | remote: https://rubygems.org/ 8 | specs: 9 | actioncable (5.2.2) 10 | actionpack (= 5.2.2) 11 | nio4r (~> 2.0) 12 | websocket-driver (>= 0.6.1) 13 | actionmailer (5.2.2) 14 | actionpack (= 5.2.2) 15 | actionview (= 5.2.2) 16 | activejob (= 5.2.2) 17 | mail (~> 2.5, >= 2.5.4) 18 | rails-dom-testing (~> 2.0) 19 | actionpack (5.2.2) 20 | actionview (= 5.2.2) 21 | activesupport (= 5.2.2) 22 | rack (~> 2.0) 23 | rack-test (>= 0.6.3) 24 | rails-dom-testing (~> 2.0) 25 | rails-html-sanitizer (~> 1.0, >= 1.0.2) 26 | actionview (5.2.2) 27 | activesupport (= 5.2.2) 28 | builder (~> 3.1) 29 | erubi (~> 1.4) 30 | rails-dom-testing (~> 2.0) 31 | rails-html-sanitizer (~> 1.0, >= 1.0.3) 32 | activejob (5.2.2) 33 | activesupport (= 5.2.2) 34 | globalid (>= 0.3.6) 35 | activemodel (5.2.2) 36 | activesupport (= 5.2.2) 37 | activerecord (5.2.2) 38 | activemodel (= 5.2.2) 39 | activesupport (= 5.2.2) 40 | arel (>= 9.0) 41 | activestorage (5.2.2) 42 | actionpack (= 5.2.2) 43 | activerecord (= 5.2.2) 44 | marcel (~> 0.3.1) 45 | activesupport (5.2.2) 46 | concurrent-ruby (~> 1.0, >= 1.0.2) 47 | i18n (>= 0.7, < 2) 48 | minitest (~> 5.1) 49 | tzinfo (~> 1.1) 50 | arel (9.0.0) 51 | builder (3.2.3) 52 | coderay (1.1.2) 53 | concurrent-ruby (1.1.3) 54 | connection_pool (2.2.2) 55 | crass (1.0.4) 56 | database_cleaner (1.7.0) 57 | delayed_job (4.1.5) 58 | activesupport (>= 3.0, < 5.3) 59 | delayed_job_active_record (4.1.3) 60 | activerecord (>= 3.0, < 5.3) 61 | delayed_job (>= 3.0, < 5) 62 | diff-lcs (1.3) 63 | erubi (1.7.1) 64 | globalid (0.4.1) 65 | activesupport (>= 4.2.0) 66 | i18n (1.2.0) 67 | concurrent-ruby (~> 1.0) 68 | loofah (2.2.3) 69 | crass (~> 1.0.2) 70 | nokogiri (>= 1.5.9) 71 | mail (2.7.1) 72 | mini_mime (>= 0.1.1) 73 | marcel (0.3.3) 74 | mimemagic (~> 0.3.2) 75 | method_source (0.9.2) 76 | mimemagic (0.3.2) 77 | mini_mime (1.0.1) 78 | mini_portile2 (2.3.0) 79 | minitest (5.11.3) 80 | nio4r (2.3.1) 81 | nokogiri (1.8.5) 82 | mini_portile2 (~> 2.3.0) 83 | pry (0.12.2) 84 | coderay (~> 1.1.0) 85 | method_source (~> 0.9.0) 86 | rack (2.0.6) 87 | rack-protection (2.0.5) 88 | rack 89 | rack-test (1.1.0) 90 | rack (>= 1.0, < 3) 91 | rails (5.2.2) 92 | actioncable (= 5.2.2) 93 | actionmailer (= 5.2.2) 94 | actionpack (= 5.2.2) 95 | actionview (= 5.2.2) 96 | activejob (= 5.2.2) 97 | activemodel (= 5.2.2) 98 | activerecord (= 5.2.2) 99 | activestorage (= 5.2.2) 100 | activesupport (= 5.2.2) 101 | bundler (>= 1.3.0) 102 | railties (= 5.2.2) 103 | sprockets-rails (>= 2.0.0) 104 | rails-dom-testing (2.0.3) 105 | activesupport (>= 4.2.0) 106 | nokogiri (>= 1.6) 107 | rails-html-sanitizer (1.0.4) 108 | loofah (~> 2.2, >= 2.2.2) 109 | railties (5.2.2) 110 | actionpack (= 5.2.2) 111 | activesupport (= 5.2.2) 112 | method_source 113 | rake (>= 0.8.7) 114 | thor (>= 0.19.0, < 2.0) 115 | rake (10.5.0) 116 | redis (4.1.0) 117 | rspec (3.8.0) 118 | rspec-core (~> 3.8.0) 119 | rspec-expectations (~> 3.8.0) 120 | rspec-mocks (~> 3.8.0) 121 | rspec-core (3.8.0) 122 | rspec-support (~> 3.8.0) 123 | rspec-expectations (3.8.2) 124 | diff-lcs (>= 1.2.0, < 2.0) 125 | rspec-support (~> 3.8.0) 126 | rspec-mocks (3.8.0) 127 | diff-lcs (>= 1.2.0, < 2.0) 128 | rspec-support (~> 3.8.0) 129 | rspec-sidekiq (3.0.3) 130 | rspec-core (~> 3.0, >= 3.0.0) 131 | sidekiq (>= 2.4.0) 132 | rspec-support (3.8.0) 133 | sidekiq (5.2.3) 134 | connection_pool (~> 2.2, >= 2.2.2) 135 | rack-protection (>= 1.5.0) 136 | redis (>= 3.3.5, < 5) 137 | sprockets (3.7.2) 138 | concurrent-ruby (~> 1.0) 139 | rack (> 1, < 3) 140 | sprockets-rails (3.2.1) 141 | actionpack (>= 4.0) 142 | activesupport (>= 4.0) 143 | sprockets (>= 3.0.0) 144 | sqlite3 (1.3.13) 145 | thor (0.20.3) 146 | thread_safe (0.3.6) 147 | tzinfo (1.2.5) 148 | thread_safe (~> 0.1) 149 | websocket-driver (0.7.0) 150 | websocket-extensions (>= 0.1.0) 151 | websocket-extensions (0.1.3) 152 | 153 | PLATFORMS 154 | ruby 155 | 156 | DEPENDENCIES 157 | activerecord (~> 5.2.0) 158 | bundler (~> 1.17) 159 | database_cleaner 160 | delayed_job_active_record (~> 4.1.3) 161 | pry (~> 0.12) 162 | rails (~> 5.2) 163 | rails_async_migrations! 164 | rake (~> 10.0) 165 | rspec (~> 3.0) 166 | rspec-sidekiq 167 | sidekiq (~> 5.2.3) 168 | sqlite3 169 | 170 | BUNDLED WITH 171 | 1.17.1 172 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2018 Laurent Schaffner 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # RailsAsyncMigrations 2 | 3 | `ActiveRecord::Migration` extension to turn your migrations asynschonous in a simple and straight forward way. 4 | 5 | ## Motives 6 | 7 | This library was made with the intent to help small companies which are struggling to scale at technical level. Small projects don't need asynchronous migrations queues, and big companies build their own parallel systems when facing scaling problems, but what about medium sized companies with limited resources ? 8 | 9 | When a project grows, your database starts to be heavy and changing the data through the deployment process can be very painful. There are numerous reasons you want this process to be [at least] partially asynchronous. 10 | 11 | Most people turn heavy data changes into `rake tasks` or split workers; there are two schools of thought about this. 12 | 13 | 1. Migrations should only mutate database structures and not its data, and if it's the case, it should be split and processed via other means. 14 | 2. Migrations are everything which alter data one time, typically during a deployment of new code and structure. 15 | 16 | Turning data changes into a `rake task` can be a good idea, and it's ideal to test it out too, but sometimes you need this **fat ass loop** of **>10,000,000 records** which will be run **once, and only once** with complicated conditions to be run fast and without locking down the deployment process itself; making a `rake task` for that is overkill. After all, it will only be used once and within a specific structure / data context. This gem is here to answer this need. 17 | 18 | Some would argue there's also `rake db:seed` for that, which I agree with. The advantage over this methodology is to be able to split in different steps the updating process and have more flow control. By using this gem you can monitor the different completed changes while they are being run, because it uses the same philosophy than step-by-step migrations. Seeding data in term of flow process isn't so different from migrating a database structure, that's why so many people hack it this way (directly within the migrations). 19 | 20 | ## Warning 21 | 22 | Migrating your data isn't easy and this gem isn't some magical technology. Putting some of your migration logic into a parallel asynchronous queue has consequences. Be careful about what you turn asynchronous: 23 | 24 | - Does it have any relation with what's run synchronously ? 25 | - Should I configure my workers to repeat the migration, or kill it after one full attempt ? 26 | - Is there a risk of crash of my synchronous migration ? If so, should I let the asynchronous being spawned before safety is ensured ? 27 | - Should I use ActiveRecord::Migration functionalities (especially DDL transactions) or use it in parallel to keep a safety net on worker idempotency ? 28 | 29 | This is all up to you, but be aware this solves some problems, but makes you think of the different strategies you should adopt, depending your project. 30 | 31 | **Your asynchronous migrations should be written being aware it'll be run on a parallel daemon which can crash, restart and try things again** 32 | 33 | ## Requirements 34 | 35 | You can use this library through different background processing technologies 36 | 37 | | Type | Version | Documentation | Default | 38 | | ---------------- | ------- | --------------------------------------------- | ------- | 39 | | **Sidekiq** | 5.2.3 | https://github.com/mperham/sidekiq | NO | 40 | | **Delayed::Job** | 4.1.3 | https://github.com/collectiveidea/delayed_job | YES | 41 | 42 | Please install and configure one of those before to use this gem. If you use other libraries to setup your workers, please hit me up and I'll create additional adapters for you. 43 | 44 | This gem has been tested and is working with `ActiveRecord 5.2.2`, if you notice abnormal behavior with other versions or want it compatible with earlier versions, please hit me up. 45 | 46 | ## Installation 47 | 48 | Add this line to your application's Gemfile: 49 | 50 | ```ruby 51 | gem 'rails_async_migrations' 52 | ``` 53 | 54 | And then execute: 55 | 56 | $ bundle 57 | 58 | After the gem has been installed, use the generator to add the needed changes 59 | 60 | $ rails generate rails_async_migrations:install 61 | 62 | This will add a new migration for the table `async_schema_migrations` which will be used by the gem. You can also add the migration yourself like so: 63 | 64 | ``` 65 | class CreateAsyncSchemaMigrations < ActiveRecord::Migration[5.2] 66 | def change 67 | create_table :async_schema_migrations do |t| 68 | t.string :version 69 | t.string :direction 70 | t.string :state 71 | 72 | t.timestamps 73 | end 74 | end 75 | end 76 | ``` 77 | 78 | ## Usage 79 | 80 | To turn some of your migrations asynchronous, generate a migration as you would normally do and use the `turn_async` keyword. 81 | 82 | ``` 83 | class Test < ActiveRecord::Migration[5.2] 84 | turn_async 85 | 86 | def change 87 | # data update logic you would put into a worker here 88 | end 89 | end 90 | ``` 91 | 92 | From now on, when you run this migration it'll simply run the normal queue, but the content of `#change` will be taken away and later on executed into an asynchronous queue. 93 | 94 | What is turned asynchronous is executed exactly the same way as a classical migration, which means you can use all keywords of the classic `ActiveRecord::Migration` such as `create_table`, `add_column`, ... 95 | 96 | **It does not mean you should use them like you would in a synchronous migration.** To avoid data inconsistency, be careful about idempotency which's a natural side effect of using workers; add up conditions to make it reliable. 97 | 98 | ## Configuration 99 | 100 | **No configuration is needed to start using this library**, but some options are given to adapt to your needs. 101 | 102 | Add the following lines into your `config/initializer/` folder 103 | 104 | ``` 105 | RailsAsyncMigrations.config do |config| 106 | # :verbose can be used if you want a full log of the execution 107 | config.mode = :quiet 108 | 109 | # which adapter worker you want to use for this library 110 | # for now you have two options: :delayed_job or :sidekiq 111 | config.workers = :sidekiq 112 | end 113 | ``` 114 | 115 | ## Queue 116 | 117 | Each migration which is turned asynchronous follows each other, once one migration of the queue is ended without problem, it passes to the next one. 118 | 119 | If it fails, the error will be raised within the worker so it retries until it eventually works, or until it's considered dead. None of the further asynchronous migrations will be run until you fix the failed one, which is a good protection for data consistency. 120 | 121 | ![RailsAsyncMigrations Schema](https://cdn-images-1.medium.com/max/1600/1*VklEFF8IWnmMI6-Cq20nVA.png "RailsAsyncMigrations Schema") 122 | 123 | 124 | You can also manually launch the queue check and fire by using: 125 | 126 | $ rake rails_async_migrations:check_queue 127 | 128 | **For now, there is no rollback mechanism authorized. It means if you rollback the asynchronous migrations will be simply ignored. Handling multiple directions complexifies the build up logic and may not be needed in asynchronous cases.** 129 | 130 | ## States 131 | 132 | | State | Description | 133 | | -------------- | ---------------------------------------------------------------------------------------------------------------------------- | 134 | | **created** | the migration has just been added through the classical migration queue | 135 | | **pending** | the migration has been spotted by the asynchronous queue and will be processed | 136 | | **processing** | the migration is being processed right now | 137 | | **done** | the migration was successful | 138 | | **failed** | the migration failed while being processed, there may be other attempts to make it pass depending your workers configuration | 139 | 140 | ## Failure handling 141 | 142 | If your migration crashes, and blocks the rest of your queue but you want to execute them anyway, you can use multiple strategies to do so: 143 | 144 | - Change the code of the migration file and push it again so it passes 145 | - Remove the matching row in the `async_schema_migrations` table. In this case, if you `rollback` your migration and run them again, it'll be newly added to the rows 146 | - Update the matching row with `async_schema_migrations.state = done`. It'll be considered processed and won't ever be tried again. 147 | 148 | To find the matching migration, be aware the `version` value is always the same as the classic migrations ones, so it's pretty easy to find things around. 149 | 150 | ## Development 151 | 152 | I created this library as my company was struggling in its deployment process. It lacks functionalities but this is a good starting point; everything is easily extendable so don't hesitate to add your own needed methods to it. 153 | 154 | You're more than welcome to open an issue with feature requests so I can work on improving this library. 155 | 156 | ## Contributing 157 | 158 | 1. Fork it ( https://github.com/[my-github-username]/rails_async_migrations/fork ) 159 | 2. Create your feature branch (`git checkout -b my-new-feature`) 160 | 3. Commit your changes (`git commit -am 'Add some feature'`) 161 | 4. Push to the branch (`git push origin my-new-feature`) 162 | 5. Create a new Pull Request 163 | 164 | ## Author 165 | 166 | [Laurent Schaffner](http://www.laurentschaffner.com) 167 | 168 | ## Credits 169 | 170 | This project and its idea was inspired by [Kir Shatrov article](https://kirshatrov.com/2018/04/01/async-migrations/) on the matter, it's worth a look! 171 | 172 | ## Articles 173 | 174 | [How to turn my ActiveRecord migrations asynchronous ?](https://medium.com/@LoschCode/how-to-turn-my-activerecord-migrations-asynchronous-c160b599f38) 175 | 176 | ## License 177 | 178 | MIT License. 179 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | require "rspec/core/rake_task" 3 | 4 | RSpec::Core::RakeTask.new(:spec) 5 | 6 | task :default => :spec 7 | -------------------------------------------------------------------------------- /_resource/schema.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Loschcode/rails_async_migrations/d1c924a94f186d23fb56aac42f91ea97a958f606/_resource/schema.png -------------------------------------------------------------------------------- /_resource/schema.sketch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Loschcode/rails_async_migrations/d1c924a94f186d23fb56aac42f91ea97a958f606/_resource/schema.sketch -------------------------------------------------------------------------------- /bin/console: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require "bundler/setup" 4 | require "rails_async_migrations" 5 | 6 | # You can add fixtures and/or initialization code here to make experimenting 7 | # with your gem easier. You can also use a different console, if you like. 8 | 9 | # (If you use this, don't forget to add pry to your Gemfile!) 10 | # require "pry" 11 | # Pry.start 12 | 13 | require "irb" 14 | IRB.start(__FILE__) 15 | -------------------------------------------------------------------------------- /bin/setup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | IFS=$'\n\t' 4 | set -vx 5 | 6 | bundle install 7 | 8 | # Do any other automated setup that you need to do here 9 | -------------------------------------------------------------------------------- /lib/generators/rails_async_migrations/install_generator.rb: -------------------------------------------------------------------------------- 1 | require 'rails/generators' 2 | require 'rails/generators/migration' 3 | 4 | module RailsAsyncMigrations 5 | module Generators 6 | class InstallGenerator < ::Rails::Generators::Base 7 | include Rails::Generators::Migration 8 | source_root File.expand_path('../templates', __FILE__) 9 | desc "Add the migrations for AsyncSchemaMigration" 10 | 11 | def self.next_migration_number(path) 12 | next_migration_number = current_migration_number(path) + 1 13 | ActiveRecord::Migration.next_migration_number(next_migration_number) 14 | end 15 | 16 | def copy_migrations 17 | migration_template "create_async_schema_migrations.rb", 18 | "db/migrate/create_async_schema_migrations.rb" 19 | end 20 | end 21 | end 22 | end -------------------------------------------------------------------------------- /lib/generators/rails_async_migrations/templates/create_async_schema_migrations.rb: -------------------------------------------------------------------------------- 1 | class CreateAsyncSchemaMigrations < ActiveRecord::Migration[5.2] 2 | def change 3 | create_table :async_schema_migrations do |t| 4 | t.string :version 5 | t.string :direction 6 | t.string :state 7 | 8 | t.timestamps 9 | end 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /lib/rails_async_migrations.rb: -------------------------------------------------------------------------------- 1 | require 'active_support' 2 | require 'active_record' 3 | 4 | begin 5 | require 'sidekiq' 6 | require 'rails_async_migrations/workers/sidekiq/check_queue_worker' 7 | require 'rails_async_migrations/workers/sidekiq/fire_migration_worker' 8 | rescue LoadError 9 | end 10 | 11 | require 'rails_async_migrations/connection/active_record' 12 | require 'rails_async_migrations/config' 13 | require 'rails_async_migrations/migration' 14 | require 'rails_async_migrations/class_mutators' 15 | require 'rails_async_migrations/instance_mutators' 16 | require 'rails_async_migrations/tracer' 17 | require 'rails_async_migrations/version' 18 | require 'rails_async_migrations/railtie' if defined?(Rails) 19 | require 'rails_async_migrations/models/async_schema_migration' 20 | require 'rails_async_migrations/workers' 21 | 22 | module RailsAsyncMigrations 23 | class Error < StandardError; end 24 | 25 | class << self 26 | def config 27 | @config ||= Config.new 28 | if block_given? 29 | yield @config 30 | else 31 | @config 32 | end 33 | end 34 | 35 | def reset 36 | @config = Config.new 37 | end 38 | end 39 | end 40 | 41 | ActiveSupport.on_load(:active_record) do 42 | ActiveRecord::Migration.extend RailsAsyncMigrations::ClassMutators 43 | ActiveRecord::Migration.include RailsAsyncMigrations::InstanceMutators 44 | end 45 | -------------------------------------------------------------------------------- /lib/rails_async_migrations/class_mutators.rb: -------------------------------------------------------------------------------- 1 | require 'rails_async_migrations/mutators/base' 2 | require 'rails_async_migrations/mutators/turn_async' 3 | require 'rails_async_migrations/mutators/trigger_callback' 4 | 5 | # this is the entry point of the gem as it adds methods to the current migration class 6 | # the `self` represents the class being ran so we have to be careful as not to conflict 7 | # with the original ActiveRecord names 8 | module RailsAsyncMigrations 9 | module ClassMutators 10 | def turn_async 11 | Mutators::TurnAsync.new(self).perform 12 | end 13 | end 14 | end 15 | 16 | # def self.descendants 17 | # ObjectSpace.each_object(Class).select do 18 | # |current| current < self 19 | # end 20 | # end 21 | # descendants = self.class.descendants 22 | # descendants.delete_at descendants.index(ActiveRecord::Migration::Current) 23 | # real_self = descendants.last 24 | 25 | # puts "REAL SELF IS #{real_self}" 26 | # -------------------------------------------------------------------------------- /lib/rails_async_migrations/config.rb: -------------------------------------------------------------------------------- 1 | # configuration of the gem and 2 | # default values set here 3 | module RailsAsyncMigrations 4 | class Config 5 | attr_accessor :taken_methods, :mode, :workers 6 | 7 | def initialize 8 | @taken_methods = %i[change up down] 9 | @mode = :quiet # :verbose, :quiet 10 | @workers = :delayed_job # :sidekiq 11 | end 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /lib/rails_async_migrations/connection/active_record.rb: -------------------------------------------------------------------------------- 1 | module RailsAsyncMigrations 2 | module Connection 3 | class ActiveRecord 4 | attr_reader :current_direction 5 | 6 | def initialize(current_direction) 7 | @current_direction = current_direction 8 | end 9 | 10 | # NOTE : down isn't available 11 | # from the public API of the gem 12 | def current_version 13 | if current_direction == :down 14 | migration_context.current_version 15 | elsif current_direction == :up 16 | pending_migrations.first 17 | end 18 | end 19 | 20 | def current_migration 21 | @current_migration ||= migration_from current_version 22 | end 23 | 24 | def migration_from(version) 25 | migration_context.migrations.find do |migration| 26 | migration.version.to_s == version.to_s 27 | end 28 | end 29 | 30 | def allowed_direction? 31 | current_direction == :up 32 | end 33 | 34 | private 35 | 36 | def pending_migrations 37 | achieved_migrations - all_migrations 38 | end 39 | 40 | def achieved_migrations 41 | migration_context.migrations.collect(&:version) 42 | end 43 | 44 | def all_migrations 45 | migration_context.get_all_versions 46 | end 47 | 48 | def migration_context 49 | connection.migration_context 50 | end 51 | 52 | # NOTE: seems at it was ActiveRecord::Migrator 53 | # in anterior versions 54 | def connection 55 | @connection || ::ActiveRecord::Base.connection 56 | end 57 | end 58 | end 59 | end -------------------------------------------------------------------------------- /lib/rails_async_migrations/instance_mutators.rb: -------------------------------------------------------------------------------- 1 | require 'rails_async_migrations/mutators/base' 2 | require 'rails_async_migrations/mutators/turn_async' 3 | require 'rails_async_migrations/mutators/trigger_callback' 4 | 5 | module RailsAsyncMigrations 6 | module InstanceMutators 7 | def trigger_callback(method) 8 | Mutators::TriggerCallback.new(self, method).perform 9 | end 10 | end 11 | end -------------------------------------------------------------------------------- /lib/rails_async_migrations/migration.rb: -------------------------------------------------------------------------------- 1 | require 'rails_async_migrations/migration/check_queue' 2 | require 'rails_async_migrations/migration/fire_migration' 3 | require 'rails_async_migrations/migration/take' 4 | require 'rails_async_migrations/migration/method_added' 5 | require 'rails_async_migrations/migration/overwrite' 6 | require 'rails_async_migrations/migration/run' 7 | require 'rails_async_migrations/migration/give' 8 | 9 | # when included this class is the gateway 10 | # to the method takeing system 11 | module RailsAsyncMigrations 12 | module Migration 13 | def self.included(base) 14 | base.extend ClassMethods 15 | end 16 | 17 | module ClassMethods 18 | def method_added(name) 19 | MethodAdded.new(self, name).perform 20 | end 21 | end 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /lib/rails_async_migrations/migration/check_queue.rb: -------------------------------------------------------------------------------- 1 | # we check the state of the queue and launch run worker if needed 2 | module RailsAsyncMigrations 3 | module Migration 4 | class CheckQueue 5 | def initialize 6 | end 7 | 8 | def perform 9 | Tracer.new.verbose 'Check queue has been triggered' 10 | 11 | return if has_failures? 12 | return if has_on_going? 13 | return if no_migration? 14 | 15 | pending! 16 | fire_migration 17 | end 18 | 19 | private 20 | 21 | def fire_migration 22 | Tracer.new.verbose "Migration `#{current_migration.version}` (\##{current_migration.id}) will now be processed" 23 | Workers.new(:fire_migration).perform(current_migration.id) 24 | end 25 | 26 | def pending! 27 | current_migration.update state: 'pending' 28 | end 29 | 30 | def current_migration 31 | created_migration 32 | end 33 | 34 | def processing_migration 35 | @processing_migration ||= AsyncSchemaMigration.processing.first 36 | end 37 | 38 | def pending_migration 39 | @pending_migration ||= AsyncSchemaMigration.pending.first 40 | end 41 | 42 | def created_migration 43 | @created_migration ||= AsyncSchemaMigration.created.first 44 | end 45 | 46 | def no_migration? 47 | unless current_migration 48 | Tracer.new.verbose 'No available migration in queue, cancelling check' 49 | true 50 | end 51 | end 52 | 53 | def has_on_going? 54 | if pending_migration || processing_migration 55 | Tracer.new.verbose 'Another migration under progress, cancelling check' 56 | true 57 | end 58 | end 59 | 60 | def has_failures? 61 | if failed_migration 62 | Tracer.new.verbose 'Failing migration blocking the queue, cancelling check' 63 | true 64 | end 65 | end 66 | 67 | def failed_migration 68 | @failed_migration ||= AsyncSchemaMigration.failed.first 69 | end 70 | end 71 | end 72 | end -------------------------------------------------------------------------------- /lib/rails_async_migrations/migration/fire_migration.rb: -------------------------------------------------------------------------------- 1 | # we check the state of the queue and launch run worker if needed 2 | module RailsAsyncMigrations 3 | module Migration 4 | class FireMigration 5 | attr_reader :migration 6 | 7 | def initialize(migration_id) 8 | @migration = AsyncSchemaMigration.find(migration_id) 9 | end 10 | 11 | def perform 12 | return if done? 13 | 14 | process! 15 | run_migration 16 | done! 17 | 18 | check_queue 19 | end 20 | 21 | private 22 | 23 | def check_queue 24 | Workers.new(:check_queue).perform 25 | end 26 | 27 | def run_migration 28 | Migration::Run.new(migration.direction, migration.version).perform 29 | rescue Exception => exception 30 | failed_with! exception 31 | raise 32 | end 33 | 34 | def done? 35 | if migration.reload.state == 'done' 36 | Tracer.new.verbose "Migration #{migration.id} is already `done`, cancelling fire" 37 | return true 38 | end 39 | end 40 | 41 | def process! 42 | migration.update! state: 'processing' 43 | end 44 | 45 | def done! 46 | migration.update! state: 'done' 47 | Tracer.new.verbose "Migration #{migration.id} was correctly processed" 48 | migration.reload 49 | end 50 | 51 | def failed_with!(error) 52 | migration.update! state: 'failed' 53 | Tracer.new.verbose "Migration #{migration.id} failed with exception `#{error}`" 54 | end 55 | end 56 | end 57 | end -------------------------------------------------------------------------------- /lib/rails_async_migrations/migration/give.rb: -------------------------------------------------------------------------------- 1 | module RailsAsyncMigrations 2 | module Migration 3 | class Give 4 | attr_reader :resource_class, :method_name 5 | 6 | def initialize(resource_class, method_name) 7 | @resource_class = resource_class 8 | @method_name = method_name 9 | end 10 | 11 | def perform 12 | restore_original_method 13 | end 14 | 15 | private 16 | 17 | def restore_original_method 18 | if valid? 19 | Take.new(resource_class, method_name).suspend_take do 20 | resource_class.define_method(method_name, &method_clone) 21 | end 22 | end 23 | end 24 | 25 | def valid? 26 | temporary_instance.respond_to? clone_method_name 27 | end 28 | 29 | def clone_method_name 30 | "async_#{method_name}" 31 | end 32 | 33 | def method_clone 34 | temporary_instance.method(clone_method_name).clone 35 | end 36 | 37 | def temporary_instance 38 | @temporary_instance ||= resource_class.new 39 | end 40 | end 41 | end 42 | end -------------------------------------------------------------------------------- /lib/rails_async_migrations/migration/method_added.rb: -------------------------------------------------------------------------------- 1 | # any method added within the synchronous migration 2 | # with asynchronous directive will trigger this class 3 | module RailsAsyncMigrations 4 | module Migration 5 | class MethodAdded 6 | attr_reader :resource_class, :method_name 7 | 8 | def initialize(resource_class, method_name) 9 | @resource_class = resource_class 10 | @method_name = method_name 11 | end 12 | 13 | def perform 14 | take_and_overwrite 15 | end 16 | 17 | private 18 | 19 | def take_and_overwrite 20 | Take.new(resource_class, method_name).perform 21 | end 22 | end 23 | end 24 | end -------------------------------------------------------------------------------- /lib/rails_async_migrations/migration/overwrite.rb: -------------------------------------------------------------------------------- 1 | module RailsAsyncMigrations 2 | module Migration 3 | class Overwrite 4 | attr_reader :instance, :method_name 5 | 6 | def initialize(instance, method_name) 7 | @instance = instance 8 | @method_name = method_name 9 | end 10 | 11 | def perform 12 | dispatch_trace 13 | trigger_callback 14 | end 15 | 16 | private 17 | 18 | def dispatch_trace 19 | Tracer.new.verbose "#{instance.class}\##{method_name} was called in a taken state" 20 | end 21 | 22 | def trigger_callback 23 | instance.send(:trigger_callback, method_name) 24 | rescue NoMethodError 25 | end 26 | end 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /lib/rails_async_migrations/migration/run.rb: -------------------------------------------------------------------------------- 1 | # to run actual migration we need to require the migration files 2 | module RailsAsyncMigrations 3 | module Migration 4 | class Run 5 | attr_reader :direction, :version, :migration 6 | 7 | def initialize(direction, version) 8 | @direction = direction 9 | @version = version 10 | @migration = migration_from version 11 | 12 | ensure_data_consistency 13 | require "#{Rails.root}/#{migration.filename}" if defined? Rails 14 | end 15 | 16 | def perform 17 | give_migration_methods 18 | delete_migration_state 19 | run_migration 20 | delete_migration_state 21 | take_migration_methods 22 | end 23 | 24 | private 25 | 26 | def migration_from(version) 27 | Connection::ActiveRecord.new(direction).migration_from version 28 | end 29 | 30 | def run_migration 31 | migrator_instance.migrate 32 | end 33 | 34 | def migrator_instance 35 | @migrator_instance ||= ::ActiveRecord::Migrator.new(direction.to_sym, [migration]) 36 | end 37 | 38 | def schema_migration 39 | @schema_migration ||= ActiveRecord::SchemaMigration.find_by(version: version) 40 | end 41 | 42 | def delete_migration_state 43 | schema_migration&.delete 44 | end 45 | 46 | def class_name 47 | migration.name.constantize 48 | end 49 | 50 | def taken_methods 51 | RailsAsyncMigrations.config.taken_methods 52 | end 53 | 54 | def give_migration_methods 55 | taken_methods.each do |method_name| 56 | Migration::Give.new(class_name, method_name).perform 57 | end 58 | end 59 | 60 | def take_migration_methods 61 | taken_methods.each do |method_name| 62 | Migration::Take.new(class_name, method_name).perform 63 | end 64 | end 65 | 66 | def ensure_data_consistency 67 | unless migration 68 | raise RailsAsyncMigrations::Error, "No migration from version `#{version}`" 69 | end 70 | end 71 | end 72 | end 73 | end -------------------------------------------------------------------------------- /lib/rails_async_migrations/migration/take.rb: -------------------------------------------------------------------------------- 1 | # takes any class methods depending on a configuration list 2 | # this allow us to ignore migration without making 3 | # a parallel pipeline system 4 | module RailsAsyncMigrations 5 | module Migration 6 | class Take 7 | attr_reader :resource_class, :method_name 8 | 9 | def initialize(resource_class, method_name) 10 | @resource_class = resource_class 11 | @method_name = method_name 12 | end 13 | 14 | def perform 15 | return false unless taken_method? 16 | return false if given? 17 | 18 | preserve_method_logics 19 | 20 | suspend_take do 21 | overwrite_method 22 | end 23 | end 24 | 25 | def suspend_take(&block) 26 | give 27 | yield if block_given? 28 | take 29 | end 30 | 31 | private 32 | 33 | def given? 34 | taken == false 35 | end 36 | 37 | def clone_method_name 38 | "async_#{method_name}" 39 | end 40 | 41 | def preserve_method_logics 42 | resource_class.define_method(clone_method_name, &captured_method) 43 | end 44 | 45 | def captured_method 46 | resource_class.new.method(method_name).clone 47 | end 48 | 49 | def overwrite_method 50 | resource_class.define_method(method_name, &overwrite_closure) 51 | end 52 | 53 | def overwrite_closure 54 | proc do 55 | Overwrite.new(self, __method__).perform 56 | end 57 | end 58 | 59 | def takeable? 60 | given? && taken_method? 61 | end 62 | 63 | def taken_method? 64 | RailsAsyncMigrations.config.taken_methods.include? method_name 65 | end 66 | 67 | def take 68 | self.taken = true 69 | end 70 | 71 | def give 72 | self.taken = false 73 | end 74 | 75 | def taken=(value) 76 | resource_class.instance_variable_set(:@taken, value) 77 | end 78 | 79 | def taken 80 | resource_class.instance_variable_get(:@taken) 81 | end 82 | end 83 | end 84 | end 85 | -------------------------------------------------------------------------------- /lib/rails_async_migrations/models/async_schema_migration.rb: -------------------------------------------------------------------------------- 1 | class AsyncSchemaMigration < ActiveRecord::Base 2 | validates :version, presence: true 3 | validates :state, inclusion: { in: %w[created pending processing done failed] } 4 | validates :direction, inclusion: { in: %w[up down] } 5 | 6 | after_save :trace 7 | 8 | scope :created, -> { where(state: 'created').by_version } 9 | scope :pending, -> { where(state: 'pending').by_version } 10 | scope :processing, -> { where(state: 'processing').by_version } 11 | scope :done, -> { where(state: 'done').by_version } 12 | scope :failed, -> { where(state: 'failed').by_version } 13 | scope :by_version, -> { order(version: :asc) } 14 | 15 | def trace 16 | RailsAsyncMigrations::Tracer.new.verbose "Asynchronous migration `#{id}` is now `#{state}`" 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /lib/rails_async_migrations/mutators/base.rb: -------------------------------------------------------------------------------- 1 | module RailsAsyncMigrations 2 | module Mutators 3 | class Base 4 | # def migration_class 5 | # instance.class 6 | # end 7 | end 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /lib/rails_async_migrations/mutators/trigger_callback.rb: -------------------------------------------------------------------------------- 1 | module RailsAsyncMigrations 2 | module Mutators 3 | class TriggerCallback < Base 4 | attr_reader :instance, :method_name 5 | 6 | def initialize(instance, method_name) 7 | @instance = instance 8 | @method_name = method_name 9 | end 10 | 11 | # this method can be called multiple times (we should see what manages this actually) 12 | # if you use up down and change it'll be called 3 times for example 13 | def perform 14 | unless active_record.allowed_direction? 15 | Tracer.new.verbose "Direction `#{direction}` not allowed." 16 | return 17 | end 18 | 19 | enqueue_asynchronous unless already_enqueued? 20 | check_queue 21 | end 22 | 23 | private 24 | 25 | def enqueue_asynchronous 26 | AsyncSchemaMigration.create!( 27 | version: active_record.current_version, 28 | direction: active_record.current_direction, 29 | state: 'created' 30 | ) 31 | end 32 | 33 | def already_enqueued? 34 | AsyncSchemaMigration.find_by( 35 | version: active_record.current_version, 36 | direction: active_record.current_direction 37 | ) 38 | end 39 | 40 | def check_queue 41 | Workers.new(:check_queue).perform 42 | end 43 | 44 | def active_record 45 | @active_record ||= Connection::ActiveRecord.new(direction) 46 | end 47 | 48 | def direction 49 | if instance.reverting? 50 | :down 51 | else 52 | :up 53 | end 54 | end 55 | end 56 | end 57 | end 58 | -------------------------------------------------------------------------------- /lib/rails_async_migrations/mutators/turn_async.rb: -------------------------------------------------------------------------------- 1 | module RailsAsyncMigrations 2 | module Mutators 3 | class TurnAsync < Base 4 | attr_reader :migration_class 5 | 6 | def initialize(migration_class) 7 | @migration_class = migration_class 8 | end 9 | 10 | def perform 11 | Tracer.new.verbose '`turn_async` has been triggered' 12 | alter_migration 13 | end 14 | 15 | private 16 | 17 | def alter_migration 18 | Tracer.new.verbose "#{migration_class} is now asynchronous" 19 | migration_class.include RailsAsyncMigrations::Migration 20 | end 21 | end 22 | end 23 | end -------------------------------------------------------------------------------- /lib/rails_async_migrations/railtie.rb: -------------------------------------------------------------------------------- 1 | module RailsAsyncMigrations 2 | class Railtie < ::Rails::Railtie 3 | rake_tasks do 4 | load 'tasks/rails_async_migrations.rake' 5 | end 6 | end 7 | end -------------------------------------------------------------------------------- /lib/rails_async_migrations/tracer.rb: -------------------------------------------------------------------------------- 1 | # log things and dispatch them wherever 2 | # depending on the context mode 3 | module RailsAsyncMigrations 4 | class Tracer 5 | def initialize 6 | end 7 | 8 | def verbose(text) 9 | return unless verbose? 10 | puts "[VERBOSE] #{text}" 11 | end 12 | 13 | private 14 | 15 | def verbose? 16 | mode == :verbose 17 | end 18 | 19 | def mode 20 | RailsAsyncMigrations.config.mode 21 | end 22 | end 23 | end -------------------------------------------------------------------------------- /lib/rails_async_migrations/version.rb: -------------------------------------------------------------------------------- 1 | module RailsAsyncMigrations 2 | VERSION = "1.0.5" 3 | end 4 | -------------------------------------------------------------------------------- /lib/rails_async_migrations/workers.rb: -------------------------------------------------------------------------------- 1 | # we check the state of the queue and launch run worker if needed 2 | module RailsAsyncMigrations 3 | class Workers 4 | ALLOWED = [:check_queue, :fire_migration].freeze 5 | attr_reader :called_worker 6 | 7 | def initialize(called_worker) 8 | @called_worker = called_worker # :check_queue, :fire_migration 9 | ensure_worker_presence 10 | end 11 | 12 | def perform(args = []) 13 | return unless ALLOWED.include? called_worker 14 | self.send called_worker, *args 15 | end 16 | 17 | private 18 | 19 | def check_queue(*args) 20 | case workers_type 21 | when :sidekiq 22 | Workers::Sidekiq::CheckQueueWorker.perform_async(*args) 23 | when :delayed_job 24 | ::Delayed::Job.enqueue Migration::CheckQueue.new 25 | end 26 | end 27 | 28 | def fire_migration(*args) 29 | case workers_type 30 | when :sidekiq 31 | Workers::Sidekiq::FireMigrationWorker.perform_async(*args) 32 | when :delayed_job 33 | ::Delayed::Job.enqueue Migration::FireMigration.new(*args) 34 | end 35 | end 36 | 37 | def workers_type 38 | RailsAsyncMigrations.config.workers 39 | end 40 | 41 | def ensure_worker_presence 42 | case workers_type 43 | when :sidekiq 44 | unless defined? ::Sidekiq::Worker 45 | raise Error, 'Please install Sidekiq before to set it as worker adapter' 46 | end 47 | when :delayed_job 48 | unless defined? ::Delayed::Job 49 | raise Error, 'Please install Delayed::Job before to set it as worker adapter' 50 | end 51 | end 52 | end 53 | end 54 | end 55 | -------------------------------------------------------------------------------- /lib/rails_async_migrations/workers/sidekiq/check_queue_worker.rb: -------------------------------------------------------------------------------- 1 | # we check the state of the queue and launch run worker if needed 2 | module RailsAsyncMigrations 3 | class Workers 4 | module Sidekiq 5 | class CheckQueueWorker 6 | include ::Sidekiq::Worker 7 | 8 | sidekiq_options queue: :default 9 | 10 | def perform 11 | Migration::CheckQueue.new.perform 12 | end 13 | end 14 | end 15 | end 16 | end 17 | -------------------------------------------------------------------------------- /lib/rails_async_migrations/workers/sidekiq/fire_migration_worker.rb: -------------------------------------------------------------------------------- 1 | # we check the state of the queue and launch run worker if needed 2 | module RailsAsyncMigrations 3 | class Workers 4 | module Sidekiq 5 | class FireMigrationWorker 6 | include ::Sidekiq::Worker 7 | 8 | sidekiq_options queue: :default 9 | 10 | def perform(migration_id) 11 | Migration::FireMigration.new( 12 | migration_id 13 | ).perform 14 | end 15 | end 16 | end 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /lib/tasks/rails_async_migrations.rake: -------------------------------------------------------------------------------- 1 | namespace :rails_async_migrations do 2 | desc 'Check our current asynchronous queue and fire migrations if possible' 3 | task :check_queue do 4 | RailsAsyncMigrations::Workers.new(:check_queue).perform 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /rails_async_migrations.gemspec: -------------------------------------------------------------------------------- 1 | lib = File.expand_path('../lib', __FILE__) 2 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 3 | require 'rails_async_migrations/version' 4 | 5 | Gem::Specification.new do |spec| 6 | spec.name = 'rails_async_migrations' 7 | spec.version = RailsAsyncMigrations::VERSION 8 | spec.authors = ['Laurent Schaffner'] 9 | spec.email = ['laurent.schaffner.code@gmail.com'] 10 | 11 | spec.summary = 'Asynchronous support for ActiveRecord::Migration' 12 | spec.homepage = 'https://www.laurent.tech' 13 | spec.license = 'MIT' 14 | 15 | spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do 16 | `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } 17 | end 18 | spec.bindir = 'exe' 19 | spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } 20 | spec.require_paths = ['lib'] 21 | 22 | spec.add_development_dependency 'bundler', '~> 1.17' 23 | spec.add_development_dependency 'pry', '~> 0.12' 24 | spec.add_development_dependency 'rails', '~> 5.2' 25 | spec.add_development_dependency 'rake', '~> 10.0' 26 | spec.add_development_dependency 'sqlite3' 27 | spec.add_development_dependency 'rspec', '~> 3.0' 28 | spec.add_development_dependency 'database_cleaner' 29 | spec.add_development_dependency 'rspec-sidekiq' 30 | end 31 | -------------------------------------------------------------------------------- /spec/rails_async_migrations/connection/active_record_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe RailsAsyncMigrations::Connection::ActiveRecord do 2 | let(:active_record_instance) { resource_class.new } 3 | let(:direction) { :up } 4 | let(:instance) { described_class.new(direction) } 5 | 6 | before do 7 | fake_migration_proxy! 8 | fake_version! 9 | fake_migrate! 10 | end 11 | 12 | context '#current_version' do 13 | subject { instance.current_version } 14 | it { is_expected.to eq('00000') } 15 | end 16 | 17 | context '#current_migration' do 18 | subject { instance.current_migration } 19 | it { is_expected.to be_instance_of(FakeMigrationProxy) } 20 | end 21 | 22 | context '#migration_from(version)' do 23 | let(:version) { '000000000' } 24 | subject { instance.migration_from(version) } 25 | it { is_expected.to be_instance_of(FakeMigrationProxy) } 26 | end 27 | 28 | context '#allowed_direction?' do 29 | subject { instance.allowed_direction? } 30 | 31 | it { is_expected.to be_truthy } 32 | context 'with a down direction' do 33 | let(:direction) { :down } 34 | it { is_expected.to be_falsey } 35 | end 36 | end 37 | end -------------------------------------------------------------------------------- /spec/rails_async_migrations/migration/check_queue_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe RailsAsyncMigrations::Migration::CheckQueue do 2 | let(:instance) { described_class.new } 3 | 4 | context '#perform' do 5 | subject { instance.perform } 6 | it { is_expected.to be_falsey } 7 | end 8 | end -------------------------------------------------------------------------------- /spec/rails_async_migrations/migration/fire_migration_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe RailsAsyncMigrations::Migration::FireMigration do 2 | let(:instance) { described_class.new(migration.id) } 3 | let(:migration) do 4 | AsyncSchemaMigration.create!( 5 | version: '00000', 6 | state: 'created', 7 | direction: 'up' 8 | ) 9 | end 10 | 11 | context '#perform' do 12 | subject { instance.perform } 13 | it { expect { subject }.to raise_error(RailsAsyncMigrations::Error) } 14 | end 15 | end -------------------------------------------------------------------------------- /spec/rails_async_migrations/migration/give_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe RailsAsyncMigrations::Migration::Give do 2 | let(:instance) { described_class.new(resource_class, method_name) } 3 | let(:resource_class) { FakeMigration } 4 | let(:method_name) { :change } 5 | 6 | context '#perform' do 7 | subject { instance.perform } 8 | it { is_expected.to be_truthy } 9 | end 10 | end -------------------------------------------------------------------------------- /spec/rails_async_migrations/migration/method_added_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe RailsAsyncMigrations::Migration::MethodAdded do 2 | let(:instance) { described_class.new(resource_class, method_name) } 3 | let(:resource_class) { FakeMigration } 4 | let(:method_name) { :change } 5 | 6 | context '#perform' do 7 | subject { instance.perform } 8 | it { is_expected.to be_truthy } 9 | end 10 | end -------------------------------------------------------------------------------- /spec/rails_async_migrations/migration/overwrite_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe RailsAsyncMigrations::Migration::Overwrite do 2 | let(:instance) { described_class.new(class_instance, method_name) } 3 | let(:class_instance) { FakeMigration.new } 4 | let(:method_name) { :change } 5 | 6 | before do 7 | fake_migration_proxy! 8 | fake_version! 9 | config_worker_as :delayed_job 10 | end 11 | 12 | context '#perform' do 13 | subject { instance.perform } 14 | it { is_expected.to be_instance_of(Delayed::Backend::ActiveRecord::Job) } 15 | 16 | context 'with sidekiq' do 17 | before do 18 | config_worker_as :sidekiq 19 | end 20 | 21 | it { is_expected.to be_instance_of(String) } 22 | end 23 | end 24 | end -------------------------------------------------------------------------------- /spec/rails_async_migrations/migration/run_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe RailsAsyncMigrations::Migration::Overwrite do 2 | let(:instance) { described_class.new(direction, version) } 3 | let(:version) { '00000000' } 4 | let(:direction) { :up } 5 | 6 | context '#perform' do 7 | subject { instance.perform } 8 | it { is_expected.to be_falsey } 9 | end 10 | end -------------------------------------------------------------------------------- /spec/rails_async_migrations/migration/take_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe RailsAsyncMigrations::Migration::Take do 2 | let(:resource_class) { FakeClass } 3 | let(:resource_instance) { resource_class.new } 4 | let(:method) { :free_method } 5 | let(:instance) { described_class.new(resource_class, method) } 6 | 7 | let(:change_to_it_passed) { resource_class.define_method(method, -> { 'it passed' }) } 8 | 9 | before { change_to_it_passed } 10 | subject { instance.perform } 11 | 12 | context 'method not in the takeable list' do 13 | it { is_expected.to be(false) } 14 | 15 | context 'does not alter the method' do 16 | before { subject } 17 | it { expect(resource_instance.send(method)).to eq('it passed') } 18 | end 19 | end 20 | 21 | context 'method in the takeable list' do 22 | let(:method) { :change } 23 | it { is_expected.to eq(true) } 24 | 25 | context 'alter the method' do 26 | before { subject } 27 | it { expect(resource_instance.send(method)).not_to eq('it passed') } 28 | end 29 | end 30 | end 31 | 32 | class FakeClass 33 | def free_method 34 | true 35 | end 36 | 37 | def change 38 | 'it passed' 39 | end 40 | 41 | def up 42 | true 43 | end 44 | 45 | def down 46 | true 47 | end 48 | end -------------------------------------------------------------------------------- /spec/rails_async_migrations/migration_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe RailsAsyncMigrations::Migration do 2 | let(:resource_class) { FakeClass } 3 | let(:instance) { resource_class.new } 4 | let(:method) { :free_method } 5 | let(:change_to_true) { resource_class.define_method(method, -> { true }) } 6 | let(:change_to_false) { resource_class.define_method(method, -> { false }) } 7 | 8 | # we reset the class each time 9 | # as RSpec won't hot-reload it 10 | before { change_to_true } 11 | 12 | context 'without taker included within a class' do 13 | it { expect { change_to_false }.to change { instance.send(method) }.from(true).to(false) } 14 | end 15 | 16 | context 'with taker included within a class' do 17 | before { resource_class.include described_class } 18 | 19 | context 'method not in the takeable list' do 20 | it { expect { change_to_false }.to change { instance.send(method) }.from(true).to(false) } 21 | end 22 | 23 | context 'method in the takeable list' do 24 | let(:method) { :change } 25 | it { expect { change_to_false }.not_to change { instance.send(method) } } 26 | end 27 | end 28 | end 29 | 30 | class FakeClass 31 | def free_method 32 | true 33 | end 34 | 35 | def change 36 | true 37 | end 38 | 39 | def up 40 | true 41 | end 42 | 43 | def down 44 | true 45 | end 46 | end -------------------------------------------------------------------------------- /spec/rails_async_migrations/models/async_schema_migration_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe AsyncSchemaMigration, type: :model do 2 | describe '#associations' do 3 | end 4 | end -------------------------------------------------------------------------------- /spec/rails_async_migrations/mutators/trigger_callback_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe RailsAsyncMigrations::Mutators::TriggerCallback do 2 | let(:instance) { described_class.new(class_instance, method_name) } 3 | let(:class_instance) { FakeMigration.new } 4 | let(:method_name) { :change } 5 | 6 | before do 7 | fake_migration_proxy! 8 | fake_version! 9 | end 10 | 11 | context '#perform' do 12 | subject { instance.perform } 13 | it { is_expected.to be_truthy } 14 | end 15 | end -------------------------------------------------------------------------------- /spec/rails_async_migrations/mutators/turn_async_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe RailsAsyncMigrations::Mutators::TurnAsync do 2 | let(:instance) { described_class.new(migration_class) } 3 | let(:migration_class) { FakeMigration } 4 | 5 | context '#perform' do 6 | subject { instance.perform } 7 | it { is_expected.to be_truthy } 8 | end 9 | end -------------------------------------------------------------------------------- /spec/rails_async_migrations/workers/sidekiq/check_queue_worker_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rails_async_migrations/workers/sidekiq/check_queue_worker' 2 | 3 | RSpec.describe RailsAsyncMigrations::Workers::Sidekiq::CheckQueueWorker do 4 | it { is_expected.to be_processed_in :default } 5 | end -------------------------------------------------------------------------------- /spec/rails_async_migrations/workers/sidekiq/fire_migration_worker_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rails_async_migrations/workers/sidekiq/fire_migration_worker' 2 | 3 | RSpec.describe RailsAsyncMigrations::Workers::Sidekiq::FireMigrationWorker do 4 | it { is_expected.to be_processed_in :default } 5 | end -------------------------------------------------------------------------------- /spec/rails_async_migrations/workers_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe RailsAsyncMigrations::Workers do 2 | let(:called_worker) { :check_queue } 3 | let(:instance) { described_class.new(called_worker) } 4 | let(:args) { [] } 5 | let(:async_schema_migration) do 6 | AsyncSchemaMigration.create!( 7 | version: '00000', 8 | direction: 'up', 9 | state: 'created' 10 | ) 11 | end 12 | 13 | subject { instance.perform(args) } 14 | 15 | context 'through delayed_job' do 16 | before do 17 | config_worker_as :delayed_job 18 | end 19 | 20 | context 'with :check_queue' do 21 | it { is_expected.to be_truthy } 22 | end 23 | 24 | context 'with :fire_migration' do 25 | let(:called_worker) { :fire_migration } 26 | let(:args) { [async_schema_migration.id] } 27 | 28 | it { expect { subject }.to raise_error(RailsAsyncMigrations::Error) } 29 | end 30 | end 31 | 32 | context 'through sidekiq' do 33 | context 'with :check_queue' do 34 | it { is_expected.to be_truthy } 35 | end 36 | 37 | context 'with :fire_migration' do 38 | let(:called_worker) { :fire_migration } 39 | let(:args) { [async_schema_migration.id] } 40 | 41 | it { expect { subject }.to raise_error(RailsAsyncMigrations::Error) } 42 | end 43 | end 44 | end -------------------------------------------------------------------------------- /spec/rails_async_migrations_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe RailsAsyncMigrations do 2 | it { expect(RailsAsyncMigrations::VERSION).not_to be_nil } 3 | it { expect(RailsAsyncMigrations.config).not_to be_nil } 4 | end 5 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require 'bundler/setup' 2 | require 'pry' 3 | require 'rails_async_migrations' 4 | 5 | require 'logger' 6 | require 'database_cleaner' 7 | require 'rspec-sidekiq' 8 | require 'delayed_job_active_record' 9 | 10 | RSpec.configure do |config| 11 | config.example_status_persistence_file_path = '.rspec_status' 12 | config.disable_monkey_patching! 13 | 14 | config.expect_with :rspec do |c| 15 | c.syntax = :expect 16 | end 17 | 18 | ActiveRecord::Base.establish_connection( 19 | adapter: 'sqlite3', 20 | database: ':memory:' 21 | ) 22 | ActiveRecord::Schema.verbose = false 23 | 24 | # Add additional requires below this line. Rails is not loaded until this point! 25 | Dir['spec/support/**/*.rb'].each do |file| 26 | load file 27 | end 28 | 29 | config.include UtilsHelpers 30 | 31 | load 'support/db/schema.rb' 32 | load 'support/db/migrate/2010010101010_fake_migration.rb' 33 | ActiveRecord::Migrator.migrations_paths << 'support/db/migrate' 34 | 35 | 36 | DatabaseCleaner.strategy = :truncation 37 | 38 | 39 | config.before :each do 40 | Delayed::Worker.delay_jobs = false 41 | DatabaseCleaner.start 42 | end 43 | 44 | config.after :each do 45 | Delayed::Worker.delay_jobs = true 46 | DatabaseCleaner.clean 47 | end 48 | end 49 | 50 | RSpec::Sidekiq.configure do |config| 51 | config.clear_all_enqueued_jobs = true 52 | config.enable_terminal_colours = true 53 | config.warn_when_jobs_not_processed_by_sidekiq = true 54 | end -------------------------------------------------------------------------------- /spec/support/db/migrate/2010010101010_fake_migration.rb: -------------------------------------------------------------------------------- 1 | class FakeMigration < ActiveRecord::Migration[5.2] 2 | turn_async 3 | 4 | def change 5 | create_table 'tests' do |t| 6 | t.string :test 7 | end 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /spec/support/db/schema.rb: -------------------------------------------------------------------------------- 1 | ActiveRecord::Schema.define(version: 2018_12_17_201613) do 2 | create_table "async_schema_migrations", force: :cascade do |t| 3 | t.string "version" 4 | t.string "direction" 5 | t.string "state" 6 | t.datetime "created_at", null: false 7 | t.datetime "updated_at", null: false 8 | end 9 | 10 | create_table "delayed_jobs", force: :cascade do |t| 11 | t.integer "priority", default: 0, null: false 12 | t.integer "attempts", default: 0, null: false 13 | t.text "handler", null: false 14 | t.text "last_error" 15 | t.datetime "run_at" 16 | t.datetime "locked_at" 17 | t.datetime "failed_at" 18 | t.string "locked_by" 19 | t.string "queue" 20 | t.datetime "created_at" 21 | t.datetime "updated_at" 22 | t.index ["priority", "run_at"], name: "delayed_jobs_priority" 23 | end 24 | end -------------------------------------------------------------------------------- /spec/support/utils_helpers.rb: -------------------------------------------------------------------------------- 1 | module UtilsHelpers 2 | def config_worker_as(worker) 3 | RailsAsyncMigrations.config do |config| 4 | config.workers = worker 5 | end 6 | end 7 | 8 | def fake_version! 9 | allow_any_instance_of( 10 | RailsAsyncMigrations::Connection::ActiveRecord 11 | ).to receive(:current_version).and_return('00000') 12 | end 13 | 14 | def fake_migrate! 15 | load 'support/db/migrate/2010010101010_fake_migration.rb' 16 | FakeMigration.new.change 17 | end 18 | 19 | def fake_migration_proxy! 20 | allow_any_instance_of( 21 | RailsAsyncMigrations::Connection::ActiveRecord 22 | ).to receive(:migration_from).and_return( 23 | FakeMigrationProxy.new 24 | ) 25 | end 26 | end 27 | 28 | class FakeMigrationProxy 29 | def disable_ddl_transaction 30 | true 31 | end 32 | 33 | def migrate(direction) 34 | true 35 | end 36 | 37 | def name 38 | 'FakeMigration' 39 | end 40 | 41 | def version 42 | '2010010101010' 43 | end 44 | 45 | def filename 46 | 'db/migrate/2010010101010_fake_migration.rb' 47 | end 48 | 49 | def scope 50 | '' 51 | end 52 | end 53 | 54 | # def fake_connection! 55 | # allow(::ActiveRecord::Base).to receive(:connection).and_return(FakeConnection.new) 56 | # end 57 | 58 | # class FakeConnection 59 | # def migration_context 60 | # FakeMigrationContext.new 61 | # end 62 | # end 63 | 64 | # class FakeMigrationContext 65 | # def migrations 66 | # [FakeProxy.new] 67 | # end 68 | 69 | # def get_all_versions 70 | # [FakeProxy.new] 71 | # end 72 | # end 73 | 74 | # class FakeProxy 75 | # def version 76 | # 000000000 77 | # end 78 | # end --------------------------------------------------------------------------------