├── .gitignore ├── .travis.yml ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── examples └── micrate ├── shard.yml ├── spec ├── cli_spec.cr ├── micrate_spec.cr ├── migration_spec.cr └── spec_helper.cr └── src ├── micrate-bin.cr ├── micrate.cr └── micrate ├── cli.cr ├── db.cr ├── db ├── dialect.cr ├── mysql.cr ├── postgres.cr └── sqlite3.cr ├── migration.cr ├── statement_builder.cr └── version.cr /.gitignore: -------------------------------------------------------------------------------- 1 | /doc/ 2 | /libs/ 3 | /lib/ 4 | /bin/ 5 | /.shards/ 6 | 7 | 8 | # Libraries don't need dependency lock 9 | # Dependencies will be locked in application that uses them 10 | /shard.lock 11 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: crystal 2 | crystal: 3 | - latest 4 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM crystallang/crystal:1.0.0 2 | 3 | # Install Dependencies 4 | ARG DEBIAN_FRONTEND=noninteractive 5 | RUN apt-get update -qq && apt-get install -y --no-install-recommends libpq-dev libsqlite3-dev libmysqlclient-dev libreadline-dev git curl vim netcat 6 | 7 | WORKDIR /opt/micrate 8 | 9 | # Build Amber 10 | ENV PATH /opt/micrate/bin:$PATH 11 | COPY . /opt/micrate 12 | RUN shards build micrate 13 | 14 | CMD ["micrate", "up"] 15 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2017-2019 Amber Team 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PREFIX=/usr/local 2 | INSTALL_DIR=$(PREFIX)/bin 3 | MICRATE_SYSTEM=$(INSTALL_DIR)/micrate 4 | 5 | OUT_DIR=$(CURDIR)/bin 6 | MICRATE=$(OUT_DIR)/micrate 7 | MICRATE_SOURCES=$(shell find src/ -type f -name '*.cr') 8 | 9 | all: build 10 | 11 | build: lib $(MICRATE) 12 | 13 | lib: 14 | @shards install --production 15 | 16 | $(MICRATE): $(MICRATE_SOURCES) | $(OUT_DIR) 17 | @echo "Building micrate in $@" 18 | @crystal build -o $@ src/micrate-bin.cr -p --no-debug 19 | 20 | $(OUT_DIR) $(INSTALL_DIR): 21 | @mkdir -p $@ 22 | 23 | run: 24 | $(MICRATE) 25 | 26 | install: build | $(INSTALL_DIR) 27 | @rm -f $(MICRATE_SYSTEM) 28 | @cp $(MICRATE) $(MICRATE_SYSTEM) 29 | 30 | link: build | $(INSTALL_DIR) 31 | @echo "Symlinking $(MICRATE) to $(MICRATE_SYSTEM)" 32 | @ln -s $(MICRATE) $(MICRATE_SYSTEM) 33 | 34 | force_link: build | $(INSTALL_DIR) 35 | @echo "Symlinking $(MICRATE) to $(MICRATE_SYSTEM)" 36 | @ln -sf $(MICRATE) $(MICRATE_SYSTEM) 37 | 38 | clean: 39 | rm -rf $(MICRATE) 40 | 41 | distclean: 42 | rm -rf $(MICRATE) .crystal .shards libs lib 43 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # micrate 2 | 3 | Micrate is a database migration tool written in Crystal. 4 | 5 | It is inspired by [goose](https://bitbucket.org/liamstask/goose/). Some code was ported from there too, so check it out. 6 | 7 | Micrate currently supports migrations for Postgres, Mysql and SQLite3, but it should be easy to add support for any other database engine with an existing [crystal-db API](https://github.com/crystal-lang/crystal-db) driver. 8 | 9 | ## Command line 10 | 11 | To install the standalone binary tool check out the releases page, or use homebrew: 12 | 13 | ``` 14 | $ brew tap amberframework/micrate 15 | $ brew install micrate 16 | ``` 17 | 18 | Execute `micrate help` for usage instructions. Micrate will connect to the database specified by the `DATABASE_URL` environment variable. 19 | 20 | To create a new migration use the `scaffold` subcommand. For example, `micrate scaffold add_users_table` will create a new SQL migration file with a name such as `db/migrations/20160524162446_add_users_table.sql` that looks like this: 21 | 22 | ```sql 23 | -- +micrate Up 24 | -- SQL in section 'Up' is executed when this migration is applied 25 | 26 | 27 | -- +micrate Down 28 | -- SQL section 'Down' is executed when this migration is rolled back 29 | ``` 30 | 31 | Comments that start with `+micrate` are interpreted by micrate when running your migrations. In this case, the `Up` and `Down` directives are used to indicate which SQL statements must be run when applying or reverting a migration. You can now go along and write your migration like this: 32 | 33 | ```sql 34 | -- +micrate Up 35 | CREATE TABLE users(id INT PRIMARY KEY, email VARCHAR NOT NULL); 36 | 37 | -- +micrate Down 38 | DROP TABLE users; 39 | ``` 40 | Now run it using `micrate up`. This command will execute all pending migrations: 41 | 42 | ``` 43 | $ micrate up 44 | Migrating db, current version: 0, target: 20160524162947 45 | OK 20160524162446_add_users_table.sql 46 | 47 | $ micrate dbversion # at any time you can find out the current version of the database 48 | 20160524162446 49 | ``` 50 | 51 | If you ever need to roll back the last migration, you can do so by executing `micrate down`. There's also `micrate redo` which rolls back the last migration and applies it again. Last but not least: use `micrate status` to find out the state of each migration: 52 | 53 | ``` 54 | $ micrate status 55 | Applied At Migration 56 | ======================================= 57 | 2016-05-24 16:31:07 UTC -- 20160524162446_add_users_table.sql 58 | Pending -- 20160524163425_add_address_to_users.sql 59 | ``` 60 | 61 | If using complex statements that might contain semicolons, you must give micrate a hint on how to split the script into separate statements. You can do this with `StatementBegin` and `StatementEnd` directives: (thanks [goose](https://bitbucket.org/liamstask/goose/) for this!) 62 | 63 | ``` 64 | -- +micrate Up 65 | -- +micrate StatementBegin 66 | CREATE OR REPLACE FUNCTION histories_partition_creation( DATE, DATE ) 67 | returns void AS $$ 68 | DECLARE 69 | create_query text; 70 | BEGIN 71 | FOR create_query IN SELECT 72 | 'CREATE TABLE IF NOT EXISTS histories_' 73 | || TO_CHAR( d, 'YYYY_MM' ) 74 | || ' ( CHECK( created_at >= timestamp ''' 75 | || TO_CHAR( d, 'YYYY-MM-DD 00:00:00' ) 76 | || ''' AND created_at < timestamp ''' 77 | || TO_CHAR( d + INTERVAL '1 month', 'YYYY-MM-DD 00:00:00' ) 78 | || ''' ) ) inherits ( histories );' 79 | FROM generate_series( $1, $2, '1 month' ) AS d 80 | LOOP 81 | EXECUTE create_query; 82 | END LOOP; -- LOOP END 83 | END; -- FUNCTION END 84 | $$ 85 | language plpgsql; 86 | -- +micrate StatementEnd 87 | ``` 88 | 89 | ## API 90 | 91 | To use the Crystal API, add this to your application's `shard.yml`: 92 | 93 | ```yaml 94 | dependencies: 95 | micrate: 96 | github: amberframework/micrate 97 | ``` 98 | 99 | This allows you to programatically use micrate's features. You'll see the `Micrate` module has an equivalent for every CLI command. If you need to use micrate's CLI without installing the tool (which could be convenient in a CI environment), you can write a runner script as follows: 100 | 101 | ```crystal 102 | #! /usr/bin/env crystal 103 | # 104 | # To build a standalone command line client, require the 105 | # driver you wish to use and use `Micrate::Cli`. 106 | # 107 | 108 | require "micrate" 109 | require "pg" 110 | 111 | Micrate::DB.connection_url = "postgresql://..." 112 | Micrate::Cli.run 113 | ``` 114 | 115 | ## Contributing 116 | 117 | 1. Fork it ( https://github.com/amberframework/micrate/fork ) 118 | 2. Create your feature branch (git checkout -b my-new-feature) 119 | 3. Commit your changes (git commit -am 'Add some feature') 120 | 4. Push to the branch (git push origin my-new-feature) 121 | 5. Create a new Pull Request 122 | 123 | ## Contributors 124 | 125 | - [juanedi](https://github.com/juanedi) - creator, maintainer 126 | -------------------------------------------------------------------------------- /examples/micrate: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env crystal 2 | 3 | # 4 | # To build a standalone command line client, require the 5 | # driver you wish to use and use `Micrate::Cli`. 6 | # 7 | 8 | require "../src/micrate" 9 | require "pg" 10 | 11 | Micrate::DB.connection_url = "postgresql://..." 12 | Micrate::Cli.run 13 | -------------------------------------------------------------------------------- /shard.yml: -------------------------------------------------------------------------------- 1 | name: micrate 2 | version: 0.15.1 3 | crystal: ">= 0.36.1, < 2.0.0" 4 | 5 | authors: 6 | - Juan Edi 7 | 8 | maintainers: 9 | - Isaac Sloan 10 | - Dru Jensen 11 | 12 | targets: 13 | micrate: 14 | main: src/micrate-bin.cr 15 | 16 | scripts: 17 | postinstall: shards build 18 | 19 | executables: 20 | - micrate 21 | 22 | dependencies: 23 | db: 24 | github: crystal-lang/crystal-db 25 | version: ~> 0.11.0 26 | pg: 27 | github: will/crystal-pg 28 | version: ~> 0.26.0 29 | mysql: 30 | github: crystal-lang/crystal-mysql 31 | version: ~> 0.14.0 32 | sqlite3: 33 | github: crystal-lang/crystal-sqlite3 34 | version: ~> 0.19.0 35 | 36 | development_dependencies: 37 | spectator: 38 | gitlab: arctic-fox/spectator 39 | version: ~> 0.11.3 40 | -------------------------------------------------------------------------------- /spec/cli_spec.cr: -------------------------------------------------------------------------------- 1 | require "./spec_helper" 2 | 3 | Spectator.describe Micrate::Cli do 4 | # mock File do 5 | # stub self.delete(path : Path | String) { nil } 6 | # end 7 | 8 | # mock Micrate::DB do 9 | # stub self.connect() { nil } 10 | # end 11 | 12 | # describe "#drop_database" do 13 | # context "sqlite3" do 14 | # it "deletes the file" do 15 | # Micrate::DB.connection_url = "sqlite3:myfile" 16 | # Micrate::Cli.drop_database 17 | # expect(File).to have_received(:delete).with("myfile") 18 | # end 19 | # end 20 | 21 | # context "postgres" do 22 | # it "calls drop database" do 23 | # Micrate::DB.connection_url = "postgres://user:pswd@host:5432/database" 24 | # Micrate::Cli.drop_database 25 | # expect(Micrate::DB).to have_received(:connect) 26 | # end 27 | # end 28 | # end 29 | 30 | # describe "#create_database" do 31 | # context "sqlite3" do 32 | # it "doesn't call connect" do 33 | # Micrate::DB.connection_url = "sqlite3:myfile" 34 | # Micrate::Cli.create_database 35 | # expect(Micrate::DB).not_to have_received(:connect) 36 | # end 37 | # end 38 | 39 | # context "postgres" do 40 | # it "calls connect" do 41 | # Micrate::DB.connection_url = "postgres://user:pswd@host:5432/database" 42 | # Micrate::Cli.create_database 43 | # expect(Micrate::DB).to have_received(:connect) 44 | # end 45 | # end 46 | # end 47 | end 48 | -------------------------------------------------------------------------------- /spec/micrate_spec.cr: -------------------------------------------------------------------------------- 1 | require "./spec_helper" 2 | 3 | Spectator.describe Micrate do 4 | describe "dbversion" do 5 | it "returns 0 if table is empty" do 6 | rows = [] of {Int64, Bool} 7 | Micrate.extract_dbversion(rows).should eq(0) 8 | end 9 | 10 | it "returns last applied migration" do 11 | # expect rows to be order by id asc 12 | rows = [ 13 | {20160101140000, true}, 14 | {20160101130000, true}, 15 | {20160101120000, true}, 16 | ] of {Int64, Bool} 17 | 18 | Micrate.extract_dbversion(rows).should eq(20160101140000) 19 | end 20 | 21 | it "ignores rolled back versions" do 22 | rows = [ 23 | {20160101140000, false}, 24 | {20160101140000, true}, 25 | {20160101120000, true}, 26 | ] of {Int64, Bool} 27 | 28 | Micrate.extract_dbversion(rows).should eq(20160101120000) 29 | end 30 | end 31 | 32 | describe "up" do 33 | context "going forward" do 34 | it "runs all migrations if starting from clean db" do 35 | plan = Micrate.migration_plan(sample_migrations, 0, 20160523142316, :forward) 36 | plan.should eq([20160523142308, 20160523142313, 20160523142316]) 37 | end 38 | 39 | it "skips already performed migrations" do 40 | plan = Micrate.migration_plan(sample_migrations, 20160523142308, 20160523142316, :forward) 41 | plan.should eq([20160523142313, 20160523142316]) 42 | end 43 | end 44 | 45 | context "going backwards" do 46 | it "skips already performed migrations" do 47 | plan = Micrate.migration_plan(sample_migrations, 20160523142316, 20160523142308, :backwards) 48 | plan.should eq([20160523142316, 20160523142313]) 49 | end 50 | end 51 | 52 | describe "detecting unordered migrations" do 53 | it "fails if there are unapplied migrations with older timestamp than current version" do 54 | migrations = { 55 | 20160523142308 => false, 56 | 20160523142313 => true, 57 | 20160523142316 => false, 58 | } 59 | 60 | expect_raises(Micrate::UnorderedMigrationsException) do 61 | Micrate.migration_plan(migrations, 20160523142313, 20160523142316, :forward) 62 | end 63 | end 64 | end 65 | end 66 | end 67 | 68 | def sample_migrations 69 | { 70 | 20160523142308 => true, 71 | 20160523142313 => true, 72 | 20160523142316 => true, 73 | } 74 | end 75 | -------------------------------------------------------------------------------- /spec/migration_spec.cr: -------------------------------------------------------------------------------- 1 | require "./spec_helper" 2 | 3 | Spectator.describe Micrate do 4 | describe "splitting in statements" do 5 | it "split simple statements" do 6 | migration = Micrate::Migration.new(20160101120000, "foo.sql", "\ 7 | -- +micrate Up 8 | CREATE TABLE foo(id INT PRIMARY KEY, name VARCHAR NOT NULL); 9 | 10 | -- +micrate Down 11 | DROP TABLE foo;") 12 | 13 | statements(migration, :forward).should eq([ 14 | "-- +micrate Up\nCREATE TABLE foo(id INT PRIMARY KEY, name VARCHAR NOT NULL);", 15 | ]) 16 | 17 | statements(migration, :backwards).should eq([ 18 | "-- +micrate Down\nDROP TABLE foo;", 19 | ]) 20 | end 21 | 22 | it "splits mixed Up and Down statements" do 23 | migration = Micrate::Migration.new(20160101120000, "foo.sql", "\ 24 | -- +micrate Up 25 | CREATE TABLE foo(id INT PRIMARY KEY, name VARCHAR NOT NULL); 26 | 27 | -- +micrate Down 28 | DROP TABLE foo; 29 | 30 | -- +micrate Up 31 | CREATE TABLE bar(id INT PRIMARY KEY); 32 | 33 | -- +micrate Down 34 | DROP TABLE bar;") 35 | 36 | statements(migration, :forward).should eq([ 37 | "-- +micrate Up\nCREATE TABLE foo(id INT PRIMARY KEY, name VARCHAR NOT NULL);", 38 | "-- +micrate Up\nCREATE TABLE bar(id INT PRIMARY KEY);", 39 | ]) 40 | 41 | statements(migration, :backwards).should eq([ 42 | "-- +micrate Down\nDROP TABLE foo;", 43 | "-- +micrate Down\nDROP TABLE bar;", 44 | ]) 45 | end 46 | 47 | # Some complex PL/psql may have semicolons within them 48 | # To understand these we need StatementBegin/StatementEnd hints 49 | it "splits complex statements with user hints" do 50 | migration = Micrate::Migration.new(20160101120000, "foo.sql", "\ 51 | -- +micrate Up 52 | -- +micrate StatementBegin 53 | CREATE OR REPLACE FUNCTION histories_partition_creation( DATE, DATE ) 54 | returns void AS $$ 55 | DECLARE 56 | create_query text; 57 | BEGIN 58 | FOR create_query IN SELECT 59 | 'CREATE TABLE IF NOT EXISTS histories_' 60 | || TO_CHAR( d, 'YYYY_MM' ) 61 | || ' ( CHECK( created_at >= timestamp ''' 62 | || TO_CHAR( d, 'YYYY-MM-DD 00:00:00' ) 63 | || ''' AND created_at < timestamp ''' 64 | || TO_CHAR( d + INTERVAL '1 month', 'YYYY-MM-DD 00:00:00' ) 65 | || ''' ) ) inherits ( histories );' 66 | FROM generate_series( $1, $2, '1 month' ) AS d 67 | LOOP 68 | EXECUTE create_query; 69 | END LOOP; -- LOOP END 70 | END; -- FUNCTION END 71 | $$ 72 | language plpgsql; 73 | -- +micrate StatementEnd") 74 | 75 | ret = statements(migration, :forward) 76 | 77 | ret.size.should eq(1) 78 | ret[0].should eq(migration.source) 79 | end 80 | 81 | it "allows up and down sections with complex scripts" do 82 | migration = Micrate::Migration.new(20160101120000, "foo.sql", "\ 83 | -- +micrate Up 84 | -- +micrate StatementBegin 85 | foo; 86 | bar; 87 | -- +micrate StatementEnd 88 | 89 | -- +micrate Down 90 | baz;") 91 | 92 | statements(migration, :forward).should eq([ 93 | "-- +micrate Up\n-- +micrate StatementBegin\nfoo;\nbar;\n-- +micrate StatementEnd", 94 | ]) 95 | 96 | statements(migration, :backwards).should eq([ 97 | "-- +micrate Down\nbaz;", 98 | ]) 99 | end 100 | end 101 | end 102 | 103 | def statements(migration, direction) 104 | migration.statements(direction).map { |stmt| stmt.strip } 105 | end 106 | -------------------------------------------------------------------------------- /spec/spec_helper.cr: -------------------------------------------------------------------------------- 1 | require "log" 2 | require "spectator" 3 | require "spectator/should" 4 | require "../src/micrate" 5 | 6 | Log.setup(:error) 7 | -------------------------------------------------------------------------------- /src/micrate-bin.cr: -------------------------------------------------------------------------------- 1 | require "log" 2 | require "pg" 3 | require "mysql" 4 | require "sqlite3" 5 | 6 | require "./micrate" 7 | 8 | Log.define_formatter Micrate::CliFormat, "#{message}" \ 9 | "#{data(before: " -- ")}#{context(before: " -- ")}#{exception}" 10 | Log.setup(:info, Log::IOBackend.new(formatter: Micrate::CliFormat)) 11 | 12 | Micrate::Cli.run 13 | -------------------------------------------------------------------------------- /src/micrate.cr: -------------------------------------------------------------------------------- 1 | require "log" 2 | 3 | require "./micrate/*" 4 | 5 | module Micrate 6 | Log = ::Log.for(self) 7 | 8 | def self.db_dir 9 | "db" 10 | end 11 | 12 | def self.migrations_dir 13 | File.join(db_dir, "migrations") 14 | end 15 | 16 | def self.dbversion(db) 17 | begin 18 | rows = DB.get_versions_last_first_order(db) 19 | return extract_dbversion(rows) 20 | rescue Exception 21 | DB.create_migrations_table(db) 22 | return 0 23 | end 24 | end 25 | 26 | def self.up(db) 27 | all_migrations = migrations_by_version 28 | 29 | if all_migrations.size == 0 30 | Log.warn { "No migrations found!" } 31 | return 32 | end 33 | 34 | current = dbversion(db) 35 | target = all_migrations.keys.sort.last 36 | migrate(all_migrations, current, target, db) 37 | end 38 | 39 | def self.down(db) 40 | all_migrations = migrations_by_version 41 | 42 | current = dbversion(db) 43 | target = previous_version(current, all_migrations.keys) 44 | migrate(all_migrations, current, target, db) 45 | end 46 | 47 | def self.redo(db) 48 | all_migrations = migrations_by_version 49 | 50 | current = dbversion(db) 51 | previous = previous_version(current, all_migrations.keys) 52 | 53 | if migrate(all_migrations, current, previous, db) == :success 54 | migrate(all_migrations, previous, current, db) 55 | end 56 | end 57 | 58 | def self.migration_status(db) : Hash(Migration, Time?) 59 | # ensure that migration table exists 60 | dbversion(db) 61 | migration_status(migrations_by_version.values, db) 62 | end 63 | 64 | def self.migration_status(migrations : Array(Migration), db) : Hash(Migration, Time?) 65 | ({} of Migration => Time?).tap do |ret| 66 | migrations.each do |m| 67 | ret[m] = DB.get_migration_status(m, db) 68 | end 69 | end 70 | end 71 | 72 | def self.create(name, dir, time) 73 | timestamp = time.to_s("%Y%m%d%H%M%S") 74 | filename = File.join(dir, "#{timestamp}_#{name}.sql") 75 | 76 | migration_template = "\ 77 | -- +micrate Up 78 | -- SQL in section 'Up' is executed when this migration is applied 79 | 80 | 81 | -- +micrate Down 82 | -- SQL section 'Down' is executed when this migration is rolled back 83 | " 84 | 85 | Dir.mkdir_p dir 86 | File.write(filename, migration_template) 87 | 88 | return filename 89 | end 90 | 91 | def self.connection_url=(connection_url) 92 | DB.connection_url = connection_url 93 | end 94 | 95 | # --------------------------------- 96 | # Private 97 | # --------------------------------- 98 | 99 | private def self.migrate(all_migrations : Hash(Int, Migration), current : Int, target : Int, db) 100 | direction = current < target ? :forward : :backwards 101 | 102 | status = migration_status(all_migrations.values, db) 103 | plan = migration_plan(status, current, target, direction) 104 | 105 | if plan.empty? 106 | Log.info { "No migrations to run. current version: #{current}" } 107 | return :nop 108 | end 109 | 110 | Log.info { "Migrating db, current version: #{current}, target: #{target}" } 111 | 112 | plan.each do |version| 113 | migration = all_migrations[version] 114 | 115 | # Wrap migration in a transaction 116 | db.transaction do |tx| 117 | migration.statements(direction).each do |stmt| 118 | tx.connection.exec(stmt) 119 | end 120 | 121 | DB.record_migration(migration, direction, tx.connection) 122 | 123 | tx.commit 124 | Log.info { "OK #{migration.name}" } 125 | rescue e : Exception 126 | tx.rollback 127 | Log.error(exception: e) { "An error occurred executing migration #{migration.version}." } 128 | return :error 129 | end 130 | end 131 | :success 132 | end 133 | 134 | private def self.verify_unordered_migrations(current, status : Hash(Int, Bool)) 135 | migrations = status.select { |version, is_applied| !is_applied && version < current } 136 | .keys 137 | 138 | if !migrations.empty? 139 | raise UnorderedMigrationsException.new(migrations) 140 | end 141 | end 142 | 143 | private def self.previous_version(current, all_versions) 144 | all_previous = all_versions.select { |version| version < current } 145 | if !all_previous.empty? 146 | return all_previous.max 147 | end 148 | 149 | if all_versions.includes? current 150 | # the given version is (likely) valid but we didn't find 151 | # anything before it. 152 | # return value must reflect that no migrations have been applied. 153 | return 0 154 | else 155 | raise "no previous version found" 156 | end 157 | end 158 | 159 | private def self.migrations_by_version 160 | Dir.entries(migrations_dir) 161 | .select { |name| File.file? File.join(migrations_dir, name) } 162 | .select { |name| /^\d+.+\.sql$/ =~ name } 163 | .map { |name| Migration.from_file(name) } 164 | .index_by { |migration| migration.version } 165 | end 166 | 167 | def self.migration_plan(status : Hash(Migration, Time?), current : Int, target : Int, direction) 168 | status = ({} of Int64 => Bool).tap do |h| 169 | status.each { |migration, migrated_at| h[migration.version] = !migrated_at.nil? } 170 | end 171 | 172 | migration_plan(status, current, target, direction) 173 | end 174 | 175 | def self.migration_plan(all_versions : Hash(Int, Bool), current : Int, target : Int, direction) 176 | verify_unordered_migrations(current, all_versions) 177 | 178 | if direction == :forward 179 | all_versions.keys 180 | .sort 181 | .select { |v| v > current && v <= target } 182 | else 183 | all_versions.keys 184 | .sort 185 | .reverse 186 | .select { |v| v <= current && v > target } 187 | end 188 | end 189 | 190 | # The most recent record for each migration specifies 191 | # whether it has been applied or rolled back. 192 | # The first version we find that has been applied is the current version. 193 | def self.extract_dbversion(rows) 194 | to_skip = [] of Int64 195 | 196 | rows.each do |r| 197 | version, is_applied = r 198 | next if to_skip.includes? version 199 | 200 | if is_applied 201 | return version 202 | else 203 | to_skip.push version 204 | end 205 | end 206 | 207 | return 0 208 | end 209 | 210 | class UnorderedMigrationsException < Exception 211 | getter :versions 212 | 213 | def initialize(@versions : Array(Int64)) 214 | super() 215 | end 216 | end 217 | end 218 | -------------------------------------------------------------------------------- /src/micrate/cli.cr: -------------------------------------------------------------------------------- 1 | require "log" 2 | 3 | module Micrate 4 | module Cli 5 | Log = ::Log.for(self) 6 | 7 | def self.drop_database 8 | url = Micrate::DB.connection_url.to_s 9 | if url.starts_with? "sqlite3:" 10 | path = url.gsub("sqlite3:", "") 11 | File.delete(path) 12 | Log.info { "Deleted file #{path}" } 13 | else 14 | name = set_database_to_schema url 15 | Micrate::DB.connect do |db| 16 | db.exec "DROP DATABASE IF EXISTS #{name};" 17 | end 18 | Log.info { "Dropped database #{name}" } 19 | end 20 | end 21 | 22 | def self.create_database 23 | url = Micrate::DB.connection_url.to_s 24 | if url.starts_with? "sqlite3:" 25 | Log.info { "For sqlite3, the database will be created during the first migration." } 26 | else 27 | name = set_database_to_schema url 28 | Micrate::DB.connect do |db| 29 | db.exec "CREATE DATABASE #{name};" 30 | end 31 | Log.info { "Created database #{name}" } 32 | end 33 | end 34 | 35 | def self.set_database_to_schema(url) 36 | uri = URI.parse(url) 37 | if path = uri.path 38 | Micrate::DB.connection_url = url.gsub(path, "/#{uri.scheme}") 39 | path.gsub("/", "") 40 | else 41 | Log.error { "Could not determine database name" } 42 | end 43 | end 44 | 45 | def self.run_up 46 | Micrate::DB.connect do |db| 47 | Micrate.up(db) 48 | end 49 | end 50 | 51 | def self.run_down 52 | Micrate::DB.connect do |db| 53 | Micrate.down(db) 54 | end 55 | end 56 | 57 | def self.run_redo 58 | Micrate::DB.connect do |db| 59 | Micrate.redo(db) 60 | end 61 | end 62 | 63 | def self.run_status 64 | Micrate::DB.connect do |db| 65 | Log.info { "Applied At Migration" } 66 | Log.info { "=======================================" } 67 | Micrate.migration_status(db).each do |migration, migrated_at| 68 | ts = migrated_at.nil? ? "Pending" : migrated_at.to_s 69 | Log.info { "%-24s -- %s\n" % [ts, migration.name] } 70 | end 71 | end 72 | end 73 | 74 | def self.run_scaffold 75 | if ARGV.size < 1 76 | raise "Migration name required" 77 | end 78 | 79 | migration_file = Micrate.create(ARGV.shift, Micrate.migrations_dir, Time.local) 80 | Log.info { "Created #{migration_file}" } 81 | end 82 | 83 | def self.run_dbversion 84 | Micrate::DB.connect do |db| 85 | begin 86 | Log.info { Micrate.dbversion(db) } 87 | rescue 88 | raise "Could not read dbversion. Please make sure the database exists and verify the connection URL." 89 | end 90 | end 91 | end 92 | 93 | def self.report_unordered_migrations(conflicting) 94 | Log.info { "The following migrations haven't been applied but have a timestamp older then the current version:" } 95 | conflicting.each do |version| 96 | Log.info { " #{Migration.from_version(version).name}" } 97 | end 98 | Log.info { " 99 | Micrate will not run these migrations because they may have been written with an older database model in mind. 100 | You should probably check if they need to be updated and rename them so they are considered a newer version." } 101 | end 102 | 103 | def self.print_help 104 | Log.info { "micrate is a database migration management system for Crystal projects, *heavily* inspired by Goose (https://bitbucket.org/liamstask/goose/). 105 | 106 | Usage: 107 | set DATABASE_URL environment variable i.e. export DATABASE_URL=postgres://user:pswd@host:port/database 108 | micrate [options] [subcommand options] 109 | 110 | Commands: 111 | create Create the database (permissions required) 112 | drop Drop the database (permissions required) 113 | up Migrate the DB to the most recent version available 114 | down Roll back the version by 1 115 | redo Re-run the latest migration 116 | status Dump the migration status for the current DB 117 | scaffold Create the scaffolding for a new migration 118 | dbversion Print the current version of the database" } 119 | end 120 | 121 | def self.run 122 | if ARGV.empty? 123 | print_help 124 | return 125 | end 126 | 127 | begin 128 | case ARGV.shift 129 | when "create" 130 | create_database 131 | when "drop" 132 | drop_database 133 | when "up" 134 | run_up 135 | when "down" 136 | run_down 137 | when "redo" 138 | run_redo 139 | when "status" 140 | run_status 141 | when "scaffold" 142 | run_scaffold 143 | when "dbversion" 144 | run_dbversion 145 | else 146 | print_help 147 | end 148 | rescue e : UnorderedMigrationsException 149 | report_unordered_migrations(e.versions) 150 | exit 1 151 | rescue e : Exception 152 | Log.error(exception: e) { "Micrate failed!" } 153 | exit 1 154 | end 155 | end 156 | end 157 | end 158 | -------------------------------------------------------------------------------- /src/micrate/db.cr: -------------------------------------------------------------------------------- 1 | require "db" 2 | require "./db/*" 3 | 4 | module Micrate 5 | module DB 6 | class_getter connection_url : String? { ENV["DATABASE_URL"]? } 7 | 8 | def self.connection_url=(connection_url) 9 | @@dialect = nil 10 | @@connection_url = connection_url 11 | end 12 | 13 | def self.connect 14 | validate_connection_url 15 | ::DB.connect(self.connection_url.not_nil!) 16 | end 17 | 18 | def self.connect(&block) 19 | validate_connection_url 20 | ::DB.open self.connection_url.not_nil! do |db| 21 | yield db 22 | end 23 | end 24 | 25 | def self.get_versions_last_first_order(db) 26 | db.query_all "SELECT version_id, is_applied from micrate_db_version ORDER BY id DESC", as: {Int64, Bool} 27 | end 28 | 29 | def self.create_migrations_table(db) 30 | dialect.query_create_migrations_table(db) 31 | end 32 | 33 | def self.record_migration(migration, direction, db) 34 | is_applied = direction == :forward 35 | dialect.query_record_migration(migration, is_applied, db) 36 | end 37 | 38 | def self.exec(statement, db) 39 | db.exec(statement) 40 | end 41 | 42 | def self.get_migration_status(migration, db) : Time? 43 | rows = dialect.query_migration_status(migration, db) 44 | 45 | if !rows.empty? && rows[0][1] 46 | rows[0][0] 47 | else 48 | nil 49 | end 50 | end 51 | 52 | private def self.dialect 53 | validate_connection_url 54 | @@dialect ||= Dialect.from_connection_url(self.connection_url.not_nil!) 55 | end 56 | 57 | private def self.validate_connection_url 58 | if !self.connection_url 59 | raise "No database connection URL is configured. Please set the DATABASE_URL environment variable." 60 | end 61 | end 62 | end 63 | end 64 | -------------------------------------------------------------------------------- /src/micrate/db/dialect.cr: -------------------------------------------------------------------------------- 1 | module Micrate::DB 2 | abstract class Dialect 3 | abstract def query_create_migrations_table(db) 4 | abstract def query_migration_status(migration, db) 5 | abstract def query_record_migration(migration, is_applied, db) 6 | 7 | def self.from_connection_url(connection_url : String) 8 | uri = URI.parse(connection_url) 9 | case uri.scheme 10 | when "postgresql", "postgres" 11 | Postgres.new 12 | when "mysql" 13 | Mysql.new 14 | when "sqlite3" 15 | Sqlite3.new 16 | else 17 | raise "Could not infer SQL dialect from connection url #{connection_url}" 18 | end 19 | end 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /src/micrate/db/mysql.cr: -------------------------------------------------------------------------------- 1 | module Micrate::DB 2 | class Mysql < Dialect 3 | def query_create_migrations_table(db) 4 | db.exec("CREATE TABLE micrate_db_version ( 5 | id serial NOT NULL, 6 | version_id bigint NOT NULL, 7 | is_applied boolean NOT NULL, 8 | tstamp timestamp NULL default now(), 9 | PRIMARY KEY(id) 10 | );") 11 | end 12 | 13 | def query_migration_status(migration, db) 14 | db.query_all "SELECT tstamp, is_applied FROM micrate_db_version WHERE version_id=? ORDER BY tstamp DESC LIMIT 1", migration.version, as: {Time, Bool} 15 | end 16 | 17 | def query_record_migration(migration, is_applied, db) 18 | db.exec("INSERT INTO micrate_db_version (version_id, is_applied) VALUES (?, ?);", migration.version, is_applied) 19 | end 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /src/micrate/db/postgres.cr: -------------------------------------------------------------------------------- 1 | module Micrate::DB 2 | class Postgres < Dialect 3 | def query_create_migrations_table(db) 4 | db.exec("CREATE TABLE micrate_db_version ( 5 | id serial NOT NULL, 6 | version_id bigint NOT NULL, 7 | is_applied boolean NOT NULL, 8 | tstamp timestamp NULL default now(), 9 | PRIMARY KEY(id) 10 | );") 11 | end 12 | 13 | def query_migration_status(migration, db) 14 | db.query_all "SELECT tstamp, is_applied FROM micrate_db_version WHERE version_id=$1 ORDER BY tstamp DESC LIMIT 1", migration.version, as: {Time, Bool} 15 | end 16 | 17 | def query_record_migration(migration, is_applied, db) 18 | db.exec("INSERT INTO micrate_db_version (version_id, is_applied) VALUES ($1, $2);", migration.version, is_applied) 19 | end 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /src/micrate/db/sqlite3.cr: -------------------------------------------------------------------------------- 1 | module Micrate::DB 2 | class Sqlite3 < Dialect 3 | def query_create_migrations_table(db) 4 | # The current sqlite drive implementation does not store timestamps in the same 5 | # format as the ones autogenerated by sqlite. 6 | # 7 | # As a workaround, we create timestamps locally so that the driver decides timestamp 8 | # formats when writing and reading. 9 | db.exec("CREATE TABLE micrate_db_version ( 10 | id INTEGER PRIMARY KEY AUTOINCREMENT, 11 | version_id INTEGER NOT NULL, 12 | is_applied INTEGER NOT NULL, 13 | tstamp TIMESTAMP 14 | );") 15 | end 16 | 17 | def query_migration_status(migration, db) 18 | db.query_all "SELECT tstamp, is_applied FROM micrate_db_version WHERE version_id=? ORDER BY tstamp DESC LIMIT 1", migration.version, as: {Time, Bool} 19 | end 20 | 21 | def query_record_migration(migration, is_applied, db) 22 | db.exec("INSERT INTO micrate_db_version (version_id, is_applied, tstamp) VALUES (?, ?, ?);", migration.version, is_applied, Time.local) 23 | end 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /src/micrate/migration.cr: -------------------------------------------------------------------------------- 1 | module Micrate 2 | class Migration 3 | SQL_CMD_PREFIX = "-- +micrate " 4 | 5 | getter version 6 | getter name 7 | getter source 8 | 9 | def initialize(@version : Int64, @name : String, @source : String) 10 | end 11 | 12 | # Algorithm ported from Goose 13 | # 14 | # Complex statements cannot be resolved by just splitting the script by semicolons. 15 | # In this cases we allow using StatementBegin and StatementEnd directives as hints. 16 | def statements(direction) 17 | statements = [] of String 18 | 19 | # track the count of each section 20 | # so we can diagnose scripts with no annotations 21 | up_sections = 0 22 | down_sections = 0 23 | 24 | buffer = Micrate::StatementBuilder.new 25 | 26 | statement_ended = false 27 | ignore_semicolons = false 28 | direction_is_active = false 29 | 30 | source.split("\n").each do |line| 31 | if line.starts_with? SQL_CMD_PREFIX 32 | cmd = line[SQL_CMD_PREFIX.size..-1].strip 33 | case cmd 34 | when "Up" 35 | direction_is_active = direction == :forward 36 | up_sections += 1 37 | when "Down" 38 | direction_is_active = direction == :backwards 39 | down_sections += 1 40 | when "StatementBegin" 41 | if direction_is_active 42 | ignore_semicolons = true 43 | end 44 | when "StatementEnd" 45 | if direction_is_active 46 | statement_ended = ignore_semicolons == true 47 | ignore_semicolons = false 48 | end 49 | else 50 | # TODO? invalid command 51 | end 52 | end 53 | 54 | next unless direction_is_active 55 | 56 | buffer.write(line + "\n") 57 | 58 | if (!ignore_semicolons && ends_with_semicolon(line)) || statement_ended 59 | statement_ended = false 60 | statements.push buffer.to_s 61 | buffer.reset 62 | end 63 | end 64 | 65 | statements 66 | end 67 | 68 | def ends_with_semicolon(s) 69 | s.split("--")[0].strip.ends_with? ";" 70 | end 71 | 72 | def self.from_file(file_name) 73 | full_path = File.join(Micrate.migrations_dir, file_name) 74 | version = file_name.split("_")[0].to_i64 75 | new(version, file_name, File.read(full_path)) 76 | end 77 | 78 | def self.from_version(version) 79 | file_name = Dir.entries(Micrate.migrations_dir) 80 | .find { |name| name.starts_with? version.to_s } 81 | .not_nil! 82 | self.from_file(file_name) 83 | end 84 | end 85 | end 86 | -------------------------------------------------------------------------------- /src/micrate/statement_builder.cr: -------------------------------------------------------------------------------- 1 | module Micrate 2 | class StatementBuilder 3 | @buffer : String::Builder 4 | 5 | def initialize 6 | @buffer = String::Builder.new 7 | end 8 | 9 | def write(s) 10 | @buffer.write(s.to_slice) 11 | end 12 | 13 | def reset 14 | @buffer = String::Builder.new 15 | end 16 | 17 | def to_s 18 | @buffer.to_s 19 | end 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /src/micrate/version.cr: -------------------------------------------------------------------------------- 1 | module Micrate 2 | VERSION = "0.10.0" 3 | end 4 | --------------------------------------------------------------------------------