├── .gitignore ├── .travis.yml ├── CONTRIBUTING.md ├── FAQ.md ├── LICENSE ├── MIGRATIONS.md ├── Makefile ├── README.md ├── cli ├── README.md ├── build_aws-s3.go ├── build_cassandra.go ├── build_clickhouse.go ├── build_cockroachdb.go ├── build_github.go ├── build_go-bindata.go ├── build_google-cloud-storage.go ├── build_mysql.go ├── build_postgres.go ├── build_ql.go ├── build_redshift.go ├── build_spanner.go ├── build_sqlite3.go ├── commands.go ├── examples │ └── Dockerfile ├── log.go ├── main.go └── version.go ├── database ├── cassandra │ ├── README.md │ ├── cassandra.go │ └── cassandra_test.go ├── clickhouse │ ├── README.md │ ├── clickhouse.go │ └── examples │ │ └── migrations │ │ ├── 001_init.down.sql │ │ ├── 001_init.up.sql │ │ ├── 002_create_table.down.sql │ │ └── 002_create_table.up.sql ├── cockroachdb │ ├── README.md │ ├── cockroachdb.go │ ├── cockroachdb_test.go │ └── examples │ │ └── migrations │ │ ├── 1085649617_create_users_table.down.sql │ │ ├── 1085649617_create_users_table.up.sql │ │ ├── 1185749658_add_city_to_users.down.sql │ │ ├── 1185749658_add_city_to_users.up.sql │ │ ├── 1285849751_add_index_on_user_emails.down.sql │ │ ├── 1285849751_add_index_on_user_emails.up.sql │ │ ├── 1385949617_create_books_table.down.sql │ │ ├── 1385949617_create_books_table.up.sql │ │ ├── 1485949617_create_movies_table.down.sql │ │ ├── 1485949617_create_movies_table.up.sql │ │ ├── 1585849751_just_a_comment.up.sql │ │ ├── 1685849751_another_comment.up.sql │ │ ├── 1785849751_another_comment.up.sql │ │ └── 1885849751_another_comment.up.sql ├── crate │ └── README.md ├── driver.go ├── driver_test.go ├── error.go ├── mongodb │ └── README.md ├── mysql │ ├── README.md │ ├── mysql.go │ └── mysql_test.go ├── neo4j │ └── README.md ├── postgres │ ├── README.md │ ├── examples │ │ └── migrations │ │ │ ├── 1085649617_create_users_table.down.sql │ │ │ ├── 1085649617_create_users_table.up.sql │ │ │ ├── 1185749658_add_city_to_users.down.sql │ │ │ ├── 1185749658_add_city_to_users.up.sql │ │ │ ├── 1285849751_add_index_on_user_emails.down.sql │ │ │ ├── 1285849751_add_index_on_user_emails.up.sql │ │ │ ├── 1385949617_create_books_table.down.sql │ │ │ ├── 1385949617_create_books_table.up.sql │ │ │ ├── 1485949617_create_movies_table.down.sql │ │ │ ├── 1485949617_create_movies_table.up.sql │ │ │ ├── 1585849751_just_a_comment.up.sql │ │ │ ├── 1685849751_another_comment.up.sql │ │ │ ├── 1785849751_another_comment.up.sql │ │ │ └── 1885849751_another_comment.up.sql │ ├── postgres.go │ └── postgres_test.go ├── ql │ ├── README.md │ ├── migration │ │ ├── 33_create_table.down.sql │ │ ├── 33_create_table.up.sql │ │ ├── 44_alter_table.down.sql │ │ └── 44_alter_table.up.sql │ ├── ql.go │ └── ql_test.go ├── redshift │ ├── README.md │ └── redshift.go ├── shell │ └── README.md ├── spanner │ ├── README.md │ ├── examples │ │ └── migrations │ │ │ ├── 1481574547_create_users_table.down.sql │ │ │ ├── 1481574547_create_users_table.up.sql │ │ │ ├── 1496539702_add_city_to_users.down.sql │ │ │ ├── 1496539702_add_city_to_users.up.sql │ │ │ ├── 1496601752_add_index_on_user_emails.down.sql │ │ │ ├── 1496601752_add_index_on_user_emails.up.sql │ │ │ ├── 1496602638_create_books_table.down.sql │ │ │ └── 1496602638_create_books_table.up.sql │ ├── spanner.go │ └── spanner_test.go ├── sqlite3 │ ├── README.md │ ├── migration │ │ ├── 33_create_table.down.sql │ │ ├── 33_create_table.up.sql │ │ ├── 44_alter_table.down.sql │ │ └── 44_alter_table.up.sql │ ├── sqlite3.go │ └── sqlite3_test.go ├── stub │ ├── stub.go │ └── stub_test.go ├── testing │ └── testing.go ├── util.go └── util_test.go ├── log.go ├── migrate.go ├── migrate_test.go ├── migration.go ├── migration_test.go ├── source ├── aws-s3 │ ├── README.md │ ├── s3.go │ └── s3_test.go ├── driver.go ├── driver_test.go ├── file │ ├── README.md │ ├── file.go │ └── file_test.go ├── github │ ├── .gitignore │ ├── README.md │ ├── examples │ │ └── migrations │ │ │ ├── 1085649617_create_users_table.down.sql │ │ │ ├── 1085649617_create_users_table.up.sql │ │ │ ├── 1185749658_add_city_to_users.down.sql │ │ │ ├── 1185749658_add_city_to_users.up.sql │ │ │ ├── 1285849751_add_index_on_user_emails.down.sql │ │ │ ├── 1285849751_add_index_on_user_emails.up.sql │ │ │ ├── 1385949617_create_books_table.down.sql │ │ │ ├── 1385949617_create_books_table.up.sql │ │ │ ├── 1485949617_create_movies_table.down.sql │ │ │ ├── 1485949617_create_movies_table.up.sql │ │ │ ├── 1585849751_just_a_comment.up.sql │ │ │ ├── 1685849751_another_comment.up.sql │ │ │ ├── 1785849751_another_comment.up.sql │ │ │ └── 1885849751_another_comment.up.sql │ ├── github.go │ └── github_test.go ├── go-bindata │ ├── README.md │ ├── examples │ │ └── migrations │ │ │ └── bindata.go │ ├── go-bindata.go │ ├── go-bindata_test.go │ └── testdata │ │ └── bindata.go ├── google-cloud-storage │ ├── README.md │ ├── storage.go │ └── storage_test.go ├── migration.go ├── migration_test.go ├── parse.go ├── parse_test.go ├── stub │ ├── stub.go │ └── stub_test.go └── testing │ └── testing.go ├── testing ├── docker.go ├── testing.go └── testing_test.go ├── util.go └── util_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | cli/build 3 | cli/cli 4 | cli/migrate 5 | .coverage 6 | .godoc.pid 7 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | sudo: required 3 | 4 | go: 5 | - 1.7 6 | - 1.8 7 | - 1.9 8 | 9 | env: 10 | - MIGRATE_TEST_CONTAINER_BOOT_DELAY=10 11 | 12 | # TODO: https://docs.docker.com/engine/installation/linux/ubuntu/ 13 | # pre-provision with travis docker setup and pin down docker version in install step 14 | services: 15 | - docker 16 | 17 | install: 18 | - make deps 19 | - (cd $GOPATH/src/github.com/docker/docker && git fetch --all --tags --prune && git checkout v17.05.0-ce) 20 | - sudo apt-get update && sudo apt-get install docker-ce=17.05.0* 21 | - go get github.com/mattn/goveralls 22 | 23 | script: 24 | - make test 25 | 26 | after_success: 27 | - goveralls -service=travis-ci -coverprofile .coverage/combined.txt 28 | - make list-external-deps > dependency_tree.txt && cat dependency_tree.txt 29 | 30 | before_deploy: 31 | - make build-cli 32 | - gem install --no-ri --no-rdoc fpm 33 | - fpm -s dir -t deb -n migrate -v "$(git describe --tags 2>/dev/null | cut -c 2-)" --license MIT -m matthias.kadenbach@gmail.com --url https://github.com/mattes/migrate --description='Database migrations' -a amd64 -p migrate.$(git describe --tags 2>/dev/null | cut -c 2-).deb --deb-no-default-config-files -f -C cli/build migrate.linux-amd64=/usr/bin/migrate 34 | 35 | deploy: 36 | - provider: releases 37 | api_key: 38 | secure: EFow50BI448HVb/uQ1Kk2Kq0xzmwIYq3V67YyymXIuqSCodvXEsMiBPUoLrxEknpPEIc67LEQTNdfHBgvyHk6oRINWAfie+7pr5tKrpOTF9ghyxoN1PlO8WKQCqwCvGMBCnc5ur5rvzp0bqfpV2rs5q9/nngy3kBuEvs12V7iho= 39 | skip_cleanup: true 40 | on: 41 | go: 1.8 42 | repo: mattes/migrate 43 | tags: true 44 | file: 45 | - cli/build/migrate.linux-amd64.tar.gz 46 | - cli/build/migrate.darwin-amd64.tar.gz 47 | - cli/build/migrate.windows-amd64.exe.tar.gz 48 | - cli/build/sha256sum.txt 49 | - dependency_tree.txt 50 | - provider: packagecloud 51 | repository: migrate 52 | username: mattes 53 | token: 54 | secure: RiHJ/+J9DvXUah/APYdWySWZ5uOOISYJ0wS7xddc7/BNStRVjzFzvJ9zmb67RkyZZrvGuVjPiL4T8mtDyCJCj47RmU/56wPdEHbar/FjsiUCgwvR19RlulkgbV4okBCePbwzMw6HNHRp14TzfQCPtnN4kef0lOI4gZJkImN7rtQ= 55 | dist: ubuntu/xenial 56 | package_glob: '*.deb' 57 | skip_cleanup: true 58 | on: 59 | go: 1.8 60 | repo: mattes/migrate 61 | tags: true 62 | 63 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Development, Testing and Contributing 2 | 3 | 1. Make sure you have a running Docker daemon 4 | (Install for [MacOS](https://docs.docker.com/docker-for-mac/)) 5 | 2. Fork this repo and `git clone` somewhere to `$GOPATH/src/github.com/%you%/migrate` 6 | 3. `make rewrite-import-paths` to update imports to your local fork 7 | 4. Confirm tests are working: `make test-short` 8 | 5. Write awesome code ... 9 | 6. `make test` to run all tests against all database versions 10 | 7. `make restore-import-paths` to restore import paths 11 | 8. Push code and open Pull Request 12 | 13 | Some more helpful commands: 14 | 15 | * You can specify which database/ source tests to run: 16 | `make test-short SOURCE='file go-bindata' DATABASE='postgres cassandra'` 17 | * After `make test`, run `make html-coverage` which opens a shiny test coverage overview. 18 | * Missing imports? `make deps` 19 | * `make build-cli` builds the CLI in directory `cli/build/`. 20 | * `make list-external-deps` lists all external dependencies for each package 21 | * `make docs && make open-docs` opens godoc in your browser, `make kill-docs` kills the godoc server. 22 | Repeatedly call `make docs` to refresh the server. 23 | -------------------------------------------------------------------------------- /FAQ.md: -------------------------------------------------------------------------------- 1 | # FAQ 2 | 3 | #### How is the code base structured? 4 | ``` 5 | / package migrate (the heart of everything) 6 | /cli the CLI wrapper 7 | /database database driver and sub directories have the actual driver implementations 8 | /source source driver and sub directories have the actual driver implementations 9 | ``` 10 | 11 | #### Why is there no `source/driver.go:Last()`? 12 | It's not needed. And unless the source has a "native" way to read a directory in reversed order, 13 | it might be expensive to do a full directory scan in order to get the last element. 14 | 15 | #### What is a NilMigration? NilVersion? 16 | NilMigration defines a migration without a body. NilVersion is defined as const -1. 17 | 18 | #### What is the difference between uint(version) and int(targetVersion)? 19 | version refers to an existing migration version coming from a source and therefor can never be negative. 20 | targetVersion can either be a version OR represent a NilVersion, which equals -1. 21 | 22 | #### What's the difference between Next/Previous and Up/Down? 23 | ``` 24 | 1_first_migration.up.extension next -> 2_second_migration.up.extension ... 25 | 1_first_migration.down.extension <- previous 2_second_migration.down.extension ... 26 | ``` 27 | 28 | #### Why two separate files (up and down) for a migration? 29 | It makes all of our lives easier. No new markup/syntax to learn for users 30 | and existing database utility tools continue to work as expected. 31 | 32 | #### How many migrations can migrate handle? 33 | Whatever the maximum positive signed integer value is for your platform. 34 | For 32bit it would be 2,147,483,647 migrations. Migrate only keeps references to 35 | the currently run and pre-fetched migrations in memory. Please note that some 36 | source drivers need to do build a full "directory" tree first, which puts some 37 | heat on the memory consumption. 38 | 39 | #### Are the table tests in migrate_test.go bloated? 40 | Yes and no. There are duplicate test cases for sure but they don't hurt here. In fact 41 | the tests are very visual now and might help new users understand expected behaviors quickly. 42 | Migrate from version x to y and y is the last migration? Just check out the test for 43 | that particular case and know what's going on instantly. 44 | 45 | #### What is Docker being used for? 46 | Only for testing. See [testing/docker.go](testing/docker.go) 47 | 48 | #### Why not just use docker-compose? 49 | It doesn't give us enough runtime control for testing. We want to be able to bring up containers fast 50 | and whenever we want, not just once at the beginning of all tests. 51 | 52 | #### Can I maintain my driver in my own repository? 53 | Yes, technically thats possible. We want to encourage you to contribute your driver to this respository though. 54 | The driver's functionality is dictated by migrate's interfaces. That means there should really 55 | just be one driver for a database/ source. We want to prevent a future where several drivers doing the exact same thing, 56 | just implemented a bit differently, co-exist somewhere on Github. If users have to do research first to find the 57 | "best" available driver for a database in order to get started, we would have failed as an open source community. 58 | 59 | #### Can I mix multiple sources during a batch of migrations? 60 | No. 61 | 62 | #### What does "dirty" database mean? 63 | Before a migration runs, each database sets a dirty flag. Execution stops if a migration fails and the dirty state persists, 64 | which prevents attempts to run more migrations on top of a failed migration. You need to manually fix the error 65 | and then "force" the expected version. 66 | 67 | 68 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Matthias Kadenbach 4 | 5 | https://github.com/mattes/migrate 6 | 7 | Permission is hereby granted, free of charge, to any person obtaining a copy 8 | of this software and associated documentation files (the "Software"), to deal 9 | in the Software without restriction, including without limitation the rights 10 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | copies of the Software, and to permit persons to whom the Software is 12 | furnished to do so, subject to the following conditions: 13 | 14 | The above copyright notice and this permission notice shall be included in 15 | all copies or substantial portions of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 23 | THE SOFTWARE. 24 | -------------------------------------------------------------------------------- /MIGRATIONS.md: -------------------------------------------------------------------------------- 1 | # Migrations 2 | 3 | ## Migration Filename Format 4 | 5 | A single logical migration is represented as two separate migration files, one 6 | to migrate "up" to the specified version from the previous version, and a second 7 | to migrate back "down" to the previous version. These migrations can be provided 8 | by any one of the supported [migration sources](./README.md#migration-sources). 9 | 10 | The ordering and direction of the migration files is determined by the filenames 11 | used for them. `migrate` expects the filenames of migrations to have the format: 12 | 13 | {version}_{title}.up.{extension} 14 | {version}_{title}.down.{extension} 15 | 16 | The `title` of each migration is unused, and is only for readability. Similarly, 17 | the `extension` of the migration files is not checked by the library, and should 18 | be an appropriate format for the database in use (`.sql` for SQL variants, for 19 | instance). 20 | 21 | Versions of migrations may be represented as any 64 bit unsigned integer. 22 | All migrations are applied upward in order of increasing version number, and 23 | downward by decreasing version number. 24 | 25 | Common versioning schemes include incrementing integers: 26 | 27 | 1_initialize_schema.down.sql 28 | 1_initialize_schema.up.sql 29 | 2_add_table.down.sql 30 | 2_add_table.up.sql 31 | ... 32 | 33 | Or timestamps at an appropriate resolution: 34 | 35 | 1500360784_initialize_schema.down.sql 36 | 1500360784_initialize_schema.up.sql 37 | 1500445949_add_table.down.sql 38 | 1500445949_add_table.up.sql 39 | ... 40 | 41 | But any scheme resulting in distinct, incrementing integers as versions is valid. 42 | 43 | It is suggested that the version number of corresponding `up` and `down` migration 44 | files be equivalent for clarity, but they are allowed to differ so long as the 45 | relative ordering of the migrations is preserved. 46 | 47 | The migration files are permitted to be empty, so in the event that a migration 48 | is a no-op or is irreversible, it is recommended to still include both migration 49 | files, and either leaving them empty or adding a comment as appropriate. 50 | 51 | ## Migration Content Format 52 | 53 | The format of the migration files themselves varies between database systems. 54 | Different databases have different semantics around schema changes and when and 55 | how they are allowed to occur (for instance, if schema changes can occur within 56 | a transaction). 57 | 58 | As such, the `migrate` library has little to no checking around the format of 59 | migration sources. The migration files are generally processed directly by the 60 | drivers as raw operations. 61 | 62 | ## Reversibility of Migrations 63 | 64 | Best practice for writing schema migration is that all migrations should be 65 | reversible. It should in theory be possible for run migrations down and back up 66 | through any and all versions with the state being fully cleaned and recreated 67 | by doing so. 68 | 69 | By adhering to this recommended practice, development and deployment of new code 70 | is cleaner and easier (cleaning database state for a new feature should be as 71 | easy as migrating down to a prior version, and back up to the latest). 72 | 73 | As opposed to some other migration libraries, `migrate` represents up and down 74 | migrations as separate files. This prevents any non-standard file syntax from 75 | being introduced which may result in unintended behavior or errors, depending 76 | on what database is processing the file. 77 | 78 | While it is technically possible for an up or down migration to exist on its own 79 | without an equivalently versioned counterpart, it is strongly recommended to 80 | always include a down migration which cleans up the state of the corresponding 81 | up migration. 82 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | SOURCE ?= file go-bindata github aws-s3 google-cloud-storage 2 | DATABASE ?= postgres mysql redshift cassandra sqlite3 spanner cockroachdb clickhouse 3 | VERSION ?= $(shell git describe --tags 2>/dev/null | cut -c 2-) 4 | TEST_FLAGS ?= 5 | REPO_OWNER ?= $(shell cd .. && basename "$$(pwd)") 6 | 7 | 8 | build-cli: clean 9 | -mkdir ./cli/build 10 | cd ./cli && CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -a -o build/migrate.linux-amd64 -ldflags='-X main.Version=$(VERSION)' -tags '$(DATABASE) $(SOURCE)' . 11 | cd ./cli && CGO_ENABLED=1 GOOS=darwin GOARCH=amd64 go build -a -o build/migrate.darwin-amd64 -ldflags='-X main.Version=$(VERSION)' -tags '$(DATABASE) $(SOURCE)' . 12 | cd ./cli && CGO_ENABLED=1 GOOS=windows GOARCH=amd64 go build -a -o build/migrate.windows-amd64.exe -ldflags='-X main.Version=$(VERSION)' -tags '$(DATABASE) $(SOURCE)' . 13 | cd ./cli/build && find . -name 'migrate*' | xargs -I{} tar czf {}.tar.gz {} 14 | cd ./cli/build && shasum -a 256 * > sha256sum.txt 15 | cat ./cli/build/sha256sum.txt 16 | 17 | 18 | clean: 19 | -rm -r ./cli/build 20 | 21 | 22 | test-short: 23 | make test-with-flags --ignore-errors TEST_FLAGS='-short' 24 | 25 | 26 | test: 27 | @-rm -r .coverage 28 | @mkdir .coverage 29 | make test-with-flags TEST_FLAGS='-v -race -covermode atomic -coverprofile .coverage/_$$(RAND).txt -bench=. -benchmem' 30 | @echo 'mode: atomic' > .coverage/combined.txt 31 | @cat .coverage/*.txt | grep -v 'mode: atomic' >> .coverage/combined.txt 32 | 33 | 34 | test-with-flags: 35 | @echo SOURCE: $(SOURCE) 36 | @echo DATABASE: $(DATABASE) 37 | 38 | @go test $(TEST_FLAGS) . 39 | @go test $(TEST_FLAGS) ./cli/... 40 | @go test $(TEST_FLAGS) ./testing/... 41 | 42 | @echo -n '$(SOURCE)' | tr -s ' ' '\n' | xargs -I{} go test $(TEST_FLAGS) ./source/{} 43 | @go test $(TEST_FLAGS) ./source/testing/... 44 | @go test $(TEST_FLAGS) ./source/stub/... 45 | 46 | @echo -n '$(DATABASE)' | tr -s ' ' '\n' | xargs -I{} go test $(TEST_FLAGS) ./database/{} 47 | @go test $(TEST_FLAGS) ./database/testing/... 48 | @go test $(TEST_FLAGS) ./database/stub/... 49 | 50 | 51 | kill-orphaned-docker-containers: 52 | docker rm -f $(shell docker ps -aq --filter label=migrate_test) 53 | 54 | 55 | html-coverage: 56 | go tool cover -html=.coverage/combined.txt 57 | 58 | 59 | deps: 60 | -go get -v -u ./... 61 | -go test -v -i ./... 62 | # TODO: why is this not being fetched with the command above? 63 | -go get -u github.com/fsouza/fake-gcs-server/fakestorage 64 | 65 | 66 | list-external-deps: 67 | $(call external_deps,'.') 68 | $(call external_deps,'./cli/...') 69 | $(call external_deps,'./testing/...') 70 | 71 | $(foreach v, $(SOURCE), $(call external_deps,'./source/$(v)/...')) 72 | $(call external_deps,'./source/testing/...') 73 | $(call external_deps,'./source/stub/...') 74 | 75 | $(foreach v, $(DATABASE), $(call external_deps,'./database/$(v)/...')) 76 | $(call external_deps,'./database/testing/...') 77 | $(call external_deps,'./database/stub/...') 78 | 79 | 80 | restore-import-paths: 81 | find . -name '*.go' -type f -execdir sed -i '' s%\"github.com/$(REPO_OWNER)/migrate%\"github.com/mattes/migrate%g '{}' \; 82 | 83 | 84 | rewrite-import-paths: 85 | find . -name '*.go' -type f -execdir sed -i '' s%\"github.com/mattes/migrate%\"github.com/$(REPO_OWNER)/migrate%g '{}' \; 86 | 87 | 88 | # example: fswatch -0 --exclude .godoc.pid --event Updated . | xargs -0 -n1 -I{} make docs 89 | docs: 90 | -make kill-docs 91 | nohup godoc -play -http=127.0.0.1:6064 /dev/null 2>&1 & echo $$! > .godoc.pid 92 | cat .godoc.pid 93 | 94 | 95 | kill-docs: 96 | @cat .godoc.pid 97 | kill -9 $$(cat .godoc.pid) 98 | rm .godoc.pid 99 | 100 | 101 | open-docs: 102 | open http://localhost:6064/pkg/github.com/$(REPO_OWNER)/migrate 103 | 104 | 105 | # example: make release V=0.0.0 106 | release: 107 | git tag v$(V) 108 | @read -p "Press enter to confirm and push to origin ..." && git push origin v$(V) 109 | 110 | 111 | define external_deps 112 | @echo '-- $(1)'; go list -f '{{join .Deps "\n"}}' $(1) | grep -v github.com/$(REPO_OWNER)/migrate | xargs go list -f '{{if not .Standard}}{{.ImportPath}}{{end}}' 113 | 114 | endef 115 | 116 | 117 | .PHONY: build-cli clean test-short test test-with-flags deps html-coverage \ 118 | restore-import-paths rewrite-import-paths list-external-deps release \ 119 | docs kill-docs open-docs kill-orphaned-docker-containers 120 | 121 | SHELL = /bin/bash 122 | RAND = $(shell echo $$RANDOM) 123 | 124 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # migrate 2 | 3 | Database migrations written in Go. Use as CLI or import as library. 4 | 5 | # DEPRECATED 6 | 7 | __mattes/migrate is now [golang-migrate/migrate](https://github.com/golang-migrate/migrate)__ 8 | 9 | Please open issues and pull requests in the new repository. 10 | -------------------------------------------------------------------------------- /cli/README.md: -------------------------------------------------------------------------------- 1 | # migrate CLI 2 | 3 | ## Installation 4 | 5 | #### With Go toolchain 6 | 7 | ``` 8 | $ go get -u -d github.com/mattes/migrate/cli github.com/lib/pq 9 | $ go build -tags 'postgres' -o /usr/local/bin/migrate github.com/mattes/migrate/cli 10 | ``` 11 | 12 | Note: This example builds the cli which will only work with postgres. In order 13 | to build the cli for use with other databases, replace the `postgres` build tag 14 | with the appropriate database tag(s) for the databases desired. The tags 15 | correspond to the names of the sub-packages underneath the 16 | [`database`](../database) package. 17 | 18 | #### MacOS 19 | 20 | ([todo #156](https://github.com/mattes/migrate/issues/156)) 21 | 22 | ``` 23 | $ brew install migrate --with-postgres 24 | ``` 25 | 26 | #### Linux (*.deb package) 27 | 28 | ``` 29 | $ curl -L https://packagecloud.io/mattes/migrate/gpgkey | apt-key add - 30 | $ echo "deb https://packagecloud.io/mattes/migrate/ubuntu/ xenial main" > /etc/apt/sources.list.d/migrate.list 31 | $ apt-get update 32 | $ apt-get install -y migrate 33 | ``` 34 | 35 | #### Download pre-build binary (Windows, MacOS, or Linux) 36 | 37 | [Release Downloads](https://github.com/mattes/migrate/releases) 38 | 39 | ``` 40 | $ curl -L https://github.com/mattes/migrate/releases/download/$version/migrate.$platform-amd64.tar.gz | tar xvz 41 | ``` 42 | 43 | 44 | 45 | ## Usage 46 | 47 | ``` 48 | $ migrate -help 49 | Usage: migrate OPTIONS COMMAND [arg...] 50 | migrate [ -version | -help ] 51 | 52 | Options: 53 | -source Location of the migrations (driver://url) 54 | -path Shorthand for -source=file://path 55 | -database Run migrations against this database (driver://url) 56 | -prefetch N Number of migrations to load in advance before executing (default 10) 57 | -lock-timeout N Allow N seconds to acquire database lock (default 15) 58 | -verbose Print verbose logging 59 | -version Print version 60 | -help Print usage 61 | 62 | Commands: 63 | create [-ext E] [-dir D] NAME 64 | Create a set of timestamped up/down migrations titled NAME, in directory D with extension E 65 | goto V Migrate to version V 66 | up [N] Apply all or N up migrations 67 | down [N] Apply all or N down migrations 68 | drop Drop everyting inside database 69 | force V Set version V but don't run migration (ignores dirty state) 70 | version Print current migration version 71 | ``` 72 | 73 | 74 | So let's say you want to run the first two migrations 75 | 76 | ``` 77 | $ migrate -database postgres://localhost:5432/database up 2 78 | ``` 79 | 80 | If your migrations are hosted on github 81 | 82 | ``` 83 | $ migrate -source github://mattes:personal-access-token@mattes/migrate_test \ 84 | -database postgres://localhost:5432/database down 2 85 | ``` 86 | 87 | The CLI will gracefully stop at a safe point when SIGINT (ctrl+c) is received. 88 | Send SIGKILL for immediate halt. 89 | 90 | 91 | 92 | ## Reading CLI arguments from somewhere else 93 | 94 | ##### ENV variables 95 | 96 | ``` 97 | $ migrate -database "$MY_MIGRATE_DATABASE" 98 | ``` 99 | 100 | ##### JSON files 101 | 102 | Check out https://stedolan.github.io/jq/ 103 | 104 | ``` 105 | $ migrate -database "$(cat config.json | jq '.database')" 106 | ``` 107 | 108 | ##### YAML files 109 | 110 | ```` 111 | $ migrate -database "$(cat config/database.yml | ruby -ryaml -e "print YAML.load(STDIN.read)['database']")" 112 | $ migrate -database "$(cat config/database.yml | python -c 'import yaml,sys;print yaml.safe_load(sys.stdin)["database"]')" 113 | ``` 114 | -------------------------------------------------------------------------------- /cli/build_aws-s3.go: -------------------------------------------------------------------------------- 1 | // +build aws-s3 2 | 3 | package main 4 | 5 | import ( 6 | _ "github.com/mattes/migrate/source/aws-s3" 7 | ) 8 | -------------------------------------------------------------------------------- /cli/build_cassandra.go: -------------------------------------------------------------------------------- 1 | // +build cassandra 2 | 3 | package main 4 | 5 | import ( 6 | _ "github.com/mattes/migrate/database/cassandra" 7 | ) 8 | -------------------------------------------------------------------------------- /cli/build_clickhouse.go: -------------------------------------------------------------------------------- 1 | // +build clickhouse 2 | 3 | package main 4 | 5 | import ( 6 | _ "github.com/kshvakov/clickhouse" 7 | _ "github.com/mattes/migrate/database/clickhouse" 8 | ) 9 | -------------------------------------------------------------------------------- /cli/build_cockroachdb.go: -------------------------------------------------------------------------------- 1 | // +build cockroachdb 2 | 3 | package main 4 | 5 | import ( 6 | _ "github.com/mattes/migrate/database/cockroachdb" 7 | ) 8 | -------------------------------------------------------------------------------- /cli/build_github.go: -------------------------------------------------------------------------------- 1 | // +build github 2 | 3 | package main 4 | 5 | import ( 6 | _ "github.com/mattes/migrate/source/github" 7 | ) 8 | -------------------------------------------------------------------------------- /cli/build_go-bindata.go: -------------------------------------------------------------------------------- 1 | // +build go-bindata 2 | 3 | package main 4 | 5 | import ( 6 | _ "github.com/mattes/migrate/source/go-bindata" 7 | ) 8 | -------------------------------------------------------------------------------- /cli/build_google-cloud-storage.go: -------------------------------------------------------------------------------- 1 | // +build google-cloud-storage 2 | 3 | package main 4 | 5 | import ( 6 | _ "github.com/mattes/migrate/source/google-cloud-storage" 7 | ) 8 | -------------------------------------------------------------------------------- /cli/build_mysql.go: -------------------------------------------------------------------------------- 1 | // +build mysql 2 | 3 | package main 4 | 5 | import ( 6 | _ "github.com/mattes/migrate/database/mysql" 7 | ) 8 | -------------------------------------------------------------------------------- /cli/build_postgres.go: -------------------------------------------------------------------------------- 1 | // +build postgres 2 | 3 | package main 4 | 5 | import ( 6 | _ "github.com/mattes/migrate/database/postgres" 7 | ) 8 | -------------------------------------------------------------------------------- /cli/build_ql.go: -------------------------------------------------------------------------------- 1 | // +build ql 2 | 3 | package main 4 | 5 | import ( 6 | _ "github.com/mattes/migrate/database/ql" 7 | ) 8 | -------------------------------------------------------------------------------- /cli/build_redshift.go: -------------------------------------------------------------------------------- 1 | // +build redshift 2 | 3 | package main 4 | 5 | import ( 6 | _ "github.com/mattes/migrate/database/redshift" 7 | ) 8 | -------------------------------------------------------------------------------- /cli/build_spanner.go: -------------------------------------------------------------------------------- 1 | // +build spanner 2 | 3 | package main 4 | 5 | import ( 6 | _ "github.com/mattes/migrate/database/spanner" 7 | ) 8 | -------------------------------------------------------------------------------- /cli/build_sqlite3.go: -------------------------------------------------------------------------------- 1 | // +build sqlite3 2 | 3 | package main 4 | 5 | import ( 6 | _ "github.com/mattes/migrate/database/sqlite3" 7 | ) 8 | -------------------------------------------------------------------------------- /cli/commands.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/mattes/migrate" 5 | _ "github.com/mattes/migrate/database/stub" // TODO remove again 6 | _ "github.com/mattes/migrate/source/file" 7 | "os" 8 | "fmt" 9 | ) 10 | 11 | func createCmd(dir string, timestamp int64, name string, ext string) { 12 | base := fmt.Sprintf("%v%v_%v.", dir, timestamp, name) 13 | os.MkdirAll(dir, os.ModePerm) 14 | createFile(base + "up" + ext) 15 | createFile(base + "down" + ext) 16 | } 17 | 18 | func createFile(fname string) { 19 | if _, err := os.Create(fname); err != nil { 20 | log.fatalErr(err) 21 | } 22 | } 23 | 24 | func gotoCmd(m *migrate.Migrate, v uint) { 25 | if err := m.Migrate(v); err != nil { 26 | if err != migrate.ErrNoChange { 27 | log.fatalErr(err) 28 | } else { 29 | log.Println(err) 30 | } 31 | } 32 | } 33 | 34 | func upCmd(m *migrate.Migrate, limit int) { 35 | if limit >= 0 { 36 | if err := m.Steps(limit); err != nil { 37 | if err != migrate.ErrNoChange { 38 | log.fatalErr(err) 39 | } else { 40 | log.Println(err) 41 | } 42 | } 43 | } else { 44 | if err := m.Up(); err != nil { 45 | if err != migrate.ErrNoChange { 46 | log.fatalErr(err) 47 | } else { 48 | log.Println(err) 49 | } 50 | } 51 | } 52 | } 53 | 54 | func downCmd(m *migrate.Migrate, limit int) { 55 | if limit >= 0 { 56 | if err := m.Steps(-limit); err != nil { 57 | if err != migrate.ErrNoChange { 58 | log.fatalErr(err) 59 | } else { 60 | log.Println(err) 61 | } 62 | } 63 | } else { 64 | if err := m.Down(); err != nil { 65 | if err != migrate.ErrNoChange { 66 | log.fatalErr(err) 67 | } else { 68 | log.Println(err) 69 | } 70 | } 71 | } 72 | } 73 | 74 | func dropCmd(m *migrate.Migrate) { 75 | if err := m.Drop(); err != nil { 76 | log.fatalErr(err) 77 | } 78 | } 79 | 80 | func forceCmd(m *migrate.Migrate, v int) { 81 | if err := m.Force(v); err != nil { 82 | log.fatalErr(err) 83 | } 84 | } 85 | 86 | func versionCmd(m *migrate.Migrate) { 87 | v, dirty, err := m.Version() 88 | if err != nil { 89 | log.fatalErr(err) 90 | } 91 | if dirty { 92 | log.Printf("%v (dirty)\n", v) 93 | } else { 94 | log.Println(v) 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /cli/examples/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:xenial 2 | 3 | RUN apt-get update && \ 4 | apt-get install -y curl apt-transport-https 5 | 6 | RUN curl -L https://packagecloud.io/mattes/migrate/gpgkey | apt-key add - && \ 7 | echo "deb https://packagecloud.io/mattes/migrate/ubuntu/ xenial main" > /etc/apt/sources.list.d/migrate.list && \ 8 | apt-get update && \ 9 | apt-get install -y migrate 10 | 11 | RUN migrate -version 12 | 13 | -------------------------------------------------------------------------------- /cli/log.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | logpkg "log" 6 | "os" 7 | ) 8 | 9 | type Log struct { 10 | verbose bool 11 | } 12 | 13 | func (l *Log) Printf(format string, v ...interface{}) { 14 | if l.verbose { 15 | logpkg.Printf(format, v...) 16 | } else { 17 | fmt.Fprintf(os.Stderr, format, v...) 18 | } 19 | } 20 | 21 | func (l *Log) Println(args ...interface{}) { 22 | if l.verbose { 23 | logpkg.Println(args...) 24 | } else { 25 | fmt.Fprintln(os.Stderr, args...) 26 | } 27 | } 28 | 29 | func (l *Log) Verbose() bool { 30 | return l.verbose 31 | } 32 | 33 | func (l *Log) fatalf(format string, v ...interface{}) { 34 | l.Printf(format, v...) 35 | os.Exit(1) 36 | } 37 | 38 | func (l *Log) fatal(args ...interface{}) { 39 | l.Println(args...) 40 | os.Exit(1) 41 | } 42 | 43 | func (l *Log) fatalErr(err error) { 44 | l.fatal("error:", err) 45 | } 46 | -------------------------------------------------------------------------------- /cli/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | "os/signal" 8 | "strconv" 9 | "strings" 10 | "syscall" 11 | "time" 12 | 13 | "github.com/mattes/migrate" 14 | ) 15 | 16 | // set main log 17 | var log = &Log{} 18 | 19 | func main() { 20 | helpPtr := flag.Bool("help", false, "") 21 | versionPtr := flag.Bool("version", false, "") 22 | verbosePtr := flag.Bool("verbose", false, "") 23 | prefetchPtr := flag.Uint("prefetch", 10, "") 24 | lockTimeoutPtr := flag.Uint("lock-timeout", 15, "") 25 | pathPtr := flag.String("path", "", "") 26 | databasePtr := flag.String("database", "", "") 27 | sourcePtr := flag.String("source", "", "") 28 | 29 | flag.Usage = func() { 30 | fmt.Fprint(os.Stderr, 31 | `Usage: migrate OPTIONS COMMAND [arg...] 32 | migrate [ -version | -help ] 33 | 34 | Options: 35 | -source Location of the migrations (driver://url) 36 | -path Shorthand for -source=file://path 37 | -database Run migrations against this database (driver://url) 38 | -prefetch N Number of migrations to load in advance before executing (default 10) 39 | -lock-timeout N Allow N seconds to acquire database lock (default 15) 40 | -verbose Print verbose logging 41 | -version Print version 42 | -help Print usage 43 | 44 | Commands: 45 | create [-ext E] [-dir D] NAME 46 | Create a set of timestamped up/down migrations titled NAME, in directory D with extension E 47 | goto V Migrate to version V 48 | up [N] Apply all or N up migrations 49 | down [N] Apply all or N down migrations 50 | drop Drop everyting inside database 51 | force V Set version V but don't run migration (ignores dirty state) 52 | version Print current migration version 53 | `) 54 | } 55 | 56 | flag.Parse() 57 | 58 | // initialize logger 59 | log.verbose = *verbosePtr 60 | 61 | // show cli version 62 | if *versionPtr { 63 | fmt.Fprintln(os.Stderr, Version) 64 | os.Exit(0) 65 | } 66 | 67 | // show help 68 | if *helpPtr { 69 | flag.Usage() 70 | os.Exit(0) 71 | } 72 | 73 | // translate -path into -source if given 74 | if *sourcePtr == "" && *pathPtr != "" { 75 | *sourcePtr = fmt.Sprintf("file://%v", *pathPtr) 76 | } 77 | 78 | // initialize migrate 79 | // don't catch migraterErr here and let each command decide 80 | // how it wants to handle the error 81 | migrater, migraterErr := migrate.New(*sourcePtr, *databasePtr) 82 | defer func() { 83 | if migraterErr == nil { 84 | migrater.Close() 85 | } 86 | }() 87 | if migraterErr == nil { 88 | migrater.Log = log 89 | migrater.PrefetchMigrations = *prefetchPtr 90 | migrater.LockTimeout = time.Duration(int64(*lockTimeoutPtr)) * time.Second 91 | 92 | // handle Ctrl+c 93 | signals := make(chan os.Signal, 1) 94 | signal.Notify(signals, syscall.SIGINT) 95 | go func() { 96 | for range signals { 97 | log.Println("Stopping after this running migration ...") 98 | migrater.GracefulStop <- true 99 | return 100 | } 101 | }() 102 | } 103 | 104 | startTime := time.Now() 105 | 106 | switch flag.Arg(0) { 107 | case "create": 108 | args := flag.Args()[1:] 109 | 110 | createFlagSet := flag.NewFlagSet("create", flag.ExitOnError) 111 | extPtr := createFlagSet.String("ext", "", "File extension") 112 | dirPtr := createFlagSet.String("dir", "", "Directory to place file in (default: current working directory)") 113 | createFlagSet.Parse(args) 114 | 115 | if createFlagSet.NArg() == 0 { 116 | log.fatal("error: please specify name") 117 | } 118 | name := createFlagSet.Arg(0) 119 | 120 | if *extPtr != "" { 121 | *extPtr = "." + strings.TrimPrefix(*extPtr, ".") 122 | } 123 | if *dirPtr != "" { 124 | *dirPtr = strings.Trim(*dirPtr, "/") + "/" 125 | } 126 | 127 | timestamp := startTime.Unix() 128 | 129 | createCmd(*dirPtr, timestamp, name, *extPtr) 130 | 131 | case "goto": 132 | if migraterErr != nil { 133 | log.fatalErr(migraterErr) 134 | } 135 | 136 | if flag.Arg(1) == "" { 137 | log.fatal("error: please specify version argument V") 138 | } 139 | 140 | v, err := strconv.ParseUint(flag.Arg(1), 10, 64) 141 | if err != nil { 142 | log.fatal("error: can't read version argument V") 143 | } 144 | 145 | gotoCmd(migrater, uint(v)) 146 | 147 | if log.verbose { 148 | log.Println("Finished after", time.Now().Sub(startTime)) 149 | } 150 | 151 | case "up": 152 | if migraterErr != nil { 153 | log.fatalErr(migraterErr) 154 | } 155 | 156 | limit := -1 157 | if flag.Arg(1) != "" { 158 | n, err := strconv.ParseUint(flag.Arg(1), 10, 64) 159 | if err != nil { 160 | log.fatal("error: can't read limit argument N") 161 | } 162 | limit = int(n) 163 | } 164 | 165 | upCmd(migrater, limit) 166 | 167 | if log.verbose { 168 | log.Println("Finished after", time.Now().Sub(startTime)) 169 | } 170 | 171 | case "down": 172 | if migraterErr != nil { 173 | log.fatalErr(migraterErr) 174 | } 175 | 176 | limit := -1 177 | if flag.Arg(1) != "" { 178 | n, err := strconv.ParseUint(flag.Arg(1), 10, 64) 179 | if err != nil { 180 | log.fatal("error: can't read limit argument N") 181 | } 182 | limit = int(n) 183 | } 184 | 185 | downCmd(migrater, limit) 186 | 187 | if log.verbose { 188 | log.Println("Finished after", time.Now().Sub(startTime)) 189 | } 190 | 191 | case "drop": 192 | if migraterErr != nil { 193 | log.fatalErr(migraterErr) 194 | } 195 | 196 | dropCmd(migrater) 197 | 198 | if log.verbose { 199 | log.Println("Finished after", time.Now().Sub(startTime)) 200 | } 201 | 202 | case "force": 203 | if migraterErr != nil { 204 | log.fatalErr(migraterErr) 205 | } 206 | 207 | if flag.Arg(1) == "" { 208 | log.fatal("error: please specify version argument V") 209 | } 210 | 211 | v, err := strconv.ParseInt(flag.Arg(1), 10, 64) 212 | if err != nil { 213 | log.fatal("error: can't read version argument V") 214 | } 215 | 216 | if v < -1 { 217 | log.fatal("error: argument V must be >= -1") 218 | } 219 | 220 | forceCmd(migrater, int(v)) 221 | 222 | if log.verbose { 223 | log.Println("Finished after", time.Now().Sub(startTime)) 224 | } 225 | 226 | case "version": 227 | if migraterErr != nil { 228 | log.fatalErr(migraterErr) 229 | } 230 | 231 | versionCmd(migrater) 232 | 233 | default: 234 | flag.Usage() 235 | os.Exit(0) 236 | } 237 | } 238 | -------------------------------------------------------------------------------- /cli/version.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // Version is set in Makefile with build flags 4 | var Version = "dev" 5 | -------------------------------------------------------------------------------- /database/cassandra/README.md: -------------------------------------------------------------------------------- 1 | # Cassandra 2 | 3 | * Drop command will not work on Cassandra 2.X because it rely on 4 | system_schema table which comes with 3.X 5 | * Other commands should work properly but are **not tested** 6 | 7 | 8 | ## Usage 9 | `cassandra://host:port/keyspace?param1=value¶m2=value2` 10 | 11 | 12 | | URL Query | Default value | Description | 13 | |------------|-------------|-----------| 14 | | `x-migrations-table` | schema_migrations | Name of the migrations table | 15 | | `port` | 9042 | The port to bind to | 16 | | `consistency` | ALL | Migration consistency 17 | | `protocol` | | Cassandra protocol version (3 or 4) 18 | | `timeout` | 1 minute | Migration timeout 19 | | `username` | nil | Username to use when authenticating. | 20 | | `password` | nil | Password to use when authenticating. | 21 | 22 | 23 | `timeout` is parsed using [time.ParseDuration(s string)](https://golang.org/pkg/time/#ParseDuration) 24 | 25 | 26 | ## Upgrading from v1 27 | 28 | 1. Write down the current migration version from schema_migrations 29 | 2. `DROP TABLE schema_migrations` 30 | 4. Download and install the latest migrate version. 31 | 5. Force the current migration version with `migrate force `. 32 | -------------------------------------------------------------------------------- /database/cassandra/cassandra.go: -------------------------------------------------------------------------------- 1 | package cassandra 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "io/ioutil" 7 | nurl "net/url" 8 | "strconv" 9 | "time" 10 | 11 | "github.com/gocql/gocql" 12 | "github.com/mattes/migrate/database" 13 | ) 14 | 15 | func init() { 16 | db := new(Cassandra) 17 | database.Register("cassandra", db) 18 | } 19 | 20 | var DefaultMigrationsTable = "schema_migrations" 21 | var dbLocked = false 22 | 23 | var ( 24 | ErrNilConfig = fmt.Errorf("no config") 25 | ErrNoKeyspace = fmt.Errorf("no keyspace provided") 26 | ErrDatabaseDirty = fmt.Errorf("database is dirty") 27 | ) 28 | 29 | type Config struct { 30 | MigrationsTable string 31 | KeyspaceName string 32 | } 33 | 34 | type Cassandra struct { 35 | session *gocql.Session 36 | isLocked bool 37 | 38 | // Open and WithInstance need to guarantee that config is never nil 39 | config *Config 40 | } 41 | 42 | func (p *Cassandra) Open(url string) (database.Driver, error) { 43 | u, err := nurl.Parse(url) 44 | if err != nil { 45 | return nil, err 46 | } 47 | 48 | // Check for missing mandatory attributes 49 | if len(u.Path) == 0 { 50 | return nil, ErrNoKeyspace 51 | } 52 | 53 | migrationsTable := u.Query().Get("x-migrations-table") 54 | if len(migrationsTable) == 0 { 55 | migrationsTable = DefaultMigrationsTable 56 | } 57 | 58 | p.config = &Config{ 59 | KeyspaceName: u.Path, 60 | MigrationsTable: migrationsTable, 61 | } 62 | 63 | cluster := gocql.NewCluster(u.Host) 64 | cluster.Keyspace = u.Path[1:len(u.Path)] 65 | cluster.Consistency = gocql.All 66 | cluster.Timeout = 1 * time.Minute 67 | 68 | if len(u.Query().Get("username")) > 0 && len(u.Query().Get("password")) > 0 { 69 | authenticator := gocql.PasswordAuthenticator{ 70 | Username: u.Query().Get("username"), 71 | Password: u.Query().Get("password"), 72 | } 73 | cluster.Authenticator = authenticator 74 | } 75 | 76 | // Retrieve query string configuration 77 | if len(u.Query().Get("consistency")) > 0 { 78 | var consistency gocql.Consistency 79 | consistency, err = parseConsistency(u.Query().Get("consistency")) 80 | if err != nil { 81 | return nil, err 82 | } 83 | 84 | cluster.Consistency = consistency 85 | } 86 | if len(u.Query().Get("protocol")) > 0 { 87 | var protoversion int 88 | protoversion, err = strconv.Atoi(u.Query().Get("protocol")) 89 | if err != nil { 90 | return nil, err 91 | } 92 | cluster.ProtoVersion = protoversion 93 | } 94 | if len(u.Query().Get("timeout")) > 0 { 95 | var timeout time.Duration 96 | timeout, err = time.ParseDuration(u.Query().Get("timeout")) 97 | if err != nil { 98 | return nil, err 99 | } 100 | cluster.Timeout = timeout 101 | } 102 | 103 | p.session, err = cluster.CreateSession() 104 | 105 | if err != nil { 106 | return nil, err 107 | } 108 | 109 | if err := p.ensureVersionTable(); err != nil { 110 | return nil, err 111 | } 112 | 113 | return p, nil 114 | } 115 | 116 | func (p *Cassandra) Close() error { 117 | p.session.Close() 118 | return nil 119 | } 120 | 121 | func (p *Cassandra) Lock() error { 122 | if dbLocked { 123 | return database.ErrLocked 124 | } 125 | dbLocked = true 126 | return nil 127 | } 128 | 129 | func (p *Cassandra) Unlock() error { 130 | dbLocked = false 131 | return nil 132 | } 133 | 134 | func (p *Cassandra) Run(migration io.Reader) error { 135 | migr, err := ioutil.ReadAll(migration) 136 | if err != nil { 137 | return err 138 | } 139 | // run migration 140 | query := string(migr[:]) 141 | if err := p.session.Query(query).Exec(); err != nil { 142 | // TODO: cast to Cassandra error and get line number 143 | return database.Error{OrigErr: err, Err: "migration failed", Query: migr} 144 | } 145 | 146 | return nil 147 | } 148 | 149 | func (p *Cassandra) SetVersion(version int, dirty bool) error { 150 | query := `TRUNCATE "` + p.config.MigrationsTable + `"` 151 | if err := p.session.Query(query).Exec(); err != nil { 152 | return &database.Error{OrigErr: err, Query: []byte(query)} 153 | } 154 | if version >= 0 { 155 | query = `INSERT INTO "` + p.config.MigrationsTable + `" (version, dirty) VALUES (?, ?)` 156 | if err := p.session.Query(query, version, dirty).Exec(); err != nil { 157 | return &database.Error{OrigErr: err, Query: []byte(query)} 158 | } 159 | } 160 | 161 | return nil 162 | } 163 | 164 | // Return current keyspace version 165 | func (p *Cassandra) Version() (version int, dirty bool, err error) { 166 | query := `SELECT version, dirty FROM "` + p.config.MigrationsTable + `" LIMIT 1` 167 | err = p.session.Query(query).Scan(&version, &dirty) 168 | switch { 169 | case err == gocql.ErrNotFound: 170 | return database.NilVersion, false, nil 171 | 172 | case err != nil: 173 | if _, ok := err.(*gocql.Error); ok { 174 | return database.NilVersion, false, nil 175 | } 176 | return 0, false, &database.Error{OrigErr: err, Query: []byte(query)} 177 | 178 | default: 179 | return version, dirty, nil 180 | } 181 | } 182 | 183 | func (p *Cassandra) Drop() error { 184 | // select all tables in current schema 185 | query := fmt.Sprintf(`SELECT table_name from system_schema.tables WHERE keyspace_name='%s'`, p.config.KeyspaceName[1:]) // Skip '/' character 186 | iter := p.session.Query(query).Iter() 187 | var tableName string 188 | for iter.Scan(&tableName) { 189 | err := p.session.Query(fmt.Sprintf(`DROP TABLE %s`, tableName)).Exec() 190 | if err != nil { 191 | return err 192 | } 193 | } 194 | // Re-create the version table 195 | if err := p.ensureVersionTable(); err != nil { 196 | return err 197 | } 198 | return nil 199 | } 200 | 201 | // Ensure version table exists 202 | func (p *Cassandra) ensureVersionTable() error { 203 | err := p.session.Query(fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (version bigint, dirty boolean, PRIMARY KEY(version))", p.config.MigrationsTable)).Exec() 204 | if err != nil { 205 | return err 206 | } 207 | if _, _, err = p.Version(); err != nil { 208 | return err 209 | } 210 | return nil 211 | } 212 | 213 | // ParseConsistency wraps gocql.ParseConsistency 214 | // to return an error instead of a panicking. 215 | func parseConsistency(consistencyStr string) (consistency gocql.Consistency, err error) { 216 | defer func() { 217 | if r := recover(); r != nil { 218 | var ok bool 219 | err, ok = r.(error) 220 | if !ok { 221 | err = fmt.Errorf("Failed to parse consistency \"%s\": %v", consistencyStr, r) 222 | } 223 | } 224 | }() 225 | consistency = gocql.ParseConsistency(consistencyStr) 226 | 227 | return consistency, nil 228 | } 229 | -------------------------------------------------------------------------------- /database/cassandra/cassandra_test.go: -------------------------------------------------------------------------------- 1 | package cassandra 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | dt "github.com/mattes/migrate/database/testing" 7 | mt "github.com/mattes/migrate/testing" 8 | "github.com/gocql/gocql" 9 | "time" 10 | "strconv" 11 | ) 12 | 13 | var versions = []mt.Version{ 14 | {Image: "cassandra:3.0.10"}, 15 | {Image: "cassandra:3.0"}, 16 | } 17 | 18 | func isReady(i mt.Instance) bool { 19 | // Cassandra exposes 5 ports (7000, 7001, 7199, 9042 & 9160) 20 | // We only need the port bound to 9042, but we can only access to the first one 21 | // through 'i.Port()' (which calls DockerContainer.firstPortMapping()) 22 | // So we need to get port mapping to retrieve correct port number bound to 9042 23 | portMap := i.NetworkSettings().Ports 24 | port, _ := strconv.Atoi(portMap["9042/tcp"][0].HostPort) 25 | 26 | cluster := gocql.NewCluster(i.Host()) 27 | cluster.Port = port 28 | //cluster.ProtoVersion = 4 29 | cluster.Consistency = gocql.All 30 | cluster.Timeout = 1 * time.Minute 31 | p, err := cluster.CreateSession() 32 | if err != nil { 33 | return false 34 | } 35 | // Create keyspace for tests 36 | p.Query("CREATE KEYSPACE testks WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor':1}").Exec() 37 | return true 38 | } 39 | 40 | func Test(t *testing.T) { 41 | mt.ParallelTest(t, versions, isReady, 42 | func(t *testing.T, i mt.Instance) { 43 | p := &Cassandra{} 44 | portMap := i.NetworkSettings().Ports 45 | port, _ := strconv.Atoi(portMap["9042/tcp"][0].HostPort) 46 | addr := fmt.Sprintf("cassandra://%v:%v/testks", i.Host(), port) 47 | d, err := p.Open(addr) 48 | if err != nil { 49 | t.Fatalf("%v", err) 50 | } 51 | dt.Test(t, d, []byte("SELECT table_name from system_schema.tables")) 52 | }) 53 | } 54 | -------------------------------------------------------------------------------- /database/clickhouse/README.md: -------------------------------------------------------------------------------- 1 | # ClickHouse 2 | 3 | `clickhouse://host:port?username=user&password=qwerty&database=clicks` 4 | 5 | | URL Query | Description | 6 | |------------|-------------| 7 | | `x-migrations-table`| Name of the migrations table | 8 | | `database` | The name of the database to connect to | 9 | | `username` | The user to sign in as | 10 | | `password` | The user's password | 11 | | `host` | The host to connect to. | 12 | | `port` | The port to bind to. | 13 | -------------------------------------------------------------------------------- /database/clickhouse/clickhouse.go: -------------------------------------------------------------------------------- 1 | package clickhouse 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "net/url" 9 | "time" 10 | 11 | "github.com/mattes/migrate" 12 | "github.com/mattes/migrate/database" 13 | ) 14 | 15 | var DefaultMigrationsTable = "schema_migrations" 16 | 17 | var ErrNilConfig = fmt.Errorf("no config") 18 | 19 | type Config struct { 20 | DatabaseName string 21 | MigrationsTable string 22 | } 23 | 24 | func init() { 25 | database.Register("clickhouse", &ClickHouse{}) 26 | } 27 | 28 | func WithInstance(conn *sql.DB, config *Config) (database.Driver, error) { 29 | if config == nil { 30 | return nil, ErrNilConfig 31 | } 32 | 33 | if err := conn.Ping(); err != nil { 34 | return nil, err 35 | } 36 | 37 | ch := &ClickHouse{ 38 | conn: conn, 39 | config: config, 40 | } 41 | 42 | if err := ch.init(); err != nil { 43 | return nil, err 44 | } 45 | 46 | return ch, nil 47 | } 48 | 49 | type ClickHouse struct { 50 | conn *sql.DB 51 | config *Config 52 | } 53 | 54 | func (ch *ClickHouse) Open(dsn string) (database.Driver, error) { 55 | purl, err := url.Parse(dsn) 56 | if err != nil { 57 | return nil, err 58 | } 59 | q := migrate.FilterCustomQuery(purl) 60 | q.Scheme = "tcp" 61 | conn, err := sql.Open("clickhouse", q.String()) 62 | if err != nil { 63 | return nil, err 64 | } 65 | 66 | ch = &ClickHouse{ 67 | conn: conn, 68 | config: &Config{ 69 | MigrationsTable: purl.Query().Get("x-migrations-table"), 70 | DatabaseName: purl.Query().Get("database"), 71 | }, 72 | } 73 | 74 | if err := ch.init(); err != nil { 75 | return nil, err 76 | } 77 | 78 | return ch, nil 79 | } 80 | 81 | func (ch *ClickHouse) init() error { 82 | if len(ch.config.DatabaseName) == 0 { 83 | if err := ch.conn.QueryRow("SELECT currentDatabase()").Scan(&ch.config.DatabaseName); err != nil { 84 | return err 85 | } 86 | } 87 | 88 | if len(ch.config.MigrationsTable) == 0 { 89 | ch.config.MigrationsTable = DefaultMigrationsTable 90 | } 91 | 92 | return ch.ensureVersionTable() 93 | } 94 | 95 | func (ch *ClickHouse) Run(r io.Reader) error { 96 | migration, err := ioutil.ReadAll(r) 97 | if err != nil { 98 | return err 99 | } 100 | if _, err := ch.conn.Exec(string(migration)); err != nil { 101 | return database.Error{OrigErr: err, Err: "migration failed", Query: migration} 102 | } 103 | 104 | return nil 105 | } 106 | func (ch *ClickHouse) Version() (int, bool, error) { 107 | var ( 108 | version int 109 | dirty uint8 110 | query = "SELECT version, dirty FROM `" + ch.config.MigrationsTable + "` ORDER BY sequence DESC LIMIT 1" 111 | ) 112 | if err := ch.conn.QueryRow(query).Scan(&version, &dirty); err != nil { 113 | if err == sql.ErrNoRows { 114 | return database.NilVersion, false, nil 115 | } 116 | return 0, false, &database.Error{OrigErr: err, Query: []byte(query)} 117 | } 118 | return version, dirty == 1, nil 119 | } 120 | 121 | func (ch *ClickHouse) SetVersion(version int, dirty bool) error { 122 | var ( 123 | bool = func(v bool) uint8 { 124 | if v { 125 | return 1 126 | } 127 | return 0 128 | } 129 | tx, err = ch.conn.Begin() 130 | ) 131 | if err != nil { 132 | return err 133 | } 134 | 135 | query := "INSERT INTO " + ch.config.MigrationsTable + " (version, dirty, sequence) VALUES (?, ?, ?)" 136 | if _, err := tx.Exec(query, version, bool(dirty), time.Now().UnixNano()); err != nil { 137 | return &database.Error{OrigErr: err, Query: []byte(query)} 138 | } 139 | 140 | return tx.Commit() 141 | } 142 | 143 | func (ch *ClickHouse) ensureVersionTable() error { 144 | var ( 145 | table string 146 | query = "SHOW TABLES FROM " + ch.config.DatabaseName + " LIKE '" + ch.config.MigrationsTable + "'" 147 | ) 148 | // check if migration table exists 149 | if err := ch.conn.QueryRow(query).Scan(&table); err != nil { 150 | if err != sql.ErrNoRows { 151 | return &database.Error{OrigErr: err, Query: []byte(query)} 152 | } 153 | } else { 154 | return nil 155 | } 156 | // if not, create the empty migration table 157 | query = ` 158 | CREATE TABLE ` + ch.config.MigrationsTable + ` ( 159 | version UInt32, 160 | dirty UInt8, 161 | sequence UInt64 162 | ) Engine=TinyLog 163 | ` 164 | if _, err := ch.conn.Exec(query); err != nil { 165 | return &database.Error{OrigErr: err, Query: []byte(query)} 166 | } 167 | return nil 168 | } 169 | 170 | func (ch *ClickHouse) Drop() error { 171 | var ( 172 | query = "SHOW TABLES FROM " + ch.config.DatabaseName 173 | tables, err = ch.conn.Query(query) 174 | ) 175 | if err != nil { 176 | return &database.Error{OrigErr: err, Query: []byte(query)} 177 | } 178 | defer tables.Close() 179 | for tables.Next() { 180 | var table string 181 | if err := tables.Scan(&table); err != nil { 182 | return err 183 | } 184 | 185 | query = "DROP TABLE IF EXISTS " + ch.config.DatabaseName + "." + table 186 | 187 | if _, err := ch.conn.Exec(query); err != nil { 188 | return &database.Error{OrigErr: err, Query: []byte(query)} 189 | } 190 | } 191 | return ch.ensureVersionTable() 192 | } 193 | 194 | func (ch *ClickHouse) Lock() error { return nil } 195 | func (ch *ClickHouse) Unlock() error { return nil } 196 | func (ch *ClickHouse) Close() error { return ch.conn.Close() } 197 | -------------------------------------------------------------------------------- /database/clickhouse/examples/migrations/001_init.down.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS test_1; -------------------------------------------------------------------------------- /database/clickhouse/examples/migrations/001_init.up.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE test_1 ( 2 | Date Date 3 | ) Engine=Memory; -------------------------------------------------------------------------------- /database/clickhouse/examples/migrations/002_create_table.down.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS test_2; -------------------------------------------------------------------------------- /database/clickhouse/examples/migrations/002_create_table.up.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE test_2 ( 2 | Date Date 3 | ) Engine=Memory; -------------------------------------------------------------------------------- /database/cockroachdb/README.md: -------------------------------------------------------------------------------- 1 | # cockroachdb 2 | 3 | `cockroachdb://user:password@host:port/dbname?query` (`cockroach://`, and `crdb-postgres://` work, too) 4 | 5 | | URL Query | WithInstance Config | Description | 6 | |------------|---------------------|-------------| 7 | | `x-migrations-table` | `MigrationsTable` | Name of the migrations table | 8 | | `x-lock-table` | `LockTable` | Name of the table which maintains the migration lock | 9 | | `x-force-lock` | `ForceLock` | Force lock acquisition to fix faulty migrations which may not have released the schema lock (Boolean, default is `false`) | 10 | | `dbname` | `DatabaseName` | The name of the database to connect to | 11 | | `user` | | The user to sign in as | 12 | | `password` | | The user's password | 13 | | `host` | | The host to connect to. Values that start with / are for unix domain sockets. (default is localhost) | 14 | | `port` | | The port to bind to. (default is 5432) | 15 | | `connect_timeout` | | Maximum wait for connection, in seconds. Zero or not specified means wait indefinitely. | 16 | | `sslcert` | | Cert file location. The file must contain PEM encoded data. | 17 | | `sslkey` | | Key file location. The file must contain PEM encoded data. | 18 | | `sslrootcert` | | The location of the root certificate file. The file must contain PEM encoded data. | 19 | | `sslmode` | | Whether or not to use SSL (disable\|require\|verify-ca\|verify-full) | 20 | -------------------------------------------------------------------------------- /database/cockroachdb/cockroachdb_test.go: -------------------------------------------------------------------------------- 1 | package cockroachdb 2 | 3 | // error codes https://github.com/lib/pq/blob/master/error.go 4 | 5 | import ( 6 | //"bytes" 7 | "database/sql" 8 | "fmt" 9 | "io" 10 | "testing" 11 | 12 | "github.com/lib/pq" 13 | dt "github.com/mattes/migrate/database/testing" 14 | mt "github.com/mattes/migrate/testing" 15 | "bytes" 16 | ) 17 | 18 | var versions = []mt.Version{ 19 | {Image: "cockroachdb/cockroach:v1.0.2", Cmd: []string{"start", "--insecure"}}, 20 | } 21 | 22 | func isReady(i mt.Instance) bool { 23 | db, err := sql.Open("postgres", fmt.Sprintf("postgres://root@%v:%v?sslmode=disable", i.Host(), i.PortFor(26257))) 24 | if err != nil { 25 | return false 26 | } 27 | defer db.Close() 28 | err = db.Ping() 29 | if err == io.EOF { 30 | _, err = db.Exec("CREATE DATABASE migrate") 31 | return err == nil; 32 | } else if e, ok := err.(*pq.Error); ok { 33 | if e.Code.Name() == "cannot_connect_now" { 34 | return false 35 | } 36 | } 37 | 38 | _, err = db.Exec("CREATE DATABASE migrate") 39 | return err == nil; 40 | 41 | return true 42 | } 43 | 44 | func Test(t *testing.T) { 45 | mt.ParallelTest(t, versions, isReady, 46 | func(t *testing.T, i mt.Instance) { 47 | c := &CockroachDb{} 48 | addr := fmt.Sprintf("cockroach://root@%v:%v/migrate?sslmode=disable", i.Host(), i.PortFor(26257)) 49 | d, err := c.Open(addr) 50 | if err != nil { 51 | t.Fatalf("%v", err) 52 | } 53 | dt.Test(t, d, []byte("SELECT 1")) 54 | }) 55 | } 56 | 57 | func TestMultiStatement(t *testing.T) { 58 | mt.ParallelTest(t, versions, isReady, 59 | func(t *testing.T, i mt.Instance) { 60 | c := &CockroachDb{} 61 | addr := fmt.Sprintf("cockroach://root@%v:%v/migrate?sslmode=disable", i.Host(), i.Port()) 62 | d, err := c.Open(addr) 63 | if err != nil { 64 | t.Fatalf("%v", err) 65 | } 66 | if err := d.Run(bytes.NewReader([]byte("CREATE TABLE foo (foo text); CREATE TABLE bar (bar text);"))); err != nil { 67 | t.Fatalf("expected err to be nil, got %v", err) 68 | } 69 | 70 | // make sure second table exists 71 | var exists bool 72 | if err := d.(*CockroachDb).db.QueryRow("SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'bar' AND table_schema = (SELECT current_schema()))").Scan(&exists); err != nil { 73 | t.Fatal(err) 74 | } 75 | if !exists { 76 | t.Fatalf("expected table bar to exist") 77 | } 78 | }) 79 | } 80 | 81 | func TestFilterCustomQuery(t *testing.T) { 82 | mt.ParallelTest(t, versions, isReady, 83 | func(t *testing.T, i mt.Instance) { 84 | c := &CockroachDb{} 85 | addr := fmt.Sprintf("cockroach://root@%v:%v/migrate?sslmode=disable&x-custom=foobar", i.Host(), i.PortFor(26257)) 86 | _, err := c.Open(addr) 87 | if err != nil { 88 | t.Fatalf("%v", err) 89 | } 90 | }) 91 | } 92 | -------------------------------------------------------------------------------- /database/cockroachdb/examples/migrations/1085649617_create_users_table.down.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS users; 2 | -------------------------------------------------------------------------------- /database/cockroachdb/examples/migrations/1085649617_create_users_table.up.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE users ( 2 | user_id INT UNIQUE, 3 | name STRING(40), 4 | email STRING(40) 5 | ); 6 | -------------------------------------------------------------------------------- /database/cockroachdb/examples/migrations/1185749658_add_city_to_users.down.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE users DROP COLUMN IF EXISTS city; 2 | -------------------------------------------------------------------------------- /database/cockroachdb/examples/migrations/1185749658_add_city_to_users.up.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE users ADD COLUMN city TEXT; 2 | -------------------------------------------------------------------------------- /database/cockroachdb/examples/migrations/1285849751_add_index_on_user_emails.down.sql: -------------------------------------------------------------------------------- 1 | DROP INDEX IF EXISTS users_email_index; 2 | -------------------------------------------------------------------------------- /database/cockroachdb/examples/migrations/1285849751_add_index_on_user_emails.up.sql: -------------------------------------------------------------------------------- 1 | CREATE UNIQUE INDEX IF NOT EXISTS users_email_index ON users (email); 2 | 3 | -- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. 4 | -------------------------------------------------------------------------------- /database/cockroachdb/examples/migrations/1385949617_create_books_table.down.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS books; 2 | -------------------------------------------------------------------------------- /database/cockroachdb/examples/migrations/1385949617_create_books_table.up.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE books ( 2 | user_id INT, 3 | name STRING(40), 4 | author STRING(40) 5 | ); 6 | -------------------------------------------------------------------------------- /database/cockroachdb/examples/migrations/1485949617_create_movies_table.down.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS movies; 2 | -------------------------------------------------------------------------------- /database/cockroachdb/examples/migrations/1485949617_create_movies_table.up.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE movies ( 2 | user_id INT, 3 | name STRING(40), 4 | director STRING(40) 5 | ); 6 | -------------------------------------------------------------------------------- /database/cockroachdb/examples/migrations/1585849751_just_a_comment.up.sql: -------------------------------------------------------------------------------- 1 | -- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. 2 | -------------------------------------------------------------------------------- /database/cockroachdb/examples/migrations/1685849751_another_comment.up.sql: -------------------------------------------------------------------------------- 1 | -- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. 2 | -------------------------------------------------------------------------------- /database/cockroachdb/examples/migrations/1785849751_another_comment.up.sql: -------------------------------------------------------------------------------- 1 | -- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. 2 | -------------------------------------------------------------------------------- /database/cockroachdb/examples/migrations/1885849751_another_comment.up.sql: -------------------------------------------------------------------------------- 1 | -- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. 2 | -------------------------------------------------------------------------------- /database/crate/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mattes/migrate/4768a648fbd9e04389a73a21139d14a4ccb1a61a/database/crate/README.md -------------------------------------------------------------------------------- /database/driver.go: -------------------------------------------------------------------------------- 1 | // Package database provides the Database interface. 2 | // All database drivers must implement this interface, register themselves, 3 | // optionally provide a `WithInstance` function and pass the tests 4 | // in package database/testing. 5 | package database 6 | 7 | import ( 8 | "fmt" 9 | "io" 10 | nurl "net/url" 11 | "sync" 12 | ) 13 | 14 | var ( 15 | ErrLocked = fmt.Errorf("can't acquire lock") 16 | ) 17 | 18 | const NilVersion int = -1 19 | 20 | var driversMu sync.RWMutex 21 | var drivers = make(map[string]Driver) 22 | 23 | // Driver is the interface every database driver must implement. 24 | // 25 | // How to implement a database driver? 26 | // 1. Implement this interface. 27 | // 2. Optionally, add a function named `WithInstance`. 28 | // This function should accept an existing DB instance and a Config{} struct 29 | // and return a driver instance. 30 | // 3. Add a test that calls database/testing.go:Test() 31 | // 4. Add own tests for Open(), WithInstance() (when provided) and Close(). 32 | // All other functions are tested by tests in database/testing. 33 | // Saves you some time and makes sure all database drivers behave the same way. 34 | // 5. Call Register in init(). 35 | // 6. Create a migrate/cli/build_.go file 36 | // 7. Add driver name in 'DATABASE' variable in Makefile 37 | // 38 | // Guidelines: 39 | // * Don't try to correct user input. Don't assume things. 40 | // When in doubt, return an error and explain the situation to the user. 41 | // * All configuration input must come from the URL string in func Open() 42 | // or the Config{} struct in WithInstance. Don't os.Getenv(). 43 | type Driver interface { 44 | // Open returns a new driver instance configured with parameters 45 | // coming from the URL string. Migrate will call this function 46 | // only once per instance. 47 | Open(url string) (Driver, error) 48 | 49 | // Close closes the underlying database instance managed by the driver. 50 | // Migrate will call this function only once per instance. 51 | Close() error 52 | 53 | // Lock should acquire a database lock so that only one migration process 54 | // can run at a time. Migrate will call this function before Run is called. 55 | // If the implementation can't provide this functionality, return nil. 56 | // Return database.ErrLocked if database is already locked. 57 | Lock() error 58 | 59 | // Unlock should release the lock. Migrate will call this function after 60 | // all migrations have been run. 61 | Unlock() error 62 | 63 | // Run applies a migration to the database. migration is garantueed to be not nil. 64 | Run(migration io.Reader) error 65 | 66 | // SetVersion saves version and dirty state. 67 | // Migrate will call this function before and after each call to Run. 68 | // version must be >= -1. -1 means NilVersion. 69 | SetVersion(version int, dirty bool) error 70 | 71 | // Version returns the currently active version and if the database is dirty. 72 | // When no migration has been applied, it must return version -1. 73 | // Dirty means, a previous migration failed and user interaction is required. 74 | Version() (version int, dirty bool, err error) 75 | 76 | // Drop deletes everything in the database. 77 | Drop() error 78 | } 79 | 80 | // Open returns a new driver instance. 81 | func Open(url string) (Driver, error) { 82 | u, err := nurl.Parse(url) 83 | if err != nil { 84 | return nil, err 85 | } 86 | 87 | if u.Scheme == "" { 88 | return nil, fmt.Errorf("database driver: invalid URL scheme") 89 | } 90 | 91 | driversMu.RLock() 92 | d, ok := drivers[u.Scheme] 93 | driversMu.RUnlock() 94 | if !ok { 95 | return nil, fmt.Errorf("database driver: unknown driver %v (forgotten import?)", u.Scheme) 96 | } 97 | 98 | return d.Open(url) 99 | } 100 | 101 | // Register globally registers a driver. 102 | func Register(name string, driver Driver) { 103 | driversMu.Lock() 104 | defer driversMu.Unlock() 105 | if driver == nil { 106 | panic("Register driver is nil") 107 | } 108 | if _, dup := drivers[name]; dup { 109 | panic("Register called twice for driver " + name) 110 | } 111 | drivers[name] = driver 112 | } 113 | -------------------------------------------------------------------------------- /database/driver_test.go: -------------------------------------------------------------------------------- 1 | package database 2 | 3 | func ExampleDriver() { 4 | // see database/stub for an example 5 | 6 | // database/stub/stub.go has the driver implementation 7 | // database/stub/stub_test.go runs database/testing/test.go:Test 8 | } 9 | -------------------------------------------------------------------------------- /database/error.go: -------------------------------------------------------------------------------- 1 | package database 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | // Error should be used for errors involving queries ran against the database 8 | type Error struct { 9 | // Optional: the line number 10 | Line uint 11 | 12 | // Query is a query excerpt 13 | Query []byte 14 | 15 | // Err is a useful/helping error message for humans 16 | Err string 17 | 18 | // OrigErr is the underlying error 19 | OrigErr error 20 | } 21 | 22 | func (e Error) Error() string { 23 | if len(e.Err) == 0 { 24 | return fmt.Sprintf("%v in line %v: %s", e.OrigErr, e.Line, e.Query) 25 | } 26 | return fmt.Sprintf("%v in line %v: %s (details: %v)", e.Err, e.Line, e.Query, e.OrigErr) 27 | } 28 | -------------------------------------------------------------------------------- /database/mongodb/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mattes/migrate/4768a648fbd9e04389a73a21139d14a4ccb1a61a/database/mongodb/README.md -------------------------------------------------------------------------------- /database/mysql/README.md: -------------------------------------------------------------------------------- 1 | # MySQL 2 | 3 | `mysql://user:password@tcp(host:port)/dbname?query` 4 | 5 | | URL Query | WithInstance Config | Description | 6 | |------------|---------------------|-------------| 7 | | `x-migrations-table` | `MigrationsTable` | Name of the migrations table | 8 | | `dbname` | `DatabaseName` | The name of the database to connect to | 9 | | `user` | | The user to sign in as | 10 | | `password` | | The user's password | 11 | | `host` | | The host to connect to. | 12 | | `port` | | The port to bind to. | 13 | | `x-tls-ca` | | The location of the root certificate file. | 14 | | `x-tls-cert` | | Cert file location. | 15 | | `x-tls-key` | | Key file location. | 16 | | `x-tls-insecure-skip-verify` | | Whether or not to use SSL (true\|false) | 17 | 18 | ## Use with existing client 19 | 20 | If you use the MySQL driver with existing database client, you must create the client with parameter `multiStatements=true`: 21 | 22 | ```go 23 | package main 24 | 25 | import ( 26 | "database/sql" 27 | 28 | _ "github.com/go-sql-driver/mysql" 29 | "github.com/mattes/migrate" 30 | "github.com/mattes/migrate/database/mysql" 31 | _ "github.com/mattes/migrate/source/file" 32 | ) 33 | 34 | func main() { 35 | db, _ := sql.Open("mysql", "user:password@tcp(host:port)/dbname?multiStatements=true") 36 | driver, _ := mysql.WithInstance(db, &mysql.Config{}) 37 | m, _ := migrate.NewWithDatabaseInstance( 38 | "file:///migrations", 39 | "mysql", 40 | driver, 41 | ) 42 | 43 | m.Steps(2) 44 | } 45 | ``` 46 | 47 | ## Upgrading from v1 48 | 49 | 1. Write down the current migration version from schema_migrations 50 | 1. `DROP TABLE schema_migrations` 51 | 2. Wrap your existing migrations in transactions ([BEGIN/COMMIT](https://dev.mysql.com/doc/refman/5.7/en/commit.html)) if you use multiple statements within one migration. 52 | 3. Download and install the latest migrate version. 53 | 4. Force the current migration version with `migrate force `. 54 | -------------------------------------------------------------------------------- /database/mysql/mysql.go: -------------------------------------------------------------------------------- 1 | package mysql 2 | 3 | import ( 4 | "crypto/tls" 5 | "crypto/x509" 6 | "database/sql" 7 | "fmt" 8 | "io" 9 | "io/ioutil" 10 | nurl "net/url" 11 | "strconv" 12 | "strings" 13 | 14 | "github.com/go-sql-driver/mysql" 15 | "github.com/mattes/migrate" 16 | "github.com/mattes/migrate/database" 17 | ) 18 | 19 | func init() { 20 | database.Register("mysql", &Mysql{}) 21 | } 22 | 23 | var DefaultMigrationsTable = "schema_migrations" 24 | 25 | var ( 26 | ErrDatabaseDirty = fmt.Errorf("database is dirty") 27 | ErrNilConfig = fmt.Errorf("no config") 28 | ErrNoDatabaseName = fmt.Errorf("no database name") 29 | ErrAppendPEM = fmt.Errorf("failed to append PEM") 30 | ) 31 | 32 | type Config struct { 33 | MigrationsTable string 34 | DatabaseName string 35 | } 36 | 37 | type Mysql struct { 38 | db *sql.DB 39 | isLocked bool 40 | 41 | config *Config 42 | } 43 | 44 | // instance must have `multiStatements` set to true 45 | func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) { 46 | if config == nil { 47 | return nil, ErrNilConfig 48 | } 49 | 50 | if err := instance.Ping(); err != nil { 51 | return nil, err 52 | } 53 | 54 | query := `SELECT DATABASE()` 55 | var databaseName sql.NullString 56 | if err := instance.QueryRow(query).Scan(&databaseName); err != nil { 57 | return nil, &database.Error{OrigErr: err, Query: []byte(query)} 58 | } 59 | 60 | if len(databaseName.String) == 0 { 61 | return nil, ErrNoDatabaseName 62 | } 63 | 64 | config.DatabaseName = databaseName.String 65 | 66 | if len(config.MigrationsTable) == 0 { 67 | config.MigrationsTable = DefaultMigrationsTable 68 | } 69 | 70 | mx := &Mysql{ 71 | db: instance, 72 | config: config, 73 | } 74 | 75 | if err := mx.ensureVersionTable(); err != nil { 76 | return nil, err 77 | } 78 | 79 | return mx, nil 80 | } 81 | 82 | func (m *Mysql) Open(url string) (database.Driver, error) { 83 | purl, err := nurl.Parse(url) 84 | if err != nil { 85 | return nil, err 86 | } 87 | 88 | q := purl.Query() 89 | q.Set("multiStatements", "true") 90 | purl.RawQuery = q.Encode() 91 | 92 | db, err := sql.Open("mysql", strings.Replace( 93 | migrate.FilterCustomQuery(purl).String(), "mysql://", "", 1)) 94 | if err != nil { 95 | return nil, err 96 | } 97 | 98 | migrationsTable := purl.Query().Get("x-migrations-table") 99 | if len(migrationsTable) == 0 { 100 | migrationsTable = DefaultMigrationsTable 101 | } 102 | 103 | // use custom TLS? 104 | ctls := purl.Query().Get("tls") 105 | if len(ctls) > 0 { 106 | if _, isBool := readBool(ctls); !isBool && strings.ToLower(ctls) != "skip-verify" { 107 | rootCertPool := x509.NewCertPool() 108 | pem, err := ioutil.ReadFile(purl.Query().Get("x-tls-ca")) 109 | if err != nil { 110 | return nil, err 111 | } 112 | 113 | if ok := rootCertPool.AppendCertsFromPEM(pem); !ok { 114 | return nil, ErrAppendPEM 115 | } 116 | 117 | certs, err := tls.LoadX509KeyPair(purl.Query().Get("x-tls-cert"), purl.Query().Get("x-tls-key")) 118 | if err != nil { 119 | return nil, err 120 | } 121 | 122 | insecureSkipVerify := false 123 | if len(purl.Query().Get("x-tls-insecure-skip-verify")) > 0 { 124 | x, err := strconv.ParseBool(purl.Query().Get("x-tls-insecure-skip-verify")) 125 | if err != nil { 126 | return nil, err 127 | } 128 | insecureSkipVerify = x 129 | } 130 | 131 | mysql.RegisterTLSConfig(ctls, &tls.Config{ 132 | RootCAs: rootCertPool, 133 | Certificates: []tls.Certificate{certs}, 134 | InsecureSkipVerify: insecureSkipVerify, 135 | }) 136 | } 137 | } 138 | 139 | mx, err := WithInstance(db, &Config{ 140 | DatabaseName: purl.Path, 141 | MigrationsTable: migrationsTable, 142 | }) 143 | if err != nil { 144 | return nil, err 145 | } 146 | 147 | return mx, nil 148 | } 149 | 150 | func (m *Mysql) Close() error { 151 | return m.db.Close() 152 | } 153 | 154 | func (m *Mysql) Lock() error { 155 | if m.isLocked { 156 | return database.ErrLocked 157 | } 158 | 159 | aid, err := database.GenerateAdvisoryLockId( 160 | fmt.Sprintf("%s:%s", m.config.DatabaseName, m.config.MigrationsTable)) 161 | if err != nil { 162 | return err 163 | } 164 | 165 | query := "SELECT GET_LOCK(?, 1)" 166 | var success bool 167 | if err := m.db.QueryRow(query, aid).Scan(&success); err != nil { 168 | return &database.Error{OrigErr: err, Err: "try lock failed", Query: []byte(query)} 169 | } 170 | 171 | if success { 172 | m.isLocked = true 173 | return nil 174 | } 175 | 176 | return database.ErrLocked 177 | } 178 | 179 | func (m *Mysql) Unlock() error { 180 | if !m.isLocked { 181 | return nil 182 | } 183 | 184 | aid, err := database.GenerateAdvisoryLockId( 185 | fmt.Sprintf("%s:%s", m.config.DatabaseName, m.config.MigrationsTable)) 186 | if err != nil { 187 | return err 188 | } 189 | 190 | query := `SELECT RELEASE_LOCK(?)` 191 | if _, err := m.db.Exec(query, aid); err != nil { 192 | return &database.Error{OrigErr: err, Query: []byte(query)} 193 | } 194 | 195 | m.isLocked = false 196 | return nil 197 | } 198 | 199 | func (m *Mysql) Run(migration io.Reader) error { 200 | migr, err := ioutil.ReadAll(migration) 201 | if err != nil { 202 | return err 203 | } 204 | 205 | query := string(migr[:]) 206 | if _, err := m.db.Exec(query); err != nil { 207 | return database.Error{OrigErr: err, Err: "migration failed", Query: migr} 208 | } 209 | 210 | return nil 211 | } 212 | 213 | func (m *Mysql) SetVersion(version int, dirty bool) error { 214 | tx, err := m.db.Begin() 215 | if err != nil { 216 | return &database.Error{OrigErr: err, Err: "transaction start failed"} 217 | } 218 | 219 | query := "TRUNCATE `" + m.config.MigrationsTable + "`" 220 | if _, err := m.db.Exec(query); err != nil { 221 | return &database.Error{OrigErr: err, Query: []byte(query)} 222 | } 223 | 224 | if version >= 0 { 225 | query := "INSERT INTO `" + m.config.MigrationsTable + "` (version, dirty) VALUES (?, ?)" 226 | if _, err := m.db.Exec(query, version, dirty); err != nil { 227 | tx.Rollback() 228 | return &database.Error{OrigErr: err, Query: []byte(query)} 229 | } 230 | } 231 | 232 | if err := tx.Commit(); err != nil { 233 | return &database.Error{OrigErr: err, Err: "transaction commit failed"} 234 | } 235 | 236 | return nil 237 | } 238 | 239 | func (m *Mysql) Version() (version int, dirty bool, err error) { 240 | query := "SELECT version, dirty FROM `" + m.config.MigrationsTable + "` LIMIT 1" 241 | err = m.db.QueryRow(query).Scan(&version, &dirty) 242 | switch { 243 | case err == sql.ErrNoRows: 244 | return database.NilVersion, false, nil 245 | 246 | case err != nil: 247 | if e, ok := err.(*mysql.MySQLError); ok { 248 | if e.Number == 0 { 249 | return database.NilVersion, false, nil 250 | } 251 | } 252 | return 0, false, &database.Error{OrigErr: err, Query: []byte(query)} 253 | 254 | default: 255 | return version, dirty, nil 256 | } 257 | } 258 | 259 | func (m *Mysql) Drop() error { 260 | // select all tables 261 | query := `SHOW TABLES LIKE '%'` 262 | tables, err := m.db.Query(query) 263 | if err != nil { 264 | return &database.Error{OrigErr: err, Query: []byte(query)} 265 | } 266 | defer tables.Close() 267 | 268 | // delete one table after another 269 | tableNames := make([]string, 0) 270 | for tables.Next() { 271 | var tableName string 272 | if err := tables.Scan(&tableName); err != nil { 273 | return err 274 | } 275 | if len(tableName) > 0 { 276 | tableNames = append(tableNames, tableName) 277 | } 278 | } 279 | 280 | if len(tableNames) > 0 { 281 | // delete one by one ... 282 | for _, t := range tableNames { 283 | query = "DROP TABLE IF EXISTS `" + t + "` CASCADE" 284 | if _, err := m.db.Exec(query); err != nil { 285 | return &database.Error{OrigErr: err, Query: []byte(query)} 286 | } 287 | } 288 | if err := m.ensureVersionTable(); err != nil { 289 | return err 290 | } 291 | } 292 | 293 | return nil 294 | } 295 | 296 | func (m *Mysql) ensureVersionTable() error { 297 | // check if migration table exists 298 | var result string 299 | query := `SHOW TABLES LIKE "` + m.config.MigrationsTable + `"` 300 | if err := m.db.QueryRow(query).Scan(&result); err != nil { 301 | if err != sql.ErrNoRows { 302 | return &database.Error{OrigErr: err, Query: []byte(query)} 303 | } 304 | } else { 305 | return nil 306 | } 307 | 308 | // if not, create the empty migration table 309 | query = "CREATE TABLE `" + m.config.MigrationsTable + "` (version bigint not null primary key, dirty boolean not null)" 310 | if _, err := m.db.Exec(query); err != nil { 311 | return &database.Error{OrigErr: err, Query: []byte(query)} 312 | } 313 | return nil 314 | } 315 | 316 | // Returns the bool value of the input. 317 | // The 2nd return value indicates if the input was a valid bool value 318 | // See https://github.com/go-sql-driver/mysql/blob/a059889267dc7170331388008528b3b44479bffb/utils.go#L71 319 | func readBool(input string) (value bool, valid bool) { 320 | switch input { 321 | case "1", "true", "TRUE", "True": 322 | return true, true 323 | case "0", "false", "FALSE", "False": 324 | return false, true 325 | } 326 | 327 | // Not a valid bool value 328 | return 329 | } 330 | -------------------------------------------------------------------------------- /database/mysql/mysql_test.go: -------------------------------------------------------------------------------- 1 | package mysql 2 | 3 | import ( 4 | "database/sql" 5 | sqldriver "database/sql/driver" 6 | "fmt" 7 | // "io/ioutil" 8 | // "log" 9 | "testing" 10 | 11 | // "github.com/go-sql-driver/mysql" 12 | dt "github.com/mattes/migrate/database/testing" 13 | mt "github.com/mattes/migrate/testing" 14 | ) 15 | 16 | var versions = []mt.Version{ 17 | {Image: "mysql:8", ENV: []string{"MYSQL_ROOT_PASSWORD=root", "MYSQL_DATABASE=public"}}, 18 | {Image: "mysql:5.7", ENV: []string{"MYSQL_ROOT_PASSWORD=root", "MYSQL_DATABASE=public"}}, 19 | {Image: "mysql:5.6", ENV: []string{"MYSQL_ROOT_PASSWORD=root", "MYSQL_DATABASE=public"}}, 20 | {Image: "mysql:5.5", ENV: []string{"MYSQL_ROOT_PASSWORD=root", "MYSQL_DATABASE=public"}}, 21 | } 22 | 23 | func isReady(i mt.Instance) bool { 24 | db, err := sql.Open("mysql", fmt.Sprintf("root:root@tcp(%v:%v)/public", i.Host(), i.Port())) 25 | if err != nil { 26 | return false 27 | } 28 | defer db.Close() 29 | err = db.Ping() 30 | 31 | if err == sqldriver.ErrBadConn { 32 | return false 33 | } 34 | 35 | return true 36 | } 37 | 38 | func Test(t *testing.T) { 39 | // mysql.SetLogger(mysql.Logger(log.New(ioutil.Discard, "", log.Ltime))) 40 | 41 | mt.ParallelTest(t, versions, isReady, 42 | func(t *testing.T, i mt.Instance) { 43 | p := &Mysql{} 44 | addr := fmt.Sprintf("mysql://root:root@tcp(%v:%v)/public", i.Host(), i.Port()) 45 | d, err := p.Open(addr) 46 | if err != nil { 47 | t.Fatalf("%v", err) 48 | } 49 | dt.Test(t, d, []byte("SELECT 1")) 50 | 51 | // check ensureVersionTable 52 | if err := d.(*Mysql).ensureVersionTable(); err != nil { 53 | t.Fatal(err) 54 | } 55 | // check again 56 | if err := d.(*Mysql).ensureVersionTable(); err != nil { 57 | t.Fatal(err) 58 | } 59 | }) 60 | } 61 | -------------------------------------------------------------------------------- /database/neo4j/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mattes/migrate/4768a648fbd9e04389a73a21139d14a4ccb1a61a/database/neo4j/README.md -------------------------------------------------------------------------------- /database/postgres/README.md: -------------------------------------------------------------------------------- 1 | # postgres 2 | 3 | `postgres://user:password@host:port/dbname?query` (`postgresql://` works, too) 4 | 5 | | URL Query | WithInstance Config | Description | 6 | |------------|---------------------|-------------| 7 | | `x-migrations-table` | `MigrationsTable` | Name of the migrations table | 8 | | `dbname` | `DatabaseName` | The name of the database to connect to | 9 | | `search_path` | | This variable specifies the order in which schemas are searched when an object is referenced by a simple name with no schema specified. | 10 | | `user` | | The user to sign in as | 11 | | `password` | | The user's password | 12 | | `host` | | The host to connect to. Values that start with / are for unix domain sockets. (default is localhost) | 13 | | `port` | | The port to bind to. (default is 5432) | 14 | | `fallback_application_name` | | An application_name to fall back to if one isn't provided. | 15 | | `connect_timeout` | | Maximum wait for connection, in seconds. Zero or not specified means wait indefinitely. | 16 | | `sslcert` | | Cert file location. The file must contain PEM encoded data. | 17 | | `sslkey` | | Key file location. The file must contain PEM encoded data. | 18 | | `sslrootcert` | | The location of the root certificate file. The file must contain PEM encoded data. | 19 | | `sslmode` | | Whether or not to use SSL (disable\|require\|verify-ca\|verify-full) | 20 | 21 | 22 | ## Upgrading from v1 23 | 24 | 1. Write down the current migration version from schema_migrations 25 | 1. `DROP TABLE schema_migrations` 26 | 2. Wrap your existing migrations in transactions ([BEGIN/COMMIT](https://www.postgresql.org/docs/current/static/transaction-iso.html)) if you use multiple statements within one migration. 27 | 3. Download and install the latest migrate version. 28 | 4. Force the current migration version with `migrate force `. 29 | -------------------------------------------------------------------------------- /database/postgres/examples/migrations/1085649617_create_users_table.down.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS users; 2 | -------------------------------------------------------------------------------- /database/postgres/examples/migrations/1085649617_create_users_table.up.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE users ( 2 | user_id integer unique, 3 | name varchar(40), 4 | email varchar(40) 5 | ); 6 | -------------------------------------------------------------------------------- /database/postgres/examples/migrations/1185749658_add_city_to_users.down.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE users DROP COLUMN IF EXISTS city; 2 | -------------------------------------------------------------------------------- /database/postgres/examples/migrations/1185749658_add_city_to_users.up.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE users ADD COLUMN city varchar(100); 2 | 3 | 4 | -------------------------------------------------------------------------------- /database/postgres/examples/migrations/1285849751_add_index_on_user_emails.down.sql: -------------------------------------------------------------------------------- 1 | DROP INDEX IF EXISTS users_email_index; 2 | -------------------------------------------------------------------------------- /database/postgres/examples/migrations/1285849751_add_index_on_user_emails.up.sql: -------------------------------------------------------------------------------- 1 | CREATE UNIQUE INDEX CONCURRENTLY users_email_index ON users (email); 2 | 3 | -- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. 4 | -------------------------------------------------------------------------------- /database/postgres/examples/migrations/1385949617_create_books_table.down.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS books; 2 | -------------------------------------------------------------------------------- /database/postgres/examples/migrations/1385949617_create_books_table.up.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE books ( 2 | user_id integer, 3 | name varchar(40), 4 | author varchar(40) 5 | ); 6 | -------------------------------------------------------------------------------- /database/postgres/examples/migrations/1485949617_create_movies_table.down.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS movies; 2 | -------------------------------------------------------------------------------- /database/postgres/examples/migrations/1485949617_create_movies_table.up.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE movies ( 2 | user_id integer, 3 | name varchar(40), 4 | director varchar(40) 5 | ); 6 | -------------------------------------------------------------------------------- /database/postgres/examples/migrations/1585849751_just_a_comment.up.sql: -------------------------------------------------------------------------------- 1 | -- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. 2 | -------------------------------------------------------------------------------- /database/postgres/examples/migrations/1685849751_another_comment.up.sql: -------------------------------------------------------------------------------- 1 | -- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. 2 | -------------------------------------------------------------------------------- /database/postgres/examples/migrations/1785849751_another_comment.up.sql: -------------------------------------------------------------------------------- 1 | -- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. 2 | -------------------------------------------------------------------------------- /database/postgres/examples/migrations/1885849751_another_comment.up.sql: -------------------------------------------------------------------------------- 1 | -- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. 2 | -------------------------------------------------------------------------------- /database/postgres/postgres.go: -------------------------------------------------------------------------------- 1 | package postgres 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | nurl "net/url" 9 | 10 | "github.com/lib/pq" 11 | "github.com/mattes/migrate" 12 | "github.com/mattes/migrate/database" 13 | ) 14 | 15 | func init() { 16 | db := Postgres{} 17 | database.Register("postgres", &db) 18 | database.Register("postgresql", &db) 19 | } 20 | 21 | var DefaultMigrationsTable = "schema_migrations" 22 | 23 | var ( 24 | ErrNilConfig = fmt.Errorf("no config") 25 | ErrNoDatabaseName = fmt.Errorf("no database name") 26 | ErrNoSchema = fmt.Errorf("no schema") 27 | ErrDatabaseDirty = fmt.Errorf("database is dirty") 28 | ) 29 | 30 | type Config struct { 31 | MigrationsTable string 32 | DatabaseName string 33 | } 34 | 35 | type Postgres struct { 36 | db *sql.DB 37 | isLocked bool 38 | 39 | // Open and WithInstance need to garantuee that config is never nil 40 | config *Config 41 | } 42 | 43 | func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) { 44 | if config == nil { 45 | return nil, ErrNilConfig 46 | } 47 | 48 | if err := instance.Ping(); err != nil { 49 | return nil, err 50 | } 51 | 52 | query := `SELECT CURRENT_DATABASE()` 53 | var databaseName string 54 | if err := instance.QueryRow(query).Scan(&databaseName); err != nil { 55 | return nil, &database.Error{OrigErr: err, Query: []byte(query)} 56 | } 57 | 58 | if len(databaseName) == 0 { 59 | return nil, ErrNoDatabaseName 60 | } 61 | 62 | config.DatabaseName = databaseName 63 | 64 | if len(config.MigrationsTable) == 0 { 65 | config.MigrationsTable = DefaultMigrationsTable 66 | } 67 | 68 | px := &Postgres{ 69 | db: instance, 70 | config: config, 71 | } 72 | 73 | if err := px.ensureVersionTable(); err != nil { 74 | return nil, err 75 | } 76 | 77 | return px, nil 78 | } 79 | 80 | func (p *Postgres) Open(url string) (database.Driver, error) { 81 | purl, err := nurl.Parse(url) 82 | if err != nil { 83 | return nil, err 84 | } 85 | 86 | db, err := sql.Open("postgres", migrate.FilterCustomQuery(purl).String()) 87 | if err != nil { 88 | return nil, err 89 | } 90 | 91 | migrationsTable := purl.Query().Get("x-migrations-table") 92 | if len(migrationsTable) == 0 { 93 | migrationsTable = DefaultMigrationsTable 94 | } 95 | 96 | px, err := WithInstance(db, &Config{ 97 | DatabaseName: purl.Path, 98 | MigrationsTable: migrationsTable, 99 | }) 100 | if err != nil { 101 | return nil, err 102 | } 103 | 104 | return px, nil 105 | } 106 | 107 | func (p *Postgres) Close() error { 108 | return p.db.Close() 109 | } 110 | 111 | // https://www.postgresql.org/docs/9.6/static/explicit-locking.html#ADVISORY-LOCKS 112 | func (p *Postgres) Lock() error { 113 | if p.isLocked { 114 | return database.ErrLocked 115 | } 116 | 117 | aid, err := database.GenerateAdvisoryLockId(p.config.DatabaseName) 118 | if err != nil { 119 | return err 120 | } 121 | 122 | // This will either obtain the lock immediately and return true, 123 | // or return false if the lock cannot be acquired immediately. 124 | query := `SELECT pg_try_advisory_lock($1)` 125 | var success bool 126 | if err := p.db.QueryRow(query, aid).Scan(&success); err != nil { 127 | return &database.Error{OrigErr: err, Err: "try lock failed", Query: []byte(query)} 128 | } 129 | 130 | if success { 131 | p.isLocked = true 132 | return nil 133 | } 134 | 135 | return database.ErrLocked 136 | } 137 | 138 | func (p *Postgres) Unlock() error { 139 | if !p.isLocked { 140 | return nil 141 | } 142 | 143 | aid, err := database.GenerateAdvisoryLockId(p.config.DatabaseName) 144 | if err != nil { 145 | return err 146 | } 147 | 148 | query := `SELECT pg_advisory_unlock($1)` 149 | if _, err := p.db.Exec(query, aid); err != nil { 150 | return &database.Error{OrigErr: err, Query: []byte(query)} 151 | } 152 | p.isLocked = false 153 | return nil 154 | } 155 | 156 | func (p *Postgres) Run(migration io.Reader) error { 157 | migr, err := ioutil.ReadAll(migration) 158 | if err != nil { 159 | return err 160 | } 161 | 162 | // run migration 163 | query := string(migr[:]) 164 | if _, err := p.db.Exec(query); err != nil { 165 | // TODO: cast to postgress error and get line number 166 | return database.Error{OrigErr: err, Err: "migration failed", Query: migr} 167 | } 168 | 169 | return nil 170 | } 171 | 172 | func (p *Postgres) SetVersion(version int, dirty bool) error { 173 | tx, err := p.db.Begin() 174 | if err != nil { 175 | return &database.Error{OrigErr: err, Err: "transaction start failed"} 176 | } 177 | 178 | query := `TRUNCATE "` + p.config.MigrationsTable + `"` 179 | if _, err := tx.Exec(query); err != nil { 180 | tx.Rollback() 181 | return &database.Error{OrigErr: err, Query: []byte(query)} 182 | } 183 | 184 | if version >= 0 { 185 | query = `INSERT INTO "` + p.config.MigrationsTable + `" (version, dirty) VALUES ($1, $2)` 186 | if _, err := tx.Exec(query, version, dirty); err != nil { 187 | tx.Rollback() 188 | return &database.Error{OrigErr: err, Query: []byte(query)} 189 | } 190 | } 191 | 192 | if err := tx.Commit(); err != nil { 193 | return &database.Error{OrigErr: err, Err: "transaction commit failed"} 194 | } 195 | 196 | return nil 197 | } 198 | 199 | func (p *Postgres) Version() (version int, dirty bool, err error) { 200 | query := `SELECT version, dirty FROM "` + p.config.MigrationsTable + `" LIMIT 1` 201 | err = p.db.QueryRow(query).Scan(&version, &dirty) 202 | switch { 203 | case err == sql.ErrNoRows: 204 | return database.NilVersion, false, nil 205 | 206 | case err != nil: 207 | if e, ok := err.(*pq.Error); ok { 208 | if e.Code.Name() == "undefined_table" { 209 | return database.NilVersion, false, nil 210 | } 211 | } 212 | return 0, false, &database.Error{OrigErr: err, Query: []byte(query)} 213 | 214 | default: 215 | return version, dirty, nil 216 | } 217 | } 218 | 219 | func (p *Postgres) Drop() error { 220 | // select all tables in current schema 221 | query := `SELECT table_name FROM information_schema.tables WHERE table_schema=(SELECT current_schema())` 222 | tables, err := p.db.Query(query) 223 | if err != nil { 224 | return &database.Error{OrigErr: err, Query: []byte(query)} 225 | } 226 | defer tables.Close() 227 | 228 | // delete one table after another 229 | tableNames := make([]string, 0) 230 | for tables.Next() { 231 | var tableName string 232 | if err := tables.Scan(&tableName); err != nil { 233 | return err 234 | } 235 | if len(tableName) > 0 { 236 | tableNames = append(tableNames, tableName) 237 | } 238 | } 239 | 240 | if len(tableNames) > 0 { 241 | // delete one by one ... 242 | for _, t := range tableNames { 243 | query = `DROP TABLE IF EXISTS ` + t + ` CASCADE` 244 | if _, err := p.db.Exec(query); err != nil { 245 | return &database.Error{OrigErr: err, Query: []byte(query)} 246 | } 247 | } 248 | if err := p.ensureVersionTable(); err != nil { 249 | return err 250 | } 251 | } 252 | 253 | return nil 254 | } 255 | 256 | func (p *Postgres) ensureVersionTable() error { 257 | // check if migration table exists 258 | var count int 259 | query := `SELECT COUNT(1) FROM information_schema.tables WHERE table_name = $1 AND table_schema = (SELECT current_schema()) LIMIT 1` 260 | if err := p.db.QueryRow(query, p.config.MigrationsTable).Scan(&count); err != nil { 261 | return &database.Error{OrigErr: err, Query: []byte(query)} 262 | } 263 | if count == 1 { 264 | return nil 265 | } 266 | 267 | // if not, create the empty migration table 268 | query = `CREATE TABLE "` + p.config.MigrationsTable + `" (version bigint not null primary key, dirty boolean not null)` 269 | if _, err := p.db.Exec(query); err != nil { 270 | return &database.Error{OrigErr: err, Query: []byte(query)} 271 | } 272 | return nil 273 | } 274 | -------------------------------------------------------------------------------- /database/postgres/postgres_test.go: -------------------------------------------------------------------------------- 1 | package postgres 2 | 3 | // error codes https://github.com/lib/pq/blob/master/error.go 4 | 5 | import ( 6 | "bytes" 7 | "database/sql" 8 | "fmt" 9 | "io" 10 | "testing" 11 | 12 | "github.com/lib/pq" 13 | dt "github.com/mattes/migrate/database/testing" 14 | mt "github.com/mattes/migrate/testing" 15 | ) 16 | 17 | var versions = []mt.Version{ 18 | {Image: "postgres:9.6"}, 19 | {Image: "postgres:9.5"}, 20 | {Image: "postgres:9.4"}, 21 | {Image: "postgres:9.3"}, 22 | {Image: "postgres:9.2"}, 23 | } 24 | 25 | func isReady(i mt.Instance) bool { 26 | db, err := sql.Open("postgres", fmt.Sprintf("postgres://postgres@%v:%v/postgres?sslmode=disable", i.Host(), i.Port())) 27 | if err != nil { 28 | return false 29 | } 30 | defer db.Close() 31 | err = db.Ping() 32 | if err == io.EOF { 33 | return false 34 | 35 | } else if e, ok := err.(*pq.Error); ok { 36 | if e.Code.Name() == "cannot_connect_now" { 37 | return false 38 | } 39 | } 40 | 41 | return true 42 | } 43 | 44 | func Test(t *testing.T) { 45 | mt.ParallelTest(t, versions, isReady, 46 | func(t *testing.T, i mt.Instance) { 47 | p := &Postgres{} 48 | addr := fmt.Sprintf("postgres://postgres@%v:%v/postgres?sslmode=disable", i.Host(), i.Port()) 49 | d, err := p.Open(addr) 50 | if err != nil { 51 | t.Fatalf("%v", err) 52 | } 53 | dt.Test(t, d, []byte("SELECT 1")) 54 | }) 55 | } 56 | 57 | func TestMultiStatement(t *testing.T) { 58 | mt.ParallelTest(t, versions, isReady, 59 | func(t *testing.T, i mt.Instance) { 60 | p := &Postgres{} 61 | addr := fmt.Sprintf("postgres://postgres@%v:%v/postgres?sslmode=disable", i.Host(), i.Port()) 62 | d, err := p.Open(addr) 63 | if err != nil { 64 | t.Fatalf("%v", err) 65 | } 66 | if err := d.Run(bytes.NewReader([]byte("CREATE TABLE foo (foo text); CREATE TABLE bar (bar text);"))); err != nil { 67 | t.Fatalf("expected err to be nil, got %v", err) 68 | } 69 | 70 | // make sure second table exists 71 | var exists bool 72 | if err := d.(*Postgres).db.QueryRow("SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'bar' AND table_schema = (SELECT current_schema()))").Scan(&exists); err != nil { 73 | t.Fatal(err) 74 | } 75 | if !exists { 76 | t.Fatalf("expected table bar to exist") 77 | } 78 | }) 79 | } 80 | 81 | func TestFilterCustomQuery(t *testing.T) { 82 | mt.ParallelTest(t, versions, isReady, 83 | func(t *testing.T, i mt.Instance) { 84 | p := &Postgres{} 85 | addr := fmt.Sprintf("postgres://postgres@%v:%v/postgres?sslmode=disable&x-custom=foobar", i.Host(), i.Port()) 86 | _, err := p.Open(addr) 87 | if err != nil { 88 | t.Fatalf("%v", err) 89 | } 90 | }) 91 | } 92 | 93 | func TestWithSchema(t *testing.T) { 94 | mt.ParallelTest(t, versions, isReady, 95 | func(t *testing.T, i mt.Instance) { 96 | p := &Postgres{} 97 | addr := fmt.Sprintf("postgres://postgres@%v:%v/postgres?sslmode=disable", i.Host(), i.Port()) 98 | d, err := p.Open(addr) 99 | if err != nil { 100 | t.Fatalf("%v", err) 101 | } 102 | 103 | // create foobar schema 104 | if err := d.Run(bytes.NewReader([]byte("CREATE SCHEMA foobar AUTHORIZATION postgres"))); err != nil { 105 | t.Fatal(err) 106 | } 107 | if err := d.SetVersion(1, false); err != nil { 108 | t.Fatal(err) 109 | } 110 | 111 | // re-connect using that schema 112 | d2, err := p.Open(fmt.Sprintf("postgres://postgres@%v:%v/postgres?sslmode=disable&search_path=foobar", i.Host(), i.Port())) 113 | if err != nil { 114 | t.Fatalf("%v", err) 115 | } 116 | 117 | version, _, err := d2.Version() 118 | if err != nil { 119 | t.Fatal(err) 120 | } 121 | if version != -1 { 122 | t.Fatal("expected NilVersion") 123 | } 124 | 125 | // now update version and compare 126 | if err := d2.SetVersion(2, false); err != nil { 127 | t.Fatal(err) 128 | } 129 | version, _, err = d2.Version() 130 | if err != nil { 131 | t.Fatal(err) 132 | } 133 | if version != 2 { 134 | t.Fatal("expected version 2") 135 | } 136 | 137 | // meanwhile, the public schema still has the other version 138 | version, _, err = d.Version() 139 | if err != nil { 140 | t.Fatal(err) 141 | } 142 | if version != 1 { 143 | t.Fatal("expected version 2") 144 | } 145 | }) 146 | } 147 | 148 | func TestWithInstance(t *testing.T) { 149 | 150 | } 151 | -------------------------------------------------------------------------------- /database/ql/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mattes/migrate/4768a648fbd9e04389a73a21139d14a4ccb1a61a/database/ql/README.md -------------------------------------------------------------------------------- /database/ql/migration/33_create_table.down.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS pets; -------------------------------------------------------------------------------- /database/ql/migration/33_create_table.up.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE pets ( 2 | name string 3 | ); -------------------------------------------------------------------------------- /database/ql/migration/44_alter_table.down.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS pets; -------------------------------------------------------------------------------- /database/ql/migration/44_alter_table.up.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE pets ADD predator bool;; -------------------------------------------------------------------------------- /database/ql/ql.go: -------------------------------------------------------------------------------- 1 | package ql 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "strings" 9 | 10 | nurl "net/url" 11 | 12 | _ "github.com/cznic/ql/driver" 13 | "github.com/mattes/migrate" 14 | "github.com/mattes/migrate/database" 15 | ) 16 | 17 | func init() { 18 | database.Register("ql", &Ql{}) 19 | } 20 | 21 | var DefaultMigrationsTable = "schema_migrations" 22 | var ( 23 | ErrDatabaseDirty = fmt.Errorf("database is dirty") 24 | ErrNilConfig = fmt.Errorf("no config") 25 | ErrNoDatabaseName = fmt.Errorf("no database name") 26 | ErrAppendPEM = fmt.Errorf("failed to append PEM") 27 | ) 28 | 29 | type Config struct { 30 | MigrationsTable string 31 | DatabaseName string 32 | } 33 | 34 | type Ql struct { 35 | db *sql.DB 36 | isLocked bool 37 | 38 | config *Config 39 | } 40 | 41 | func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) { 42 | if config == nil { 43 | return nil, ErrNilConfig 44 | } 45 | 46 | if err := instance.Ping(); err != nil { 47 | return nil, err 48 | } 49 | if len(config.MigrationsTable) == 0 { 50 | config.MigrationsTable = DefaultMigrationsTable 51 | } 52 | 53 | mx := &Ql{ 54 | db: instance, 55 | config: config, 56 | } 57 | if err := mx.ensureVersionTable(); err != nil { 58 | return nil, err 59 | } 60 | return mx, nil 61 | } 62 | func (m *Ql) ensureVersionTable() error { 63 | tx, err := m.db.Begin() 64 | if err != nil { 65 | return err 66 | } 67 | if _, err := tx.Exec(fmt.Sprintf(` 68 | CREATE TABLE IF NOT EXISTS %s (version uint64,dirty bool); 69 | CREATE UNIQUE INDEX IF NOT EXISTS version_unique ON %s (version); 70 | `, m.config.MigrationsTable, m.config.MigrationsTable)); err != nil { 71 | if err := tx.Rollback(); err != nil { 72 | return err 73 | } 74 | return err 75 | } 76 | if err := tx.Commit(); err != nil { 77 | return err 78 | } 79 | return nil 80 | } 81 | 82 | func (m *Ql) Open(url string) (database.Driver, error) { 83 | purl, err := nurl.Parse(url) 84 | if err != nil { 85 | return nil, err 86 | } 87 | dbfile := strings.Replace(migrate.FilterCustomQuery(purl).String(), "ql://", "", 1) 88 | db, err := sql.Open("ql", dbfile) 89 | if err != nil { 90 | return nil, err 91 | } 92 | migrationsTable := purl.Query().Get("x-migrations-table") 93 | if len(migrationsTable) == 0 { 94 | migrationsTable = DefaultMigrationsTable 95 | } 96 | mx, err := WithInstance(db, &Config{ 97 | DatabaseName: purl.Path, 98 | MigrationsTable: migrationsTable, 99 | }) 100 | if err != nil { 101 | return nil, err 102 | } 103 | return mx, nil 104 | } 105 | func (m *Ql) Close() error { 106 | return m.db.Close() 107 | } 108 | func (m *Ql) Drop() error { 109 | query := `SELECT Name FROM __Table` 110 | tables, err := m.db.Query(query) 111 | if err != nil { 112 | return &database.Error{OrigErr: err, Query: []byte(query)} 113 | } 114 | defer tables.Close() 115 | tableNames := make([]string, 0) 116 | for tables.Next() { 117 | var tableName string 118 | if err := tables.Scan(&tableName); err != nil { 119 | return err 120 | } 121 | if len(tableName) > 0 { 122 | if strings.HasPrefix(tableName, "__") == false { 123 | tableNames = append(tableNames, tableName) 124 | } 125 | } 126 | } 127 | if len(tableNames) > 0 { 128 | for _, t := range tableNames { 129 | query := "DROP TABLE " + t 130 | err = m.executeQuery(query) 131 | if err != nil { 132 | return &database.Error{OrigErr: err, Query: []byte(query)} 133 | } 134 | } 135 | if err := m.ensureVersionTable(); err != nil { 136 | return err 137 | } 138 | } 139 | 140 | return nil 141 | } 142 | func (m *Ql) Lock() error { 143 | if m.isLocked { 144 | return database.ErrLocked 145 | } 146 | m.isLocked = true 147 | return nil 148 | } 149 | func (m *Ql) Unlock() error { 150 | if !m.isLocked { 151 | return nil 152 | } 153 | m.isLocked = false 154 | return nil 155 | } 156 | func (m *Ql) Run(migration io.Reader) error { 157 | migr, err := ioutil.ReadAll(migration) 158 | if err != nil { 159 | return err 160 | } 161 | query := string(migr[:]) 162 | 163 | return m.executeQuery(query) 164 | } 165 | func (m *Ql) executeQuery(query string) error { 166 | tx, err := m.db.Begin() 167 | if err != nil { 168 | return &database.Error{OrigErr: err, Err: "transaction start failed"} 169 | } 170 | if _, err := tx.Exec(query); err != nil { 171 | tx.Rollback() 172 | return &database.Error{OrigErr: err, Query: []byte(query)} 173 | } 174 | if err := tx.Commit(); err != nil { 175 | return &database.Error{OrigErr: err, Err: "transaction commit failed"} 176 | } 177 | return nil 178 | } 179 | func (m *Ql) SetVersion(version int, dirty bool) error { 180 | tx, err := m.db.Begin() 181 | if err != nil { 182 | return &database.Error{OrigErr: err, Err: "transaction start failed"} 183 | } 184 | 185 | query := "TRUNCATE TABLE " + m.config.MigrationsTable 186 | if _, err := tx.Exec(query); err != nil { 187 | return &database.Error{OrigErr: err, Query: []byte(query)} 188 | } 189 | 190 | if version >= 0 { 191 | query := fmt.Sprintf(`INSERT INTO %s (version, dirty) VALUES (%d, %t)`, m.config.MigrationsTable, version, dirty) 192 | if _, err := tx.Exec(query); err != nil { 193 | tx.Rollback() 194 | return &database.Error{OrigErr: err, Query: []byte(query)} 195 | } 196 | } 197 | 198 | if err := tx.Commit(); err != nil { 199 | return &database.Error{OrigErr: err, Err: "transaction commit failed"} 200 | } 201 | 202 | return nil 203 | } 204 | 205 | func (m *Ql) Version() (version int, dirty bool, err error) { 206 | query := "SELECT version, dirty FROM " + m.config.MigrationsTable + " LIMIT 1" 207 | err = m.db.QueryRow(query).Scan(&version, &dirty) 208 | if err != nil { 209 | return database.NilVersion, false, nil 210 | } 211 | return version, dirty, nil 212 | } 213 | -------------------------------------------------------------------------------- /database/ql/ql_test.go: -------------------------------------------------------------------------------- 1 | package ql 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "path/filepath" 9 | "testing" 10 | 11 | _ "github.com/cznic/ql/driver" 12 | "github.com/mattes/migrate" 13 | dt "github.com/mattes/migrate/database/testing" 14 | _ "github.com/mattes/migrate/source/file" 15 | ) 16 | 17 | func Test(t *testing.T) { 18 | dir, err := ioutil.TempDir("", "ql-driver-test") 19 | if err != nil { 20 | return 21 | } 22 | defer func() { 23 | os.RemoveAll(dir) 24 | }() 25 | fmt.Printf("DB path : %s\n", filepath.Join(dir, "ql.db")) 26 | p := &Ql{} 27 | addr := fmt.Sprintf("ql://%s", filepath.Join(dir, "ql.db")) 28 | d, err := p.Open(addr) 29 | if err != nil { 30 | t.Fatalf("%v", err) 31 | } 32 | 33 | db, err := sql.Open("ql", filepath.Join(dir, "ql.db")) 34 | if err != nil { 35 | return 36 | } 37 | defer func() { 38 | if err := db.Close(); err != nil { 39 | return 40 | } 41 | }() 42 | dt.Test(t, d, []byte("CREATE TABLE t (Qty int, Name string);")) 43 | driver, err := WithInstance(db, &Config{}) 44 | if err != nil { 45 | t.Fatalf("%v", err) 46 | } 47 | if err := d.Drop(); err != nil { 48 | t.Fatal(err) 49 | } 50 | 51 | m, err := migrate.NewWithDatabaseInstance( 52 | "file://./migration", 53 | "ql", driver) 54 | if err != nil { 55 | t.Fatalf("%v", err) 56 | } 57 | fmt.Println("UP") 58 | err = m.Up() 59 | if err != nil { 60 | t.Fatalf("%v", err) 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /database/redshift/README.md: -------------------------------------------------------------------------------- 1 | Redshift 2 | === 3 | 4 | This provides a Redshift driver for migrations. It is used whenever the URL of the database starts with `redshift://`. 5 | 6 | Redshift is PostgreSQL compatible but has some specific features (or lack thereof) that require slightly different behavior. 7 | -------------------------------------------------------------------------------- /database/redshift/redshift.go: -------------------------------------------------------------------------------- 1 | package redshift 2 | 3 | import ( 4 | "net/url" 5 | 6 | "github.com/mattes/migrate/database" 7 | "github.com/mattes/migrate/database/postgres" 8 | ) 9 | 10 | // init registers the driver under the name 'redshift' 11 | func init() { 12 | db := new(Redshift) 13 | db.Driver = new(postgres.Postgres) 14 | 15 | database.Register("redshift", db) 16 | } 17 | 18 | // Redshift is a wrapper around the PostgreSQL driver which implements Redshift-specific behavior. 19 | // 20 | // Currently, the only different behaviour is the lack of locking in Redshift. The (Un)Lock() method(s) have been overridden from the PostgreSQL adapter to simply return nil. 21 | type Redshift struct { 22 | // The wrapped PostgreSQL driver. 23 | database.Driver 24 | } 25 | 26 | // Open implements the database.Driver interface by parsing the URL, switching the scheme from "redshift" to "postgres", and delegating to the underlying PostgreSQL driver. 27 | func (driver *Redshift) Open(dsn string) (database.Driver, error) { 28 | parsed, err := url.Parse(dsn) 29 | if err != nil { 30 | return nil, err 31 | } 32 | 33 | parsed.Scheme = "postgres" 34 | psql, err := driver.Driver.Open(parsed.String()) 35 | if err != nil { 36 | return nil, err 37 | } 38 | 39 | return &Redshift{Driver: psql}, nil 40 | } 41 | 42 | // Lock implements the database.Driver interface by not locking and returning nil. 43 | func (driver *Redshift) Lock() error { return nil } 44 | 45 | // Unlock implements the database.Driver interface by not unlocking and returning nil. 46 | func (driver *Redshift) Unlock() error { return nil } 47 | -------------------------------------------------------------------------------- /database/shell/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mattes/migrate/4768a648fbd9e04389a73a21139d14a4ccb1a61a/database/shell/README.md -------------------------------------------------------------------------------- /database/spanner/README.md: -------------------------------------------------------------------------------- 1 | # Google Cloud Spanner 2 | 3 | ## Usage 4 | 5 | The DSN must be given in the following format. 6 | 7 | `spanner://projects/{projectId}/instances/{instanceId}/databases/{databaseName}` 8 | 9 | See [Google Spanner Documentation](https://cloud.google.com/spanner/docs) for details. 10 | 11 | 12 | | Param | WithInstance Config | Description | 13 | | ----- | ------------------- | ----------- | 14 | | `x-migrations-table` | `MigrationsTable` | Name of the migrations table | 15 | | `url` | `DatabaseName` | The full path to the Spanner database resource. If provided as part of `Config` it must not contain a scheme or query string to match the format `projects/{projectId}/instances/{instanceId}/databases/{databaseName}`| 16 | | `projectId` || The Google Cloud Platform project id 17 | | `instanceId` || The id of the instance running Spanner 18 | | `databaseName` || The name of the Spanner database 19 | 20 | 21 | > **Note:** Google Cloud Spanner migrations can take a considerable amount of 22 | > time. The migrations provided as part of the example take about 6 minutes to 23 | > run on a small instance. 24 | > 25 | > ```log 26 | > 1481574547/u create_users_table (21.354507597s) 27 | > 1496539702/u add_city_to_users (41.647359754s) 28 | > 1496601752/u add_index_on_user_emails (2m12.155787369s) 29 | > 1496602638/u create_books_table (2m30.77299181s) 30 | 31 | ## Testing 32 | 33 | To unit test the `spanner` driver, `SPANNER_DATABASE` needs to be set. You'll 34 | need to sign-up to Google Cloud Platform (GCP) and have a running Spanner 35 | instance since it is not possible to run Google Spanner outside GCP. -------------------------------------------------------------------------------- /database/spanner/examples/migrations/1481574547_create_users_table.down.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE Users 2 | -------------------------------------------------------------------------------- /database/spanner/examples/migrations/1481574547_create_users_table.up.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE Users ( 2 | UserId INT64, 3 | Name STRING(40), 4 | Email STRING(83) 5 | ) PRIMARY KEY(UserId) -------------------------------------------------------------------------------- /database/spanner/examples/migrations/1496539702_add_city_to_users.down.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE Users DROP COLUMN city -------------------------------------------------------------------------------- /database/spanner/examples/migrations/1496539702_add_city_to_users.up.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE Users ADD COLUMN city STRING(100) -------------------------------------------------------------------------------- /database/spanner/examples/migrations/1496601752_add_index_on_user_emails.down.sql: -------------------------------------------------------------------------------- 1 | DROP INDEX UsersEmailIndex 2 | -------------------------------------------------------------------------------- /database/spanner/examples/migrations/1496601752_add_index_on_user_emails.up.sql: -------------------------------------------------------------------------------- 1 | CREATE UNIQUE INDEX UsersEmailIndex ON Users (Email) 2 | -------------------------------------------------------------------------------- /database/spanner/examples/migrations/1496602638_create_books_table.down.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE Books -------------------------------------------------------------------------------- /database/spanner/examples/migrations/1496602638_create_books_table.up.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE Books ( 2 | UserId INT64, 3 | Name STRING(40), 4 | Author STRING(40) 5 | ) PRIMARY KEY(UserId, Name), 6 | INTERLEAVE IN PARENT Users ON DELETE CASCADE 7 | -------------------------------------------------------------------------------- /database/spanner/spanner.go: -------------------------------------------------------------------------------- 1 | package spanner 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "io/ioutil" 7 | "log" 8 | nurl "net/url" 9 | "regexp" 10 | "strings" 11 | 12 | "golang.org/x/net/context" 13 | 14 | "cloud.google.com/go/spanner" 15 | sdb "cloud.google.com/go/spanner/admin/database/apiv1" 16 | 17 | "github.com/mattes/migrate" 18 | "github.com/mattes/migrate/database" 19 | 20 | "google.golang.org/api/iterator" 21 | adminpb "google.golang.org/genproto/googleapis/spanner/admin/database/v1" 22 | ) 23 | 24 | func init() { 25 | db := Spanner{} 26 | database.Register("spanner", &db) 27 | } 28 | 29 | // DefaultMigrationsTable is used if no custom table is specified 30 | const DefaultMigrationsTable = "SchemaMigrations" 31 | 32 | // Driver errors 33 | var ( 34 | ErrNilConfig = fmt.Errorf("no config") 35 | ErrNoDatabaseName = fmt.Errorf("no database name") 36 | ErrNoSchema = fmt.Errorf("no schema") 37 | ErrDatabaseDirty = fmt.Errorf("database is dirty") 38 | ) 39 | 40 | // Config used for a Spanner instance 41 | type Config struct { 42 | MigrationsTable string 43 | DatabaseName string 44 | } 45 | 46 | // Spanner implements database.Driver for Google Cloud Spanner 47 | type Spanner struct { 48 | db *DB 49 | 50 | config *Config 51 | } 52 | 53 | type DB struct { 54 | admin *sdb.DatabaseAdminClient 55 | data *spanner.Client 56 | } 57 | 58 | // WithInstance implements database.Driver 59 | func WithInstance(instance *DB, config *Config) (database.Driver, error) { 60 | if config == nil { 61 | return nil, ErrNilConfig 62 | } 63 | 64 | if len(config.DatabaseName) == 0 { 65 | return nil, ErrNoDatabaseName 66 | } 67 | 68 | if len(config.MigrationsTable) == 0 { 69 | config.MigrationsTable = DefaultMigrationsTable 70 | } 71 | 72 | sx := &Spanner{ 73 | db: instance, 74 | config: config, 75 | } 76 | 77 | if err := sx.ensureVersionTable(); err != nil { 78 | return nil, err 79 | } 80 | 81 | return sx, nil 82 | } 83 | 84 | // Open implements database.Driver 85 | func (s *Spanner) Open(url string) (database.Driver, error) { 86 | purl, err := nurl.Parse(url) 87 | if err != nil { 88 | return nil, err 89 | } 90 | 91 | ctx := context.Background() 92 | 93 | adminClient, err := sdb.NewDatabaseAdminClient(ctx) 94 | if err != nil { 95 | return nil, err 96 | } 97 | dbname := strings.Replace(migrate.FilterCustomQuery(purl).String(), "spanner://", "", 1) 98 | dataClient, err := spanner.NewClient(ctx, dbname) 99 | if err != nil { 100 | log.Fatal(err) 101 | } 102 | 103 | migrationsTable := purl.Query().Get("x-migrations-table") 104 | if len(migrationsTable) == 0 { 105 | migrationsTable = DefaultMigrationsTable 106 | } 107 | 108 | db := &DB{admin: adminClient, data: dataClient} 109 | return WithInstance(db, &Config{ 110 | DatabaseName: dbname, 111 | MigrationsTable: migrationsTable, 112 | }) 113 | } 114 | 115 | // Close implements database.Driver 116 | func (s *Spanner) Close() error { 117 | s.db.data.Close() 118 | return s.db.admin.Close() 119 | } 120 | 121 | // Lock implements database.Driver but doesn't do anything because Spanner only 122 | // enqueues the UpdateDatabaseDdlRequest. 123 | func (s *Spanner) Lock() error { 124 | return nil 125 | } 126 | 127 | // Unlock implements database.Driver but no action required, see Lock. 128 | func (s *Spanner) Unlock() error { 129 | return nil 130 | } 131 | 132 | // Run implements database.Driver 133 | func (s *Spanner) Run(migration io.Reader) error { 134 | migr, err := ioutil.ReadAll(migration) 135 | if err != nil { 136 | return err 137 | } 138 | 139 | // run migration 140 | stmts := migrationStatements(migr) 141 | ctx := context.Background() 142 | 143 | op, err := s.db.admin.UpdateDatabaseDdl(ctx, &adminpb.UpdateDatabaseDdlRequest{ 144 | Database: s.config.DatabaseName, 145 | Statements: stmts, 146 | }) 147 | 148 | if err != nil { 149 | return &database.Error{OrigErr: err, Err: "migration failed", Query: migr} 150 | } 151 | 152 | if err := op.Wait(ctx); err != nil { 153 | return &database.Error{OrigErr: err, Err: "migration failed", Query: migr} 154 | } 155 | 156 | return nil 157 | } 158 | 159 | // SetVersion implements database.Driver 160 | func (s *Spanner) SetVersion(version int, dirty bool) error { 161 | ctx := context.Background() 162 | 163 | _, err := s.db.data.ReadWriteTransaction(ctx, 164 | func(ctx context.Context, txn *spanner.ReadWriteTransaction) error { 165 | m := []*spanner.Mutation{ 166 | spanner.Delete(s.config.MigrationsTable, spanner.AllKeys()), 167 | spanner.Insert(s.config.MigrationsTable, 168 | []string{"Version", "Dirty"}, 169 | []interface{}{version, dirty}, 170 | )} 171 | return txn.BufferWrite(m) 172 | }) 173 | if err != nil { 174 | return &database.Error{OrigErr: err} 175 | } 176 | 177 | return nil 178 | } 179 | 180 | // Version implements database.Driver 181 | func (s *Spanner) Version() (version int, dirty bool, err error) { 182 | ctx := context.Background() 183 | 184 | stmt := spanner.Statement{ 185 | SQL: `SELECT Version, Dirty FROM ` + s.config.MigrationsTable + ` LIMIT 1`, 186 | } 187 | iter := s.db.data.Single().Query(ctx, stmt) 188 | defer iter.Stop() 189 | 190 | row, err := iter.Next() 191 | switch err { 192 | case iterator.Done: 193 | return database.NilVersion, false, nil 194 | case nil: 195 | var v int64 196 | if err = row.Columns(&v, &dirty); err != nil { 197 | return 0, false, &database.Error{OrigErr: err, Query: []byte(stmt.SQL)} 198 | } 199 | version = int(v) 200 | default: 201 | return 0, false, &database.Error{OrigErr: err, Query: []byte(stmt.SQL)} 202 | } 203 | 204 | return version, dirty, nil 205 | } 206 | 207 | // Drop implements database.Driver. Retrieves the database schema first and 208 | // creates statements to drop the indexes and tables accordingly. 209 | // Note: The drop statements are created in reverse order to how they're 210 | // provided in the schema. Assuming the schema describes how the database can 211 | // be "build up", it seems logical to "unbuild" the database simply by going the 212 | // opposite direction. More testing 213 | func (s *Spanner) Drop() error { 214 | ctx := context.Background() 215 | res, err := s.db.admin.GetDatabaseDdl(ctx, &adminpb.GetDatabaseDdlRequest{ 216 | Database: s.config.DatabaseName, 217 | }) 218 | if err != nil { 219 | return &database.Error{OrigErr: err, Err: "drop failed"} 220 | } 221 | if len(res.Statements) == 0 { 222 | return nil 223 | } 224 | 225 | r := regexp.MustCompile(`(CREATE TABLE\s(\S+)\s)|(CREATE.+INDEX\s(\S+)\s)`) 226 | stmts := make([]string, 0) 227 | for i := len(res.Statements) - 1; i >= 0; i-- { 228 | s := res.Statements[i] 229 | m := r.FindSubmatch([]byte(s)) 230 | 231 | if len(m) == 0 { 232 | continue 233 | } else if tbl := m[2]; len(tbl) > 0 { 234 | stmts = append(stmts, fmt.Sprintf(`DROP TABLE %s`, tbl)) 235 | } else if idx := m[4]; len(idx) > 0 { 236 | stmts = append(stmts, fmt.Sprintf(`DROP INDEX %s`, idx)) 237 | } 238 | } 239 | 240 | op, err := s.db.admin.UpdateDatabaseDdl(ctx, &adminpb.UpdateDatabaseDdlRequest{ 241 | Database: s.config.DatabaseName, 242 | Statements: stmts, 243 | }) 244 | if err != nil { 245 | return &database.Error{OrigErr: err, Query: []byte(strings.Join(stmts, "; "))} 246 | } 247 | if err := op.Wait(ctx); err != nil { 248 | return &database.Error{OrigErr: err, Query: []byte(strings.Join(stmts, "; "))} 249 | } 250 | 251 | if err := s.ensureVersionTable(); err != nil { 252 | return err 253 | } 254 | 255 | return nil 256 | } 257 | 258 | func (s *Spanner) ensureVersionTable() error { 259 | ctx := context.Background() 260 | tbl := s.config.MigrationsTable 261 | iter := s.db.data.Single().Read(ctx, tbl, spanner.AllKeys(), []string{"Version"}) 262 | if err := iter.Do(func(r *spanner.Row) error { return nil }); err == nil { 263 | return nil 264 | } 265 | 266 | stmt := fmt.Sprintf(`CREATE TABLE %s ( 267 | Version INT64 NOT NULL, 268 | Dirty BOOL NOT NULL 269 | ) PRIMARY KEY(Version)`, tbl) 270 | 271 | op, err := s.db.admin.UpdateDatabaseDdl(ctx, &adminpb.UpdateDatabaseDdlRequest{ 272 | Database: s.config.DatabaseName, 273 | Statements: []string{stmt}, 274 | }) 275 | 276 | if err != nil { 277 | return &database.Error{OrigErr: err, Query: []byte(stmt)} 278 | } 279 | if err := op.Wait(ctx); err != nil { 280 | return &database.Error{OrigErr: err, Query: []byte(stmt)} 281 | } 282 | 283 | return nil 284 | } 285 | 286 | func migrationStatements(migration []byte) []string { 287 | regex := regexp.MustCompile(";$") 288 | migrationString := string(migration[:]) 289 | migrationString = strings.TrimSpace(migrationString) 290 | migrationString = regex.ReplaceAllString(migrationString, "") 291 | 292 | statements := strings.Split(migrationString, ";") 293 | return statements 294 | } 295 | -------------------------------------------------------------------------------- /database/spanner/spanner_test.go: -------------------------------------------------------------------------------- 1 | package spanner 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "testing" 7 | 8 | dt "github.com/mattes/migrate/database/testing" 9 | ) 10 | 11 | func Test(t *testing.T) { 12 | if testing.Short() { 13 | t.Skip("skipping test in short mode.") 14 | } 15 | 16 | db, ok := os.LookupEnv("SPANNER_DATABASE") 17 | if !ok { 18 | t.Skip("SPANNER_DATABASE not set, skipping test.") 19 | } 20 | 21 | s := &Spanner{} 22 | addr := fmt.Sprintf("spanner://%v", db) 23 | d, err := s.Open(addr) 24 | if err != nil { 25 | t.Fatalf("%v", err) 26 | } 27 | dt.Test(t, d, []byte("SELECT 1")) 28 | } 29 | -------------------------------------------------------------------------------- /database/sqlite3/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mattes/migrate/4768a648fbd9e04389a73a21139d14a4ccb1a61a/database/sqlite3/README.md -------------------------------------------------------------------------------- /database/sqlite3/migration/33_create_table.down.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS pets; -------------------------------------------------------------------------------- /database/sqlite3/migration/33_create_table.up.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE pets ( 2 | name string 3 | ); -------------------------------------------------------------------------------- /database/sqlite3/migration/44_alter_table.down.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS pets; -------------------------------------------------------------------------------- /database/sqlite3/migration/44_alter_table.up.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE pets ADD predator bool; 2 | -------------------------------------------------------------------------------- /database/sqlite3/sqlite3.go: -------------------------------------------------------------------------------- 1 | package sqlite3 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "github.com/mattes/migrate" 7 | "github.com/mattes/migrate/database" 8 | _ "github.com/mattn/go-sqlite3" 9 | "io" 10 | "io/ioutil" 11 | nurl "net/url" 12 | "strings" 13 | ) 14 | 15 | func init() { 16 | database.Register("sqlite3", &Sqlite{}) 17 | } 18 | 19 | var DefaultMigrationsTable = "schema_migrations" 20 | var ( 21 | ErrDatabaseDirty = fmt.Errorf("database is dirty") 22 | ErrNilConfig = fmt.Errorf("no config") 23 | ErrNoDatabaseName = fmt.Errorf("no database name") 24 | ) 25 | 26 | type Config struct { 27 | MigrationsTable string 28 | DatabaseName string 29 | } 30 | 31 | type Sqlite struct { 32 | db *sql.DB 33 | isLocked bool 34 | 35 | config *Config 36 | } 37 | 38 | func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) { 39 | if config == nil { 40 | return nil, ErrNilConfig 41 | } 42 | 43 | if err := instance.Ping(); err != nil { 44 | return nil, err 45 | } 46 | if len(config.MigrationsTable) == 0 { 47 | config.MigrationsTable = DefaultMigrationsTable 48 | } 49 | 50 | mx := &Sqlite{ 51 | db: instance, 52 | config: config, 53 | } 54 | if err := mx.ensureVersionTable(); err != nil { 55 | return nil, err 56 | } 57 | return mx, nil 58 | } 59 | 60 | func (m *Sqlite) ensureVersionTable() error { 61 | 62 | query := fmt.Sprintf(` 63 | CREATE TABLE IF NOT EXISTS %s (version uint64,dirty bool); 64 | CREATE UNIQUE INDEX IF NOT EXISTS version_unique ON %s (version); 65 | `, DefaultMigrationsTable, DefaultMigrationsTable) 66 | 67 | if _, err := m.db.Exec(query); err != nil { 68 | return err 69 | } 70 | return nil 71 | } 72 | 73 | func (m *Sqlite) Open(url string) (database.Driver, error) { 74 | purl, err := nurl.Parse(url) 75 | if err != nil { 76 | return nil, err 77 | } 78 | dbfile := strings.Replace(migrate.FilterCustomQuery(purl).String(), "sqlite3://", "", 1) 79 | db, err := sql.Open("sqlite3", dbfile) 80 | if err != nil { 81 | return nil, err 82 | } 83 | 84 | migrationsTable := purl.Query().Get("x-migrations-table") 85 | if len(migrationsTable) == 0 { 86 | migrationsTable = DefaultMigrationsTable 87 | } 88 | mx, err := WithInstance(db, &Config{ 89 | DatabaseName: purl.Path, 90 | MigrationsTable: migrationsTable, 91 | }) 92 | if err != nil { 93 | return nil, err 94 | } 95 | return mx, nil 96 | } 97 | 98 | func (m *Sqlite) Close() error { 99 | return m.db.Close() 100 | } 101 | 102 | func (m *Sqlite) Drop() error { 103 | query := `SELECT name FROM sqlite_master WHERE type = 'table';` 104 | tables, err := m.db.Query(query) 105 | if err != nil { 106 | return &database.Error{OrigErr: err, Query: []byte(query)} 107 | } 108 | defer tables.Close() 109 | tableNames := make([]string, 0) 110 | for tables.Next() { 111 | var tableName string 112 | if err := tables.Scan(&tableName); err != nil { 113 | return err 114 | } 115 | if len(tableName) > 0 { 116 | tableNames = append(tableNames, tableName) 117 | } 118 | } 119 | if len(tableNames) > 0 { 120 | for _, t := range tableNames { 121 | query := "DROP TABLE " + t 122 | err = m.executeQuery(query) 123 | if err != nil { 124 | return &database.Error{OrigErr: err, Query: []byte(query)} 125 | } 126 | } 127 | if err := m.ensureVersionTable(); err != nil { 128 | return err 129 | } 130 | query := "VACUUM" 131 | _, err = m.db.Query(query) 132 | if err != nil { 133 | return &database.Error{OrigErr: err, Query: []byte(query)} 134 | } 135 | } 136 | 137 | return nil 138 | } 139 | 140 | func (m *Sqlite) Lock() error { 141 | if m.isLocked { 142 | return database.ErrLocked 143 | } 144 | m.isLocked = true 145 | return nil 146 | } 147 | 148 | func (m *Sqlite) Unlock() error { 149 | if !m.isLocked { 150 | return nil 151 | } 152 | m.isLocked = false 153 | return nil 154 | } 155 | 156 | func (m *Sqlite) Run(migration io.Reader) error { 157 | migr, err := ioutil.ReadAll(migration) 158 | if err != nil { 159 | return err 160 | } 161 | query := string(migr[:]) 162 | 163 | return m.executeQuery(query) 164 | } 165 | 166 | func (m *Sqlite) executeQuery(query string) error { 167 | tx, err := m.db.Begin() 168 | if err != nil { 169 | return &database.Error{OrigErr: err, Err: "transaction start failed"} 170 | } 171 | if _, err := tx.Exec(query); err != nil { 172 | tx.Rollback() 173 | return &database.Error{OrigErr: err, Query: []byte(query)} 174 | } 175 | if err := tx.Commit(); err != nil { 176 | return &database.Error{OrigErr: err, Err: "transaction commit failed"} 177 | } 178 | return nil 179 | } 180 | 181 | func (m *Sqlite) SetVersion(version int, dirty bool) error { 182 | tx, err := m.db.Begin() 183 | if err != nil { 184 | return &database.Error{OrigErr: err, Err: "transaction start failed"} 185 | } 186 | 187 | query := "DELETE FROM " + m.config.MigrationsTable 188 | if _, err := tx.Exec(query); err != nil { 189 | return &database.Error{OrigErr: err, Query: []byte(query)} 190 | } 191 | 192 | if version >= 0 { 193 | query := fmt.Sprintf(`INSERT INTO %s (version, dirty) VALUES (%d, '%t')`, m.config.MigrationsTable, version, dirty) 194 | if _, err := tx.Exec(query); err != nil { 195 | tx.Rollback() 196 | return &database.Error{OrigErr: err, Query: []byte(query)} 197 | } 198 | } 199 | 200 | if err := tx.Commit(); err != nil { 201 | return &database.Error{OrigErr: err, Err: "transaction commit failed"} 202 | } 203 | 204 | return nil 205 | } 206 | 207 | func (m *Sqlite) Version() (version int, dirty bool, err error) { 208 | query := "SELECT version, dirty FROM " + m.config.MigrationsTable + " LIMIT 1" 209 | err = m.db.QueryRow(query).Scan(&version, &dirty) 210 | if err != nil { 211 | return database.NilVersion, false, nil 212 | } 213 | return version, dirty, nil 214 | } 215 | -------------------------------------------------------------------------------- /database/sqlite3/sqlite3_test.go: -------------------------------------------------------------------------------- 1 | package sqlite3 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "github.com/mattes/migrate" 7 | dt "github.com/mattes/migrate/database/testing" 8 | _ "github.com/mattes/migrate/source/file" 9 | _ "github.com/mattn/go-sqlite3" 10 | "io/ioutil" 11 | "os" 12 | "path/filepath" 13 | "testing" 14 | ) 15 | 16 | func Test(t *testing.T) { 17 | dir, err := ioutil.TempDir("", "sqlite3-driver-test") 18 | if err != nil { 19 | return 20 | } 21 | defer func() { 22 | os.RemoveAll(dir) 23 | }() 24 | fmt.Printf("DB path : %s\n", filepath.Join(dir, "sqlite3.db")) 25 | p := &Sqlite{} 26 | addr := fmt.Sprintf("sqlite3://%s", filepath.Join(dir, "sqlite3.db")) 27 | d, err := p.Open(addr) 28 | if err != nil { 29 | t.Fatalf("%v", err) 30 | } 31 | 32 | db, err := sql.Open("sqlite3", filepath.Join(dir, "sqlite3.db")) 33 | if err != nil { 34 | return 35 | } 36 | defer func() { 37 | if err := db.Close(); err != nil { 38 | return 39 | } 40 | }() 41 | dt.Test(t, d, []byte("CREATE TABLE t (Qty int, Name string);")) 42 | driver, err := WithInstance(db, &Config{}) 43 | if err != nil { 44 | t.Fatalf("%v", err) 45 | } 46 | if err := d.Drop(); err != nil { 47 | t.Fatal(err) 48 | } 49 | 50 | m, err := migrate.NewWithDatabaseInstance( 51 | "file://./migration", 52 | "ql", driver) 53 | if err != nil { 54 | t.Fatalf("%v", err) 55 | } 56 | fmt.Println("UP") 57 | err = m.Up() 58 | if err != nil { 59 | t.Fatalf("%v", err) 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /database/stub/stub.go: -------------------------------------------------------------------------------- 1 | package stub 2 | 3 | import ( 4 | "io" 5 | "io/ioutil" 6 | "reflect" 7 | 8 | "github.com/mattes/migrate/database" 9 | ) 10 | 11 | func init() { 12 | database.Register("stub", &Stub{}) 13 | } 14 | 15 | type Stub struct { 16 | Url string 17 | Instance interface{} 18 | CurrentVersion int 19 | MigrationSequence []string 20 | LastRunMigration []byte // todo: make []string 21 | IsDirty bool 22 | IsLocked bool 23 | 24 | Config *Config 25 | } 26 | 27 | func (s *Stub) Open(url string) (database.Driver, error) { 28 | return &Stub{ 29 | Url: url, 30 | CurrentVersion: -1, 31 | MigrationSequence: make([]string, 0), 32 | Config: &Config{}, 33 | }, nil 34 | } 35 | 36 | type Config struct{} 37 | 38 | func WithInstance(instance interface{}, config *Config) (database.Driver, error) { 39 | return &Stub{ 40 | Instance: instance, 41 | CurrentVersion: -1, 42 | MigrationSequence: make([]string, 0), 43 | Config: config, 44 | }, nil 45 | } 46 | 47 | func (s *Stub) Close() error { 48 | return nil 49 | } 50 | 51 | func (s *Stub) Lock() error { 52 | if s.IsLocked { 53 | return database.ErrLocked 54 | } 55 | s.IsLocked = true 56 | return nil 57 | } 58 | 59 | func (s *Stub) Unlock() error { 60 | s.IsLocked = false 61 | return nil 62 | } 63 | 64 | func (s *Stub) Run(migration io.Reader) error { 65 | m, err := ioutil.ReadAll(migration) 66 | if err != nil { 67 | return err 68 | } 69 | s.LastRunMigration = m 70 | s.MigrationSequence = append(s.MigrationSequence, string(m[:])) 71 | return nil 72 | } 73 | 74 | func (s *Stub) SetVersion(version int, state bool) error { 75 | s.CurrentVersion = version 76 | s.IsDirty = state 77 | return nil 78 | } 79 | 80 | func (s *Stub) Version() (version int, dirty bool, err error) { 81 | return s.CurrentVersion, s.IsDirty, nil 82 | } 83 | 84 | const DROP = "DROP" 85 | 86 | func (s *Stub) Drop() error { 87 | s.CurrentVersion = -1 88 | s.LastRunMigration = nil 89 | s.MigrationSequence = append(s.MigrationSequence, DROP) 90 | return nil 91 | } 92 | 93 | func (s *Stub) EqualSequence(seq []string) bool { 94 | return reflect.DeepEqual(seq, s.MigrationSequence) 95 | } 96 | -------------------------------------------------------------------------------- /database/stub/stub_test.go: -------------------------------------------------------------------------------- 1 | package stub 2 | 3 | import ( 4 | "testing" 5 | 6 | dt "github.com/mattes/migrate/database/testing" 7 | ) 8 | 9 | func Test(t *testing.T) { 10 | s := &Stub{} 11 | d, err := s.Open("") 12 | if err != nil { 13 | t.Fatal(err) 14 | } 15 | dt.Test(t, d, []byte("/* foobar migration */")) 16 | } 17 | -------------------------------------------------------------------------------- /database/testing/testing.go: -------------------------------------------------------------------------------- 1 | // Package testing has the database tests. 2 | // All database drivers must pass the Test function. 3 | // This lives in it's own package so it stays a test dependency. 4 | package testing 5 | 6 | import ( 7 | "bytes" 8 | "fmt" 9 | "io" 10 | "testing" 11 | "time" 12 | 13 | "github.com/mattes/migrate/database" 14 | ) 15 | 16 | // Test runs tests against database implementations. 17 | func Test(t *testing.T, d database.Driver, migration []byte) { 18 | if migration == nil { 19 | panic("test must provide migration reader") 20 | } 21 | 22 | TestNilVersion(t, d) // test first 23 | TestLockAndUnlock(t, d) 24 | TestRun(t, d, bytes.NewReader(migration)) 25 | TestDrop(t, d) 26 | TestSetVersion(t, d) // also tests Version() 27 | } 28 | 29 | func TestNilVersion(t *testing.T, d database.Driver) { 30 | v, _, err := d.Version() 31 | if err != nil { 32 | t.Fatal(err) 33 | } 34 | if v != database.NilVersion { 35 | t.Fatalf("Version: expected version to be NilVersion (-1), got %v", v) 36 | } 37 | } 38 | 39 | func TestLockAndUnlock(t *testing.T, d database.Driver) { 40 | // add a timeout, in case there is a deadlock 41 | done := make(chan bool, 1) 42 | go func() { 43 | timeout := time.After(15 * time.Second) 44 | for { 45 | select { 46 | case <-done: 47 | return 48 | case <-timeout: 49 | panic(fmt.Sprintf("Timeout after 15 seconds. Looks like a deadlock in Lock/UnLock.\n%#v", d)) 50 | } 51 | } 52 | }() 53 | defer func() { 54 | done <- true 55 | }() 56 | 57 | // run the locking test ... 58 | 59 | if err := d.Lock(); err != nil { 60 | t.Fatal(err) 61 | } 62 | 63 | // try to acquire lock again 64 | if err := d.Lock(); err == nil { 65 | t.Fatal("Lock: expected err not to be nil") 66 | } 67 | 68 | // unlock 69 | if err := d.Unlock(); err != nil { 70 | t.Fatal(err) 71 | } 72 | 73 | // try to lock 74 | if err := d.Lock(); err != nil { 75 | t.Fatal(err) 76 | } 77 | if err := d.Unlock(); err != nil { 78 | t.Fatal(err) 79 | } 80 | } 81 | 82 | func TestRun(t *testing.T, d database.Driver, migration io.Reader) { 83 | if migration == nil { 84 | panic("migration can't be nil") 85 | } 86 | 87 | if err := d.Run(migration); err != nil { 88 | t.Fatal(err) 89 | } 90 | } 91 | 92 | func TestDrop(t *testing.T, d database.Driver) { 93 | if err := d.Drop(); err != nil { 94 | t.Fatal(err) 95 | } 96 | } 97 | 98 | func TestSetVersion(t *testing.T, d database.Driver) { 99 | if err := d.SetVersion(1, true); err != nil { 100 | t.Fatal(err) 101 | } 102 | 103 | // call again 104 | if err := d.SetVersion(1, true); err != nil { 105 | t.Fatal(err) 106 | } 107 | 108 | v, dirty, err := d.Version() 109 | if err != nil { 110 | t.Fatal(err) 111 | } 112 | if !dirty { 113 | t.Fatal("expected dirty") 114 | } 115 | if v != 1 { 116 | t.Fatal("expected version to be 1") 117 | } 118 | 119 | if err := d.SetVersion(2, false); err != nil { 120 | t.Fatal(err) 121 | } 122 | 123 | // call again 124 | if err := d.SetVersion(2, false); err != nil { 125 | t.Fatal(err) 126 | } 127 | 128 | v, dirty, err = d.Version() 129 | if err != nil { 130 | t.Fatal(err) 131 | } 132 | if dirty { 133 | t.Fatal("expected not dirty") 134 | } 135 | if v != 2 { 136 | t.Fatal("expected version to be 2") 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /database/util.go: -------------------------------------------------------------------------------- 1 | package database 2 | 3 | import ( 4 | "fmt" 5 | "hash/crc32" 6 | ) 7 | 8 | const advisoryLockIdSalt uint = 1486364155 9 | 10 | // inspired by rails migrations, see https://goo.gl/8o9bCT 11 | func GenerateAdvisoryLockId(databaseName string) (string, error) { 12 | sum := crc32.ChecksumIEEE([]byte(databaseName)) 13 | sum = sum * uint32(advisoryLockIdSalt) 14 | return fmt.Sprintf("%v", sum), nil 15 | } 16 | -------------------------------------------------------------------------------- /database/util_test.go: -------------------------------------------------------------------------------- 1 | package database 2 | 3 | func TestGenerateAdvisoryLockId(t *testing.T) { 4 | id, err := p.generateAdvisoryLockId("database_name") 5 | if err != nil { 6 | t.Errorf("expected err to be nil, got %v", err) 7 | } 8 | if len(id) == 0 { 9 | t.Errorf("expected generated id not to be empty") 10 | } 11 | t.Logf("generated id: %v", id) 12 | } 13 | -------------------------------------------------------------------------------- /log.go: -------------------------------------------------------------------------------- 1 | package migrate 2 | 3 | // Logger is an interface so you can pass in your own 4 | // logging implementation. 5 | type Logger interface { 6 | 7 | // Printf is like fmt.Printf 8 | Printf(format string, v ...interface{}) 9 | 10 | // Verbose should return true when verbose logging output is wanted 11 | Verbose() bool 12 | } 13 | -------------------------------------------------------------------------------- /migration.go: -------------------------------------------------------------------------------- 1 | package migrate 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "io" 7 | "time" 8 | ) 9 | 10 | // DefaultBufferSize sets the in memory buffer size (in Bytes) for every 11 | // pre-read migration (see DefaultPrefetchMigrations). 12 | var DefaultBufferSize = uint(100000) 13 | 14 | // Migration holds information about a migration. 15 | // It is initially created from data coming from the source and then 16 | // used when run against the database. 17 | type Migration struct { 18 | // Identifier can be any string to help identifying 19 | // the migration in the source. 20 | Identifier string 21 | 22 | // Version is the version of this migration. 23 | Version uint 24 | 25 | // TargetVersion is the migration version after this migration 26 | // has been applied to the database. 27 | // Can be -1, implying that this is a NilVersion. 28 | TargetVersion int 29 | 30 | // Body holds an io.ReadCloser to the source. 31 | Body io.ReadCloser 32 | 33 | // BufferedBody holds an buffered io.Reader to the underlying Body. 34 | BufferedBody io.Reader 35 | 36 | // BufferSize defaults to DefaultBufferSize 37 | BufferSize uint 38 | 39 | // bufferWriter holds an io.WriteCloser and pipes to BufferBody. 40 | // It's an *Closer for flow control. 41 | bufferWriter io.WriteCloser 42 | 43 | // Scheduled is the time when the migration was scheduled/ queued. 44 | Scheduled time.Time 45 | 46 | // StartedBuffering is the time when buffering of the migration source started. 47 | StartedBuffering time.Time 48 | 49 | // FinishedBuffering is the time when buffering of the migration source finished. 50 | FinishedBuffering time.Time 51 | 52 | // FinishedReading is the time when the migration source is fully read. 53 | FinishedReading time.Time 54 | 55 | // BytesRead holds the number of Bytes read from the migration source. 56 | BytesRead int64 57 | } 58 | 59 | // NewMigration returns a new Migration and sets the body, identifier, 60 | // version and targetVersion. Body can be nil, which turns this migration 61 | // into a "NilMigration". If no identifier is provided, it will default to "". 62 | // targetVersion can be -1, implying it is a NilVersion. 63 | // 64 | // What is a NilMigration? 65 | // Usually each migration version coming from source is expected to have an 66 | // Up and Down migration. This is not a hard requirement though, leading to 67 | // a situation where only the Up or Down migration is present. So let's say 68 | // the user wants to migrate up to a version that doesn't have the actual Up 69 | // migration, in that case we still want to apply the version, but with an empty 70 | // body. We are calling that a NilMigration, a migration with an empty body. 71 | // 72 | // What is a NilVersion? 73 | // NilVersion is a const(-1). When running down migrations and we are at the 74 | // last down migration, there is no next down migration, the targetVersion should 75 | // be nil. Nil in this case is represented by -1 (because type int). 76 | func NewMigration(body io.ReadCloser, identifier string, 77 | version uint, targetVersion int) (*Migration, error) { 78 | tnow := time.Now() 79 | m := &Migration{ 80 | Identifier: identifier, 81 | Version: version, 82 | TargetVersion: targetVersion, 83 | Scheduled: tnow, 84 | } 85 | 86 | if body == nil { 87 | if len(identifier) == 0 { 88 | m.Identifier = "" 89 | } 90 | 91 | m.StartedBuffering = tnow 92 | m.FinishedBuffering = tnow 93 | m.FinishedReading = tnow 94 | return m, nil 95 | } 96 | 97 | br, bw := io.Pipe() 98 | m.Body = body // want to simulate low latency? newSlowReader(body) 99 | m.BufferSize = DefaultBufferSize 100 | m.BufferedBody = br 101 | m.bufferWriter = bw 102 | return m, nil 103 | } 104 | 105 | // String implements string.Stringer and is used in tests. 106 | func (m *Migration) String() string { 107 | return fmt.Sprintf("%v [%v=>%v]", m.Identifier, m.Version, m.TargetVersion) 108 | } 109 | 110 | // LogString returns a string describing this migration to humans. 111 | func (m *Migration) LogString() string { 112 | directionStr := "u" 113 | if m.TargetVersion < int(m.Version) { 114 | directionStr = "d" 115 | } 116 | return fmt.Sprintf("%v/%v %v", m.Version, directionStr, m.Identifier) 117 | } 118 | 119 | // Buffer buffers Body up to BufferSize. 120 | // Calling this function blocks. Call with goroutine. 121 | func (m *Migration) Buffer() error { 122 | if m.Body == nil { 123 | return nil 124 | } 125 | 126 | m.StartedBuffering = time.Now() 127 | 128 | b := bufio.NewReaderSize(m.Body, int(m.BufferSize)) 129 | 130 | // start reading from body, peek won't move the read pointer though 131 | // poor man's solution? 132 | b.Peek(int(m.BufferSize)) 133 | 134 | m.FinishedBuffering = time.Now() 135 | 136 | // write to bufferWriter, this will block until 137 | // something starts reading from m.Buffer 138 | n, err := b.WriteTo(m.bufferWriter) 139 | if err != nil { 140 | return err 141 | } 142 | 143 | m.FinishedReading = time.Now() 144 | m.BytesRead = n 145 | 146 | // close bufferWriter so Buffer knows that there is no 147 | // more data coming 148 | m.bufferWriter.Close() 149 | 150 | // it's safe to close the Body too 151 | m.Body.Close() 152 | 153 | return nil 154 | } 155 | -------------------------------------------------------------------------------- /migration_test.go: -------------------------------------------------------------------------------- 1 | package migrate 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "log" 7 | "strings" 8 | ) 9 | 10 | func ExampleNewMigration() { 11 | // Create a dummy migration body, this is coming from the source usually. 12 | body := ioutil.NopCloser(strings.NewReader("dumy migration that creates users table")) 13 | 14 | // Create a new Migration that represents version 1486686016. 15 | // Once this migration has been applied to the database, the new 16 | // migration version will be 1486689359. 17 | migr, err := NewMigration(body, "create_users_table", 1486686016, 1486689359) 18 | if err != nil { 19 | log.Fatal(err) 20 | } 21 | 22 | fmt.Print(migr.LogString()) 23 | // Output: 24 | // 1486686016/u create_users_table 25 | } 26 | 27 | func ExampleNewMigration_nilMigration() { 28 | // Create a new Migration that represents a NilMigration. 29 | // Once this migration has been applied to the database, the new 30 | // migration version will be 1486689359. 31 | migr, err := NewMigration(nil, "", 1486686016, 1486689359) 32 | if err != nil { 33 | log.Fatal(err) 34 | } 35 | 36 | fmt.Print(migr.LogString()) 37 | // Output: 38 | // 1486686016/u 39 | } 40 | 41 | func ExampleNewMigration_nilVersion() { 42 | // Create a dummy migration body, this is coming from the source usually. 43 | body := ioutil.NopCloser(strings.NewReader("dumy migration that deletes users table")) 44 | 45 | // Create a new Migration that represents version 1486686016. 46 | // This is the last available down migration, so the migration version 47 | // will be -1, meaning NilVersion once this migration ran. 48 | migr, err := NewMigration(body, "drop_users_table", 1486686016, -1) 49 | if err != nil { 50 | log.Fatal(err) 51 | } 52 | 53 | fmt.Print(migr.LogString()) 54 | // Output: 55 | // 1486686016/d drop_users_table 56 | } 57 | -------------------------------------------------------------------------------- /source/aws-s3/README.md: -------------------------------------------------------------------------------- 1 | # aws-s3 2 | 3 | `s3:///` 4 | -------------------------------------------------------------------------------- /source/aws-s3/s3.go: -------------------------------------------------------------------------------- 1 | package awss3 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net/url" 7 | "os" 8 | "path" 9 | "strings" 10 | 11 | "github.com/aws/aws-sdk-go/aws" 12 | "github.com/aws/aws-sdk-go/aws/session" 13 | "github.com/aws/aws-sdk-go/service/s3" 14 | "github.com/aws/aws-sdk-go/service/s3/s3iface" 15 | "github.com/mattes/migrate/source" 16 | ) 17 | 18 | func init() { 19 | source.Register("s3", &s3Driver{}) 20 | } 21 | 22 | type s3Driver struct { 23 | s3client s3iface.S3API 24 | bucket string 25 | prefix string 26 | migrations *source.Migrations 27 | } 28 | 29 | func (s *s3Driver) Open(folder string) (source.Driver, error) { 30 | u, err := url.Parse(folder) 31 | if err != nil { 32 | return nil, err 33 | } 34 | sess, err := session.NewSession() 35 | if err != nil { 36 | return nil, err 37 | } 38 | driver := s3Driver{ 39 | bucket: u.Host, 40 | prefix: strings.Trim(u.Path, "/") + "/", 41 | s3client: s3.New(sess), 42 | migrations: source.NewMigrations(), 43 | } 44 | err = driver.loadMigrations() 45 | if err != nil { 46 | return nil, err 47 | } 48 | return &driver, nil 49 | } 50 | 51 | func (s *s3Driver) loadMigrations() error { 52 | output, err := s.s3client.ListObjects(&s3.ListObjectsInput{ 53 | Bucket: aws.String(s.bucket), 54 | Prefix: aws.String(s.prefix), 55 | Delimiter: aws.String("/"), 56 | }) 57 | if err != nil { 58 | return err 59 | } 60 | for _, object := range output.Contents { 61 | _, fileName := path.Split(aws.StringValue(object.Key)) 62 | m, err := source.DefaultParse(fileName) 63 | if err != nil { 64 | continue 65 | } 66 | if !s.migrations.Append(m) { 67 | return fmt.Errorf("unable to parse file %v", aws.StringValue(object.Key)) 68 | } 69 | } 70 | return nil 71 | } 72 | 73 | func (s *s3Driver) Close() error { 74 | return nil 75 | } 76 | 77 | func (s *s3Driver) First() (uint, error) { 78 | v, ok := s.migrations.First() 79 | if !ok { 80 | return 0, os.ErrNotExist 81 | } 82 | return v, nil 83 | } 84 | 85 | func (s *s3Driver) Prev(version uint) (uint, error) { 86 | v, ok := s.migrations.Prev(version) 87 | if !ok { 88 | return 0, os.ErrNotExist 89 | } 90 | return v, nil 91 | } 92 | 93 | func (s *s3Driver) Next(version uint) (uint, error) { 94 | v, ok := s.migrations.Next(version) 95 | if !ok { 96 | return 0, os.ErrNotExist 97 | } 98 | return v, nil 99 | } 100 | 101 | func (s *s3Driver) ReadUp(version uint) (io.ReadCloser, string, error) { 102 | if m, ok := s.migrations.Up(version); ok { 103 | return s.open(m) 104 | } 105 | return nil, "", os.ErrNotExist 106 | } 107 | 108 | func (s *s3Driver) ReadDown(version uint) (io.ReadCloser, string, error) { 109 | if m, ok := s.migrations.Down(version); ok { 110 | return s.open(m) 111 | } 112 | return nil, "", os.ErrNotExist 113 | } 114 | 115 | func (s *s3Driver) open(m *source.Migration) (io.ReadCloser, string, error) { 116 | key := path.Join(s.prefix, m.Raw) 117 | object, err := s.s3client.GetObject(&s3.GetObjectInput{ 118 | Bucket: aws.String(s.bucket), 119 | Key: aws.String(key), 120 | }) 121 | if err != nil { 122 | return nil, "", err 123 | } 124 | return object.Body, m.Identifier, nil 125 | } 126 | -------------------------------------------------------------------------------- /source/aws-s3/s3_test.go: -------------------------------------------------------------------------------- 1 | package awss3 2 | 3 | import ( 4 | "errors" 5 | "io/ioutil" 6 | "strings" 7 | "testing" 8 | 9 | "github.com/aws/aws-sdk-go/aws" 10 | "github.com/aws/aws-sdk-go/service/s3" 11 | "github.com/mattes/migrate/source" 12 | st "github.com/mattes/migrate/source/testing" 13 | ) 14 | 15 | func Test(t *testing.T) { 16 | s3Client := fakeS3{ 17 | bucket: "some-bucket", 18 | objects: map[string]string{ 19 | "staging/migrations/1_foobar.up.sql": "1 up", 20 | "staging/migrations/1_foobar.down.sql": "1 down", 21 | "prod/migrations/1_foobar.up.sql": "1 up", 22 | "prod/migrations/1_foobar.down.sql": "1 down", 23 | "prod/migrations/3_foobar.up.sql": "3 up", 24 | "prod/migrations/4_foobar.up.sql": "4 up", 25 | "prod/migrations/4_foobar.down.sql": "4 down", 26 | "prod/migrations/5_foobar.down.sql": "5 down", 27 | "prod/migrations/7_foobar.up.sql": "7 up", 28 | "prod/migrations/7_foobar.down.sql": "7 down", 29 | "prod/migrations/not-a-migration.txt": "", 30 | "prod/migrations/0-random-stuff/whatever.txt": "", 31 | }, 32 | } 33 | driver := s3Driver{ 34 | bucket: "some-bucket", 35 | prefix: "prod/migrations/", 36 | migrations: source.NewMigrations(), 37 | s3client: &s3Client, 38 | } 39 | err := driver.loadMigrations() 40 | if err != nil { 41 | t.Fatal(err) 42 | } 43 | st.Test(t, &driver) 44 | } 45 | 46 | type fakeS3 struct { 47 | s3.S3 48 | bucket string 49 | objects map[string]string 50 | } 51 | 52 | func (s *fakeS3) ListObjects(input *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) { 53 | bucket := aws.StringValue(input.Bucket) 54 | if bucket != s.bucket { 55 | return nil, errors.New("bucket not found") 56 | } 57 | prefix := aws.StringValue(input.Prefix) 58 | delimiter := aws.StringValue(input.Delimiter) 59 | var output s3.ListObjectsOutput 60 | for name := range s.objects { 61 | if strings.HasPrefix(name, prefix) { 62 | if delimiter == "" || !strings.Contains(strings.Replace(name, prefix, "", 1), delimiter) { 63 | output.Contents = append(output.Contents, &s3.Object{ 64 | Key: aws.String(name), 65 | }) 66 | } 67 | } 68 | } 69 | return &output, nil 70 | } 71 | 72 | func (s *fakeS3) GetObject(input *s3.GetObjectInput) (*s3.GetObjectOutput, error) { 73 | bucket := aws.StringValue(input.Bucket) 74 | if bucket != s.bucket { 75 | return nil, errors.New("bucket not found") 76 | } 77 | if data, ok := s.objects[aws.StringValue(input.Key)]; ok { 78 | body := ioutil.NopCloser(strings.NewReader(data)) 79 | return &s3.GetObjectOutput{Body: body}, nil 80 | } 81 | return nil, errors.New("object not found") 82 | } 83 | -------------------------------------------------------------------------------- /source/driver.go: -------------------------------------------------------------------------------- 1 | // Package source provides the Source interface. 2 | // All source drivers must implement this interface, register themselves, 3 | // optionally provide a `WithInstance` function and pass the tests 4 | // in package source/testing. 5 | package source 6 | 7 | import ( 8 | "fmt" 9 | "io" 10 | nurl "net/url" 11 | "sync" 12 | ) 13 | 14 | var driversMu sync.RWMutex 15 | var drivers = make(map[string]Driver) 16 | 17 | // Driver is the interface every source driver must implement. 18 | // 19 | // How to implement a source driver? 20 | // 1. Implement this interface. 21 | // 2. Optionally, add a function named `WithInstance`. 22 | // This function should accept an existing source instance and a Config{} struct 23 | // and return a driver instance. 24 | // 3. Add a test that calls source/testing.go:Test() 25 | // 4. Add own tests for Open(), WithInstance() (when provided) and Close(). 26 | // All other functions are tested by tests in source/testing. 27 | // Saves you some time and makes sure all source drivers behave the same way. 28 | // 5. Call Register in init(). 29 | // 30 | // Guidelines: 31 | // * All configuration input must come from the URL string in func Open() 32 | // or the Config{} struct in WithInstance. Don't os.Getenv(). 33 | // * Drivers are supposed to be read only. 34 | // * Ideally don't load any contents (into memory) in Open or WithInstance. 35 | type Driver interface { 36 | // Open returns a a new driver instance configured with parameters 37 | // coming from the URL string. Migrate will call this function 38 | // only once per instance. 39 | Open(url string) (Driver, error) 40 | 41 | // Close closes the underlying source instance managed by the driver. 42 | // Migrate will call this function only once per instance. 43 | Close() error 44 | 45 | // First returns the very first migration version available to the driver. 46 | // Migrate will call this function multiple times. 47 | // If there is no version available, it must return os.ErrNotExist. 48 | First() (version uint, err error) 49 | 50 | // Prev returns the previous version for a given version available to the driver. 51 | // Migrate will call this function multiple times. 52 | // If there is no previous version available, it must return os.ErrNotExist. 53 | Prev(version uint) (prevVersion uint, err error) 54 | 55 | // Next returns the next version for a given version available to the driver. 56 | // Migrate will call this function multiple times. 57 | // If there is no next version available, it must return os.ErrNotExist. 58 | Next(version uint) (nextVersion uint, err error) 59 | 60 | // ReadUp returns the UP migration body and an identifier that helps 61 | // finding this migration in the source for a given version. 62 | // If there is no up migration available for this version, 63 | // it must return os.ErrNotExist. 64 | // Do not start reading, just return the ReadCloser! 65 | ReadUp(version uint) (r io.ReadCloser, identifier string, err error) 66 | 67 | // ReadDown returns the DOWN migration body and an identifier that helps 68 | // finding this migration in the source for a given version. 69 | // If there is no down migration available for this version, 70 | // it must return os.ErrNotExist. 71 | // Do not start reading, just return the ReadCloser! 72 | ReadDown(version uint) (r io.ReadCloser, identifier string, err error) 73 | } 74 | 75 | // Open returns a new driver instance. 76 | func Open(url string) (Driver, error) { 77 | u, err := nurl.Parse(url) 78 | if err != nil { 79 | return nil, err 80 | } 81 | 82 | if u.Scheme == "" { 83 | return nil, fmt.Errorf("source driver: invalid URL scheme") 84 | } 85 | 86 | driversMu.RLock() 87 | d, ok := drivers[u.Scheme] 88 | driversMu.RUnlock() 89 | if !ok { 90 | return nil, fmt.Errorf("source driver: unknown driver %v (forgotten import?)", u.Scheme) 91 | } 92 | 93 | return d.Open(url) 94 | } 95 | 96 | // Register globally registers a driver. 97 | func Register(name string, driver Driver) { 98 | driversMu.Lock() 99 | defer driversMu.Unlock() 100 | if driver == nil { 101 | panic("Register driver is nil") 102 | } 103 | if _, dup := drivers[name]; dup { 104 | panic("Register called twice for driver " + name) 105 | } 106 | drivers[name] = driver 107 | } 108 | -------------------------------------------------------------------------------- /source/driver_test.go: -------------------------------------------------------------------------------- 1 | package source 2 | 3 | func ExampleDriver() { 4 | // see source/stub for an example 5 | 6 | // source/stub/stub.go has the driver implementation 7 | // source/stub/stub_test.go runs source/testing/test.go:Test 8 | } 9 | -------------------------------------------------------------------------------- /source/file/README.md: -------------------------------------------------------------------------------- 1 | # file 2 | 3 | `file:///absolute/path` 4 | `file://relative/path` 5 | -------------------------------------------------------------------------------- /source/file/file.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "io/ioutil" 7 | nurl "net/url" 8 | "os" 9 | "path" 10 | "path/filepath" 11 | 12 | "github.com/mattes/migrate/source" 13 | ) 14 | 15 | func init() { 16 | source.Register("file", &File{}) 17 | } 18 | 19 | type File struct { 20 | url string 21 | path string 22 | migrations *source.Migrations 23 | } 24 | 25 | func (f *File) Open(url string) (source.Driver, error) { 26 | u, err := nurl.Parse(url) 27 | if err != nil { 28 | return nil, err 29 | } 30 | 31 | // concat host and path to restore full path 32 | // host might be `.` 33 | p := u.Host + u.Path 34 | 35 | if len(p) == 0 { 36 | // default to current directory if no path 37 | wd, err := os.Getwd() 38 | if err != nil { 39 | return nil, err 40 | } 41 | p = wd 42 | 43 | } else if p[0:1] == "." || p[0:1] != "/" { 44 | // make path absolute if relative 45 | abs, err := filepath.Abs(p) 46 | if err != nil { 47 | return nil, err 48 | } 49 | p = abs 50 | } 51 | 52 | // scan directory 53 | files, err := ioutil.ReadDir(p) 54 | if err != nil { 55 | return nil, err 56 | } 57 | 58 | nf := &File{ 59 | url: url, 60 | path: p, 61 | migrations: source.NewMigrations(), 62 | } 63 | 64 | for _, fi := range files { 65 | if !fi.IsDir() { 66 | m, err := source.DefaultParse(fi.Name()) 67 | if err != nil { 68 | continue // ignore files that we can't parse 69 | } 70 | if !nf.migrations.Append(m) { 71 | return nil, fmt.Errorf("unable to parse file %v", fi.Name()) 72 | } 73 | } 74 | } 75 | return nf, nil 76 | } 77 | 78 | func (f *File) Close() error { 79 | // nothing do to here 80 | return nil 81 | } 82 | 83 | func (f *File) First() (version uint, err error) { 84 | if v, ok := f.migrations.First(); !ok { 85 | return 0, &os.PathError{"first", f.path, os.ErrNotExist} 86 | } else { 87 | return v, nil 88 | } 89 | } 90 | 91 | func (f *File) Prev(version uint) (prevVersion uint, err error) { 92 | if v, ok := f.migrations.Prev(version); !ok { 93 | return 0, &os.PathError{fmt.Sprintf("prev for version %v", version), f.path, os.ErrNotExist} 94 | } else { 95 | return v, nil 96 | } 97 | } 98 | 99 | func (f *File) Next(version uint) (nextVersion uint, err error) { 100 | if v, ok := f.migrations.Next(version); !ok { 101 | return 0, &os.PathError{fmt.Sprintf("next for version %v", version), f.path, os.ErrNotExist} 102 | } else { 103 | return v, nil 104 | } 105 | } 106 | 107 | func (f *File) ReadUp(version uint) (r io.ReadCloser, identifier string, err error) { 108 | if m, ok := f.migrations.Up(version); ok { 109 | r, err := os.Open(path.Join(f.path, m.Raw)) 110 | if err != nil { 111 | return nil, "", err 112 | } 113 | return r, m.Identifier, nil 114 | } 115 | return nil, "", &os.PathError{fmt.Sprintf("read version %v", version), f.path, os.ErrNotExist} 116 | } 117 | 118 | func (f *File) ReadDown(version uint) (r io.ReadCloser, identifier string, err error) { 119 | if m, ok := f.migrations.Down(version); ok { 120 | r, err := os.Open(path.Join(f.path, m.Raw)) 121 | if err != nil { 122 | return nil, "", err 123 | } 124 | return r, m.Identifier, nil 125 | } 126 | return nil, "", &os.PathError{fmt.Sprintf("read version %v", version), f.path, os.ErrNotExist} 127 | } 128 | -------------------------------------------------------------------------------- /source/file/file_test.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "path" 8 | "path/filepath" 9 | "testing" 10 | 11 | st "github.com/mattes/migrate/source/testing" 12 | ) 13 | 14 | func Test(t *testing.T) { 15 | tmpDir, err := ioutil.TempDir("", "") 16 | if err != nil { 17 | t.Fatal(err) 18 | } 19 | defer os.RemoveAll(tmpDir) 20 | 21 | // write files that meet driver test requirements 22 | mustWriteFile(t, tmpDir, "1_foobar.up.sql", "1 up") 23 | mustWriteFile(t, tmpDir, "1_foobar.down.sql", "1 down") 24 | 25 | mustWriteFile(t, tmpDir, "3_foobar.up.sql", "3 up") 26 | 27 | mustWriteFile(t, tmpDir, "4_foobar.up.sql", "4 up") 28 | mustWriteFile(t, tmpDir, "4_foobar.down.sql", "4 down") 29 | 30 | mustWriteFile(t, tmpDir, "5_foobar.down.sql", "5 down") 31 | 32 | mustWriteFile(t, tmpDir, "7_foobar.up.sql", "7 up") 33 | mustWriteFile(t, tmpDir, "7_foobar.down.sql", "7 down") 34 | 35 | f := &File{} 36 | d, err := f.Open("file://" + tmpDir) 37 | if err != nil { 38 | t.Fatal(err) 39 | } 40 | 41 | st.Test(t, d) 42 | } 43 | 44 | func TestOpen(t *testing.T) { 45 | tmpDir, err := ioutil.TempDir("", "TestOpen") 46 | if err != nil { 47 | t.Fatal(err) 48 | } 49 | defer os.RemoveAll(tmpDir) 50 | 51 | mustWriteFile(t, tmpDir, "1_foobar.up.sql", "") 52 | mustWriteFile(t, tmpDir, "1_foobar.down.sql", "") 53 | 54 | if !filepath.IsAbs(tmpDir) { 55 | t.Fatal("expected tmpDir to be absolute path") 56 | } 57 | 58 | f := &File{} 59 | _, err = f.Open("file://" + tmpDir) // absolute path 60 | if err != nil { 61 | t.Fatal(err) 62 | } 63 | } 64 | 65 | func TestOpenWithRelativePath(t *testing.T) { 66 | tmpDir, err := ioutil.TempDir("", "TestOpen") 67 | if err != nil { 68 | t.Fatal(err) 69 | } 70 | defer os.RemoveAll(tmpDir) 71 | 72 | wd, err := os.Getwd() 73 | if err != nil { 74 | t.Fatal(err) 75 | } 76 | defer os.Chdir(wd) // rescue working dir after we are done 77 | 78 | if err := os.Chdir(tmpDir); err != nil { 79 | t.Fatal(err) 80 | } 81 | 82 | if err := os.Mkdir(filepath.Join(tmpDir, "foo"), os.ModePerm); err != nil { 83 | t.Fatal(err) 84 | } 85 | 86 | mustWriteFile(t, filepath.Join(tmpDir, "foo"), "1_foobar.up.sql", "") 87 | 88 | f := &File{} 89 | 90 | // dir: foo 91 | d, err := f.Open("file://foo") 92 | if err != nil { 93 | t.Fatal(err) 94 | } 95 | _, err = d.First() 96 | if err != nil { 97 | t.Fatalf("expected first file in working dir %v for foo", tmpDir) 98 | } 99 | 100 | // dir: ./foo 101 | d, err = f.Open("file://./foo") 102 | if err != nil { 103 | t.Fatal(err) 104 | } 105 | _, err = d.First() 106 | if err != nil { 107 | t.Fatalf("expected first file in working dir %v for ./foo", tmpDir) 108 | } 109 | } 110 | 111 | func TestOpenDefaultsToCurrentDirectory(t *testing.T) { 112 | wd, err := os.Getwd() 113 | if err != nil { 114 | t.Fatal(err) 115 | } 116 | 117 | f := &File{} 118 | d, err := f.Open("file://") 119 | if err != nil { 120 | t.Fatal(err) 121 | } 122 | 123 | if d.(*File).path != wd { 124 | t.Fatal("expected driver to default to current directory") 125 | } 126 | } 127 | 128 | func TestOpenWithDuplicateVersion(t *testing.T) { 129 | tmpDir, err := ioutil.TempDir("", "TestOpenWithDuplicateVersion") 130 | if err != nil { 131 | t.Fatal(err) 132 | } 133 | defer os.RemoveAll(tmpDir) 134 | 135 | mustWriteFile(t, tmpDir, "1_foo.up.sql", "") // 1 up 136 | mustWriteFile(t, tmpDir, "1_bar.up.sql", "") // 1 up 137 | 138 | f := &File{} 139 | _, err = f.Open("file://" + tmpDir) 140 | if err == nil { 141 | t.Fatal("expected err") 142 | } 143 | } 144 | 145 | func TestClose(t *testing.T) { 146 | tmpDir, err := ioutil.TempDir("", "TestOpen") 147 | if err != nil { 148 | t.Fatal(err) 149 | } 150 | defer os.RemoveAll(tmpDir) 151 | 152 | f := &File{} 153 | d, err := f.Open("file://" + tmpDir) 154 | if err != nil { 155 | t.Fatal(err) 156 | } 157 | 158 | if d.Close() != nil { 159 | t.Fatal("expected nil") 160 | } 161 | } 162 | 163 | func mustWriteFile(t testing.TB, dir, file string, body string) { 164 | if err := ioutil.WriteFile(path.Join(dir, file), []byte(body), 06444); err != nil { 165 | t.Fatal(err) 166 | } 167 | } 168 | 169 | func mustCreateBenchmarkDir(t *testing.B) (dir string) { 170 | tmpDir, err := ioutil.TempDir("", "Benchmark") 171 | if err != nil { 172 | t.Fatal(err) 173 | } 174 | 175 | for i := 0; i < 1000; i++ { 176 | mustWriteFile(t, tmpDir, fmt.Sprintf("%v_foobar.up.sql", i), "") 177 | mustWriteFile(t, tmpDir, fmt.Sprintf("%v_foobar.down.sql", i), "") 178 | } 179 | 180 | return tmpDir 181 | } 182 | 183 | func BenchmarkOpen(b *testing.B) { 184 | dir := mustCreateBenchmarkDir(b) 185 | defer os.RemoveAll(dir) 186 | b.ResetTimer() 187 | for n := 0; n < b.N; n++ { 188 | f := &File{} 189 | f.Open("file://" + dir) 190 | } 191 | b.StopTimer() 192 | } 193 | 194 | func BenchmarkNext(b *testing.B) { 195 | dir := mustCreateBenchmarkDir(b) 196 | defer os.RemoveAll(dir) 197 | f := &File{} 198 | d, _ := f.Open("file://" + dir) 199 | b.ResetTimer() 200 | v, err := d.First() 201 | for n := 0; n < b.N; n++ { 202 | for !os.IsNotExist(err) { 203 | v, err = d.Next(v) 204 | } 205 | } 206 | b.StopTimer() 207 | } 208 | -------------------------------------------------------------------------------- /source/github/.gitignore: -------------------------------------------------------------------------------- 1 | .github_test_secrets 2 | -------------------------------------------------------------------------------- /source/github/README.md: -------------------------------------------------------------------------------- 1 | # github 2 | 3 | `github://user:personal-access-token@owner/repo/path` 4 | 5 | | URL Query | WithInstance Config | Description | 6 | |------------|---------------------|-------------| 7 | | user | | The username of the user connecting | 8 | | personal-access-token | | An access token from Github (https://github.com/settings/tokens) | 9 | | owner | | the repo owner | 10 | | repo | | the name of the repository | 11 | | path | | path in repo to migrations | 12 | -------------------------------------------------------------------------------- /source/github/examples/migrations/1085649617_create_users_table.down.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS users; 2 | -------------------------------------------------------------------------------- /source/github/examples/migrations/1085649617_create_users_table.up.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE users ( 2 | user_id integer unique, 3 | name varchar(40), 4 | email varchar(40) 5 | ); 6 | -------------------------------------------------------------------------------- /source/github/examples/migrations/1185749658_add_city_to_users.down.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE users DROP COLUMN IF EXISTS city; 2 | -------------------------------------------------------------------------------- /source/github/examples/migrations/1185749658_add_city_to_users.up.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE users ADD COLUMN city varchar(100); 2 | 3 | 4 | -------------------------------------------------------------------------------- /source/github/examples/migrations/1285849751_add_index_on_user_emails.down.sql: -------------------------------------------------------------------------------- 1 | DROP INDEX IF EXISTS users_email_index; 2 | -------------------------------------------------------------------------------- /source/github/examples/migrations/1285849751_add_index_on_user_emails.up.sql: -------------------------------------------------------------------------------- 1 | CREATE UNIQUE INDEX CONCURRENTLY users_email_index ON users (email); 2 | 3 | -- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. 4 | -------------------------------------------------------------------------------- /source/github/examples/migrations/1385949617_create_books_table.down.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS books; 2 | -------------------------------------------------------------------------------- /source/github/examples/migrations/1385949617_create_books_table.up.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE books ( 2 | user_id integer, 3 | name varchar(40), 4 | author varchar(40) 5 | ); 6 | -------------------------------------------------------------------------------- /source/github/examples/migrations/1485949617_create_movies_table.down.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS movies; 2 | -------------------------------------------------------------------------------- /source/github/examples/migrations/1485949617_create_movies_table.up.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE movies ( 2 | user_id integer, 3 | name varchar(40), 4 | director varchar(40) 5 | ); 6 | -------------------------------------------------------------------------------- /source/github/examples/migrations/1585849751_just_a_comment.up.sql: -------------------------------------------------------------------------------- 1 | -- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. 2 | -------------------------------------------------------------------------------- /source/github/examples/migrations/1685849751_another_comment.up.sql: -------------------------------------------------------------------------------- 1 | -- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. 2 | -------------------------------------------------------------------------------- /source/github/examples/migrations/1785849751_another_comment.up.sql: -------------------------------------------------------------------------------- 1 | -- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. 2 | -------------------------------------------------------------------------------- /source/github/examples/migrations/1885849751_another_comment.up.sql: -------------------------------------------------------------------------------- 1 | -- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. 2 | -------------------------------------------------------------------------------- /source/github/github.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "io" 8 | "io/ioutil" 9 | nurl "net/url" 10 | "os" 11 | "path" 12 | "strings" 13 | 14 | "github.com/google/go-github/github" 15 | "github.com/mattes/migrate/source" 16 | ) 17 | 18 | func init() { 19 | source.Register("github", &Github{}) 20 | } 21 | 22 | var ( 23 | ErrNoUserInfo = fmt.Errorf("no username:token provided") 24 | ErrNoAccessToken = fmt.Errorf("no access token") 25 | ErrInvalidRepo = fmt.Errorf("invalid repo") 26 | ErrInvalidGithubClient = fmt.Errorf("expected *github.Client") 27 | ErrNoDir = fmt.Errorf("no directory") 28 | ) 29 | 30 | type Github struct { 31 | client *github.Client 32 | url string 33 | 34 | pathOwner string 35 | pathRepo string 36 | path string 37 | migrations *source.Migrations 38 | } 39 | 40 | type Config struct { 41 | } 42 | 43 | func (g *Github) Open(url string) (source.Driver, error) { 44 | u, err := nurl.Parse(url) 45 | if err != nil { 46 | return nil, err 47 | } 48 | 49 | if u.User == nil { 50 | return nil, ErrNoUserInfo 51 | } 52 | 53 | password, ok := u.User.Password() 54 | if !ok { 55 | return nil, ErrNoUserInfo 56 | } 57 | 58 | tr := &github.BasicAuthTransport{ 59 | Username: u.User.Username(), 60 | Password: password, 61 | } 62 | 63 | gn := &Github{ 64 | client: github.NewClient(tr.Client()), 65 | url: url, 66 | migrations: source.NewMigrations(), 67 | } 68 | 69 | // set owner, repo and path in repo 70 | gn.pathOwner = u.Host 71 | pe := strings.Split(strings.Trim(u.Path, "/"), "/") 72 | if len(pe) < 1 { 73 | return nil, ErrInvalidRepo 74 | } 75 | gn.pathRepo = pe[0] 76 | if len(pe) > 1 { 77 | gn.path = strings.Join(pe[1:], "/") 78 | } 79 | 80 | if err := gn.readDirectory(); err != nil { 81 | return nil, err 82 | } 83 | 84 | return gn, nil 85 | } 86 | 87 | func WithInstance(client *github.Client, config *Config) (source.Driver, error) { 88 | gn := &Github{ 89 | client: client, 90 | migrations: source.NewMigrations(), 91 | } 92 | if err := gn.readDirectory(); err != nil { 93 | return nil, err 94 | } 95 | return gn, nil 96 | } 97 | 98 | func (g *Github) readDirectory() error { 99 | fileContent, dirContents, _, err := g.client.Repositories.GetContents(context.Background(), g.pathOwner, g.pathRepo, g.path, &github.RepositoryContentGetOptions{}) 100 | if err != nil { 101 | return err 102 | } 103 | if fileContent != nil { 104 | return ErrNoDir 105 | } 106 | 107 | for _, fi := range dirContents { 108 | m, err := source.DefaultParse(*fi.Name) 109 | if err != nil { 110 | continue // ignore files that we can't parse 111 | } 112 | if !g.migrations.Append(m) { 113 | return fmt.Errorf("unable to parse file %v", *fi.Name) 114 | } 115 | } 116 | 117 | return nil 118 | } 119 | 120 | func (g *Github) Close() error { 121 | return nil 122 | } 123 | 124 | func (g *Github) First() (version uint, er error) { 125 | if v, ok := g.migrations.First(); !ok { 126 | return 0, &os.PathError{"first", g.path, os.ErrNotExist} 127 | } else { 128 | return v, nil 129 | } 130 | } 131 | 132 | func (g *Github) Prev(version uint) (prevVersion uint, err error) { 133 | if v, ok := g.migrations.Prev(version); !ok { 134 | return 0, &os.PathError{fmt.Sprintf("prev for version %v", version), g.path, os.ErrNotExist} 135 | } else { 136 | return v, nil 137 | } 138 | } 139 | 140 | func (g *Github) Next(version uint) (nextVersion uint, err error) { 141 | if v, ok := g.migrations.Next(version); !ok { 142 | return 0, &os.PathError{fmt.Sprintf("next for version %v", version), g.path, os.ErrNotExist} 143 | } else { 144 | return v, nil 145 | } 146 | } 147 | 148 | func (g *Github) ReadUp(version uint) (r io.ReadCloser, identifier string, err error) { 149 | if m, ok := g.migrations.Up(version); ok { 150 | file, _, _, err := g.client.Repositories.GetContents(context.Background(), g.pathOwner, g.pathRepo, path.Join(g.path, m.Raw), &github.RepositoryContentGetOptions{}) 151 | if err != nil { 152 | return nil, "", err 153 | } 154 | if file != nil { 155 | r, err := file.GetContent() 156 | if err != nil { 157 | return nil, "", err 158 | } 159 | return ioutil.NopCloser(bytes.NewReader([]byte(r))), m.Identifier, nil 160 | } 161 | } 162 | return nil, "", &os.PathError{fmt.Sprintf("read version %v", version), g.path, os.ErrNotExist} 163 | } 164 | 165 | func (g *Github) ReadDown(version uint) (r io.ReadCloser, identifier string, err error) { 166 | if m, ok := g.migrations.Down(version); ok { 167 | file, _, _, err := g.client.Repositories.GetContents(context.Background(), g.pathOwner, g.pathRepo, path.Join(g.path, m.Raw), &github.RepositoryContentGetOptions{}) 168 | if err != nil { 169 | return nil, "", err 170 | } 171 | if file != nil { 172 | r, err := file.GetContent() 173 | if err != nil { 174 | return nil, "", err 175 | } 176 | return ioutil.NopCloser(bytes.NewReader([]byte(r))), m.Identifier, nil 177 | } 178 | } 179 | return nil, "", &os.PathError{fmt.Sprintf("read version %v", version), g.path, os.ErrNotExist} 180 | } 181 | -------------------------------------------------------------------------------- /source/github/github_test.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | "bytes" 5 | "io/ioutil" 6 | "testing" 7 | 8 | st "github.com/mattes/migrate/source/testing" 9 | ) 10 | 11 | var GithubTestSecret = "" // username:token 12 | 13 | func init() { 14 | secrets, err := ioutil.ReadFile(".github_test_secrets") 15 | if err == nil { 16 | GithubTestSecret = string(bytes.TrimSpace(secrets)[:]) 17 | } 18 | } 19 | 20 | func Test(t *testing.T) { 21 | if len(GithubTestSecret) == 0 { 22 | t.Skip("test requires .github_test_secrets") 23 | } 24 | 25 | g := &Github{} 26 | d, err := g.Open("github://" + GithubTestSecret + "@mattes/migrate_test_tmp/test") 27 | if err != nil { 28 | t.Fatal(err) 29 | } 30 | 31 | st.Test(t, d) 32 | } 33 | -------------------------------------------------------------------------------- /source/go-bindata/README.md: -------------------------------------------------------------------------------- 1 | # go-bindata 2 | 3 | ## Usage 4 | 5 | 6 | 7 | ### Read bindata with NewWithSourceInstance 8 | 9 | ```shell 10 | go get -u github.com/jteeuwen/go-bindata/... 11 | cd examples/migrations && go-bindata -pkg migrations . 12 | ``` 13 | 14 | ```go 15 | import ( 16 | "github.com/mattes/migrate" 17 | "github.com/mattes/migrate/source/go-bindata" 18 | "github.com/mattes/migrate/source/go-bindata/examples/migrations" 19 | ) 20 | 21 | func main() { 22 | // wrap assets into Resource 23 | s := bindata.Resource(migrations.AssetNames(), 24 | func(name string) ([]byte, error) { 25 | return migrations.Asset(name) 26 | }) 27 | 28 | d, err := bindata.WithInstance(s) 29 | m, err := migrate.NewWithSourceInstance("go-bindata", d, "database://foobar") 30 | m.Up() // run your migrations and handle the errors above of course 31 | } 32 | ``` 33 | 34 | ### Read bindata with URL (todo) 35 | 36 | This will restore the assets in a tmp directory and then 37 | proxy to source/file. go-bindata must be in your `$PATH`. 38 | 39 | ``` 40 | migrate -source go-bindata://examples/migrations/bindata.go 41 | ``` 42 | 43 | 44 | -------------------------------------------------------------------------------- /source/go-bindata/go-bindata.go: -------------------------------------------------------------------------------- 1 | package bindata 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "os" 9 | 10 | "github.com/mattes/migrate/source" 11 | ) 12 | 13 | type AssetFunc func(name string) ([]byte, error) 14 | 15 | func Resource(names []string, afn AssetFunc) *AssetSource { 16 | return &AssetSource{ 17 | Names: names, 18 | AssetFunc: afn, 19 | } 20 | } 21 | 22 | type AssetSource struct { 23 | Names []string 24 | AssetFunc AssetFunc 25 | } 26 | 27 | func init() { 28 | source.Register("go-bindata", &Bindata{}) 29 | } 30 | 31 | type Bindata struct { 32 | path string 33 | assetSource *AssetSource 34 | migrations *source.Migrations 35 | } 36 | 37 | func (b *Bindata) Open(url string) (source.Driver, error) { 38 | return nil, fmt.Errorf("not yet implemented") 39 | } 40 | 41 | var ( 42 | ErrNoAssetSource = fmt.Errorf("expects *AssetSource") 43 | ) 44 | 45 | func WithInstance(instance interface{}) (source.Driver, error) { 46 | if _, ok := instance.(*AssetSource); !ok { 47 | return nil, ErrNoAssetSource 48 | } 49 | as := instance.(*AssetSource) 50 | 51 | bn := &Bindata{ 52 | path: "", 53 | assetSource: as, 54 | migrations: source.NewMigrations(), 55 | } 56 | 57 | for _, fi := range as.Names { 58 | m, err := source.DefaultParse(fi) 59 | if err != nil { 60 | continue // ignore files that we can't parse 61 | } 62 | 63 | if !bn.migrations.Append(m) { 64 | return nil, fmt.Errorf("unable to parse file %v", fi) 65 | } 66 | } 67 | 68 | return bn, nil 69 | } 70 | 71 | func (b *Bindata) Close() error { 72 | return nil 73 | } 74 | 75 | func (b *Bindata) First() (version uint, err error) { 76 | if v, ok := b.migrations.First(); !ok { 77 | return 0, &os.PathError{"first", b.path, os.ErrNotExist} 78 | } else { 79 | return v, nil 80 | } 81 | } 82 | 83 | func (b *Bindata) Prev(version uint) (prevVersion uint, err error) { 84 | if v, ok := b.migrations.Prev(version); !ok { 85 | return 0, &os.PathError{fmt.Sprintf("prev for version %v", version), b.path, os.ErrNotExist} 86 | } else { 87 | return v, nil 88 | } 89 | } 90 | 91 | func (b *Bindata) Next(version uint) (nextVersion uint, err error) { 92 | if v, ok := b.migrations.Next(version); !ok { 93 | return 0, &os.PathError{fmt.Sprintf("next for version %v", version), b.path, os.ErrNotExist} 94 | } else { 95 | return v, nil 96 | } 97 | } 98 | 99 | func (b *Bindata) ReadUp(version uint) (r io.ReadCloser, identifier string, err error) { 100 | if m, ok := b.migrations.Up(version); ok { 101 | body, err := b.assetSource.AssetFunc(m.Raw) 102 | if err != nil { 103 | return nil, "", err 104 | } 105 | return ioutil.NopCloser(bytes.NewReader(body)), m.Identifier, nil 106 | } 107 | return nil, "", &os.PathError{fmt.Sprintf("read version %v", version), b.path, os.ErrNotExist} 108 | } 109 | 110 | func (b *Bindata) ReadDown(version uint) (r io.ReadCloser, identifier string, err error) { 111 | if m, ok := b.migrations.Down(version); ok { 112 | body, err := b.assetSource.AssetFunc(m.Raw) 113 | if err != nil { 114 | return nil, "", err 115 | } 116 | return ioutil.NopCloser(bytes.NewReader(body)), m.Identifier, nil 117 | } 118 | return nil, "", &os.PathError{fmt.Sprintf("read version %v", version), b.path, os.ErrNotExist} 119 | } 120 | -------------------------------------------------------------------------------- /source/go-bindata/go-bindata_test.go: -------------------------------------------------------------------------------- 1 | package bindata 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/mattes/migrate/source/go-bindata/testdata" 7 | st "github.com/mattes/migrate/source/testing" 8 | ) 9 | 10 | func Test(t *testing.T) { 11 | // wrap assets into Resource first 12 | s := Resource(testdata.AssetNames(), 13 | func(name string) ([]byte, error) { 14 | return testdata.Asset(name) 15 | }) 16 | 17 | d, err := WithInstance(s) 18 | if err != nil { 19 | t.Fatal(err) 20 | } 21 | st.Test(t, d) 22 | } 23 | 24 | func TestWithInstance(t *testing.T) { 25 | // wrap assets into Resource 26 | s := Resource(testdata.AssetNames(), 27 | func(name string) ([]byte, error) { 28 | return testdata.Asset(name) 29 | }) 30 | 31 | _, err := WithInstance(s) 32 | if err != nil { 33 | t.Fatal(err) 34 | } 35 | } 36 | 37 | func TestOpen(t *testing.T) { 38 | b := &Bindata{} 39 | _, err := b.Open("") 40 | if err == nil { 41 | t.Fatal("expected err, because it's not implemented yet") 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /source/google-cloud-storage/README.md: -------------------------------------------------------------------------------- 1 | # google-cloud-storage 2 | 3 | `gcs:///` 4 | -------------------------------------------------------------------------------- /source/google-cloud-storage/storage.go: -------------------------------------------------------------------------------- 1 | package googlecloudstorage 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net/url" 7 | "os" 8 | "path" 9 | "strings" 10 | 11 | "cloud.google.com/go/storage" 12 | "github.com/mattes/migrate/source" 13 | "golang.org/x/net/context" 14 | "google.golang.org/api/iterator" 15 | ) 16 | 17 | func init() { 18 | source.Register("gcs", &gcs{}) 19 | } 20 | 21 | type gcs struct { 22 | bucket *storage.BucketHandle 23 | prefix string 24 | migrations *source.Migrations 25 | } 26 | 27 | func (g *gcs) Open(folder string) (source.Driver, error) { 28 | u, err := url.Parse(folder) 29 | if err != nil { 30 | return nil, err 31 | } 32 | client, err := storage.NewClient(context.Background()) 33 | if err != nil { 34 | return nil, err 35 | } 36 | driver := gcs{ 37 | bucket: client.Bucket(u.Host), 38 | prefix: strings.Trim(u.Path, "/") + "/", 39 | migrations: source.NewMigrations(), 40 | } 41 | err = driver.loadMigrations() 42 | if err != nil { 43 | return nil, err 44 | } 45 | return &driver, nil 46 | } 47 | 48 | func (g *gcs) loadMigrations() error { 49 | iter := g.bucket.Objects(context.Background(), &storage.Query{ 50 | Prefix: g.prefix, 51 | Delimiter: "/", 52 | }) 53 | object, err := iter.Next() 54 | for ; err == nil; object, err = iter.Next() { 55 | _, fileName := path.Split(object.Name) 56 | m, parseErr := source.DefaultParse(fileName) 57 | if parseErr != nil { 58 | continue 59 | } 60 | if !g.migrations.Append(m) { 61 | return fmt.Errorf("unable to parse file %v", object.Name) 62 | } 63 | } 64 | if err != iterator.Done { 65 | return err 66 | } 67 | return nil 68 | } 69 | 70 | func (g *gcs) Close() error { 71 | return nil 72 | } 73 | 74 | func (g *gcs) First() (uint, error) { 75 | v, ok := g.migrations.First() 76 | if !ok { 77 | return 0, os.ErrNotExist 78 | } 79 | return v, nil 80 | } 81 | 82 | func (g *gcs) Prev(version uint) (uint, error) { 83 | v, ok := g.migrations.Prev(version) 84 | if !ok { 85 | return 0, os.ErrNotExist 86 | } 87 | return v, nil 88 | } 89 | 90 | func (g *gcs) Next(version uint) (uint, error) { 91 | v, ok := g.migrations.Next(version) 92 | if !ok { 93 | return 0, os.ErrNotExist 94 | } 95 | return v, nil 96 | } 97 | 98 | func (g *gcs) ReadUp(version uint) (io.ReadCloser, string, error) { 99 | if m, ok := g.migrations.Up(version); ok { 100 | return g.open(m) 101 | } 102 | return nil, "", os.ErrNotExist 103 | } 104 | 105 | func (g *gcs) ReadDown(version uint) (io.ReadCloser, string, error) { 106 | if m, ok := g.migrations.Down(version); ok { 107 | return g.open(m) 108 | } 109 | return nil, "", os.ErrNotExist 110 | } 111 | 112 | func (g *gcs) open(m *source.Migration) (io.ReadCloser, string, error) { 113 | objectPath := path.Join(g.prefix, m.Raw) 114 | reader, err := g.bucket.Object(objectPath).NewReader(context.Background()) 115 | if err != nil { 116 | return nil, "", err 117 | } 118 | return reader, m.Identifier, nil 119 | } 120 | -------------------------------------------------------------------------------- /source/google-cloud-storage/storage_test.go: -------------------------------------------------------------------------------- 1 | package googlecloudstorage 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/fsouza/fake-gcs-server/fakestorage" 7 | "github.com/mattes/migrate/source" 8 | st "github.com/mattes/migrate/source/testing" 9 | ) 10 | 11 | func Test(t *testing.T) { 12 | server := fakestorage.NewServer([]fakestorage.Object{ 13 | {BucketName: "some-bucket", Name: "staging/migrations/1_foobar.up.sql", Content: []byte("1 up")}, 14 | {BucketName: "some-bucket", Name: "staging/migrations/1_foobar.down.sql", Content: []byte("1 down")}, 15 | {BucketName: "some-bucket", Name: "prod/migrations/1_foobar.up.sql", Content: []byte("1 up")}, 16 | {BucketName: "some-bucket", Name: "prod/migrations/1_foobar.down.sql", Content: []byte("1 down")}, 17 | {BucketName: "some-bucket", Name: "prod/migrations/3_foobar.up.sql", Content: []byte("3 up")}, 18 | {BucketName: "some-bucket", Name: "prod/migrations/4_foobar.up.sql", Content: []byte("4 up")}, 19 | {BucketName: "some-bucket", Name: "prod/migrations/4_foobar.down.sql", Content: []byte("4 down")}, 20 | {BucketName: "some-bucket", Name: "prod/migrations/5_foobar.down.sql", Content: []byte("5 down")}, 21 | {BucketName: "some-bucket", Name: "prod/migrations/7_foobar.up.sql", Content: []byte("7 up")}, 22 | {BucketName: "some-bucket", Name: "prod/migrations/7_foobar.down.sql", Content: []byte("7 down")}, 23 | {BucketName: "some-bucket", Name: "prod/migrations/not-a-migration.txt"}, 24 | {BucketName: "some-bucket", Name: "prod/migrations/0-random-stuff/whatever.txt"}, 25 | }) 26 | defer server.Stop() 27 | driver := gcs{ 28 | bucket: server.Client().Bucket("some-bucket"), 29 | prefix: "prod/migrations/", 30 | migrations: source.NewMigrations(), 31 | } 32 | err := driver.loadMigrations() 33 | if err != nil { 34 | t.Fatal(err) 35 | } 36 | st.Test(t, &driver) 37 | } 38 | -------------------------------------------------------------------------------- /source/migration.go: -------------------------------------------------------------------------------- 1 | package source 2 | 3 | import ( 4 | "sort" 5 | ) 6 | 7 | // Direction is either up or down. 8 | type Direction string 9 | 10 | const ( 11 | Down Direction = "down" 12 | Up = "up" 13 | ) 14 | 15 | // Migration is a helper struct for source drivers that need to 16 | // build the full directory tree in memory. 17 | // Migration is fully independent from migrate.Migration. 18 | type Migration struct { 19 | // Version is the version of this migration. 20 | Version uint 21 | 22 | // Identifier can be any string that helps identifying 23 | // this migration in the source. 24 | Identifier string 25 | 26 | // Direction is either Up or Down. 27 | Direction Direction 28 | 29 | // Raw holds the raw location path to this migration in source. 30 | // ReadUp and ReadDown will use this. 31 | Raw string 32 | } 33 | 34 | // Migrations wraps Migration and has an internal index 35 | // to keep track of Migration order. 36 | type Migrations struct { 37 | index uintSlice 38 | migrations map[uint]map[Direction]*Migration 39 | } 40 | 41 | func NewMigrations() *Migrations { 42 | return &Migrations{ 43 | index: make(uintSlice, 0), 44 | migrations: make(map[uint]map[Direction]*Migration), 45 | } 46 | } 47 | 48 | func (i *Migrations) Append(m *Migration) (ok bool) { 49 | if m == nil { 50 | return false 51 | } 52 | 53 | if i.migrations[m.Version] == nil { 54 | i.migrations[m.Version] = make(map[Direction]*Migration) 55 | } 56 | 57 | // reject duplicate versions 58 | if _, dup := i.migrations[m.Version][m.Direction]; dup { 59 | return false 60 | } 61 | 62 | i.migrations[m.Version][m.Direction] = m 63 | i.buildIndex() 64 | 65 | return true 66 | } 67 | 68 | func (i *Migrations) buildIndex() { 69 | i.index = make(uintSlice, 0) 70 | for version, _ := range i.migrations { 71 | i.index = append(i.index, version) 72 | } 73 | sort.Sort(i.index) 74 | } 75 | 76 | func (i *Migrations) First() (version uint, ok bool) { 77 | if len(i.index) == 0 { 78 | return 0, false 79 | } 80 | return i.index[0], true 81 | } 82 | 83 | func (i *Migrations) Prev(version uint) (prevVersion uint, ok bool) { 84 | pos := i.findPos(version) 85 | if pos >= 1 && len(i.index) > pos-1 { 86 | return i.index[pos-1], true 87 | } 88 | return 0, false 89 | } 90 | 91 | func (i *Migrations) Next(version uint) (nextVersion uint, ok bool) { 92 | pos := i.findPos(version) 93 | if pos >= 0 && len(i.index) > pos+1 { 94 | return i.index[pos+1], true 95 | } 96 | return 0, false 97 | } 98 | 99 | func (i *Migrations) Up(version uint) (m *Migration, ok bool) { 100 | if _, ok := i.migrations[version]; ok { 101 | if mx, ok := i.migrations[version][Up]; ok { 102 | return mx, true 103 | } 104 | } 105 | return nil, false 106 | } 107 | 108 | func (i *Migrations) Down(version uint) (m *Migration, ok bool) { 109 | if _, ok := i.migrations[version]; ok { 110 | if mx, ok := i.migrations[version][Down]; ok { 111 | return mx, true 112 | } 113 | } 114 | return nil, false 115 | } 116 | 117 | func (i *Migrations) findPos(version uint) int { 118 | if len(i.index) > 0 { 119 | ix := i.index.Search(version) 120 | if ix < len(i.index) && i.index[ix] == version { 121 | return ix 122 | } 123 | } 124 | return -1 125 | } 126 | 127 | type uintSlice []uint 128 | 129 | func (s uintSlice) Len() int { 130 | return len(s) 131 | } 132 | 133 | func (s uintSlice) Swap(i, j int) { 134 | s[i], s[j] = s[j], s[i] 135 | } 136 | 137 | func (s uintSlice) Less(i, j int) bool { 138 | return s[i] < s[j] 139 | } 140 | 141 | func (s uintSlice) Search(x uint) int { 142 | return sort.Search(len(s), func(i int) bool { return s[i] >= x }) 143 | } 144 | -------------------------------------------------------------------------------- /source/migration_test.go: -------------------------------------------------------------------------------- 1 | package source 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestNewMigrations(t *testing.T) { 8 | // TODO 9 | } 10 | 11 | func TestAppend(t *testing.T) { 12 | // TODO 13 | } 14 | 15 | func TestBuildIndex(t *testing.T) { 16 | // TODO 17 | } 18 | 19 | func TestFirst(t *testing.T) { 20 | // TODO 21 | } 22 | 23 | func TestPrev(t *testing.T) { 24 | // TODO 25 | } 26 | 27 | func TestUp(t *testing.T) { 28 | // TODO 29 | } 30 | 31 | func TestDown(t *testing.T) { 32 | // TODO 33 | } 34 | 35 | func TestFindPos(t *testing.T) { 36 | m := Migrations{index: uintSlice{1, 2, 3}} 37 | if p := m.findPos(0); p != -1 { 38 | t.Errorf("expected -1, got %v", p) 39 | } 40 | if p := m.findPos(1); p != 0 { 41 | t.Errorf("expected 0, got %v", p) 42 | } 43 | if p := m.findPos(3); p != 2 { 44 | t.Errorf("expected 2, got %v", p) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /source/parse.go: -------------------------------------------------------------------------------- 1 | package source 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | "strconv" 7 | ) 8 | 9 | var ( 10 | ErrParse = fmt.Errorf("no match") 11 | ) 12 | 13 | var ( 14 | DefaultParse = Parse 15 | DefaultRegex = Regex 16 | ) 17 | 18 | // Regex matches the following pattern: 19 | // 123_name.up.ext 20 | // 123_name.down.ext 21 | var Regex = regexp.MustCompile(`^([0-9]+)_(.*)\.(` + string(Down) + `|` + string(Up) + `)\.(.*)$`) 22 | 23 | // Parse returns Migration for matching Regex pattern. 24 | func Parse(raw string) (*Migration, error) { 25 | m := Regex.FindStringSubmatch(raw) 26 | if len(m) == 5 { 27 | versionUint64, err := strconv.ParseUint(m[1], 10, 64) 28 | if err != nil { 29 | return nil, err 30 | } 31 | return &Migration{ 32 | Version: uint(versionUint64), 33 | Identifier: m[2], 34 | Direction: Direction(m[3]), 35 | Raw: raw, 36 | }, nil 37 | } 38 | return nil, ErrParse 39 | } 40 | -------------------------------------------------------------------------------- /source/parse_test.go: -------------------------------------------------------------------------------- 1 | package source 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestParse(t *testing.T) { 8 | tt := []struct { 9 | name string 10 | expectErr error 11 | expectMigration *Migration 12 | }{ 13 | { 14 | name: "1_foobar.up.sql", 15 | expectErr: nil, 16 | expectMigration: &Migration{ 17 | Version: 1, 18 | Identifier: "foobar", 19 | Direction: Up, 20 | Raw: "1_foobar.up.sql", 21 | }, 22 | }, 23 | { 24 | name: "1_foobar.down.sql", 25 | expectErr: nil, 26 | expectMigration: &Migration{ 27 | Version: 1, 28 | Identifier: "foobar", 29 | Direction: Down, 30 | Raw: "1_foobar.down.sql", 31 | }, 32 | }, 33 | { 34 | name: "1_f-o_ob+ar.up.sql", 35 | expectErr: nil, 36 | expectMigration: &Migration{ 37 | Version: 1, 38 | Identifier: "f-o_ob+ar", 39 | Direction: Up, 40 | Raw: "1_f-o_ob+ar.up.sql", 41 | }, 42 | }, 43 | { 44 | name: "1485385885_foobar.up.sql", 45 | expectErr: nil, 46 | expectMigration: &Migration{ 47 | Version: 1485385885, 48 | Identifier: "foobar", 49 | Direction: Up, 50 | Raw: "1485385885_foobar.up.sql", 51 | }, 52 | }, 53 | { 54 | name: "20170412214116_date_foobar.up.sql", 55 | expectErr: nil, 56 | expectMigration: &Migration{ 57 | Version: 20170412214116, 58 | Identifier: "date_foobar", 59 | Direction: Up, 60 | Raw: "20170412214116_date_foobar.up.sql", 61 | }, 62 | }, 63 | { 64 | name: "-1_foobar.up.sql", 65 | expectErr: ErrParse, 66 | expectMigration: nil, 67 | }, 68 | { 69 | name: "foobar.up.sql", 70 | expectErr: ErrParse, 71 | expectMigration: nil, 72 | }, 73 | { 74 | name: "1.up.sql", 75 | expectErr: ErrParse, 76 | expectMigration: nil, 77 | }, 78 | { 79 | name: "1_foobar.sql", 80 | expectErr: ErrParse, 81 | expectMigration: nil, 82 | }, 83 | { 84 | name: "1_foobar.up", 85 | expectErr: ErrParse, 86 | expectMigration: nil, 87 | }, 88 | { 89 | name: "1_foobar.down", 90 | expectErr: ErrParse, 91 | expectMigration: nil, 92 | }, 93 | } 94 | 95 | for i, v := range tt { 96 | f, err := Parse(v.name) 97 | 98 | if err != v.expectErr { 99 | t.Errorf("expected %v, got %v, in %v", v.expectErr, err, i) 100 | } 101 | 102 | if v.expectMigration != nil && *f != *v.expectMigration { 103 | t.Errorf("expected %+v, got %+v, in %v", *v.expectMigration, *f, i) 104 | } 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /source/stub/stub.go: -------------------------------------------------------------------------------- 1 | package stub 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "os" 9 | 10 | "github.com/mattes/migrate/source" 11 | ) 12 | 13 | func init() { 14 | source.Register("stub", &Stub{}) 15 | } 16 | 17 | type Config struct{} 18 | 19 | // d, _ := source.Open("stub://") 20 | // d.(*stub.Stub).Migrations = 21 | 22 | type Stub struct { 23 | Url string 24 | Instance interface{} 25 | Migrations *source.Migrations 26 | Config *Config 27 | } 28 | 29 | func (s *Stub) Open(url string) (source.Driver, error) { 30 | return &Stub{ 31 | Url: url, 32 | Migrations: source.NewMigrations(), 33 | Config: &Config{}, 34 | }, nil 35 | } 36 | 37 | func WithInstance(instance interface{}, config *Config) (source.Driver, error) { 38 | return &Stub{ 39 | Instance: instance, 40 | Migrations: source.NewMigrations(), 41 | Config: config, 42 | }, nil 43 | } 44 | 45 | func (s *Stub) Close() error { 46 | return nil 47 | } 48 | 49 | func (s *Stub) First() (version uint, err error) { 50 | if v, ok := s.Migrations.First(); !ok { 51 | return 0, &os.PathError{"first", s.Url, os.ErrNotExist} // TODO: s.Url can be empty when called with WithInstance 52 | } else { 53 | return v, nil 54 | } 55 | } 56 | 57 | func (s *Stub) Prev(version uint) (prevVersion uint, err error) { 58 | if v, ok := s.Migrations.Prev(version); !ok { 59 | return 0, &os.PathError{fmt.Sprintf("prev for version %v", version), s.Url, os.ErrNotExist} 60 | } else { 61 | return v, nil 62 | } 63 | } 64 | 65 | func (s *Stub) Next(version uint) (nextVersion uint, err error) { 66 | if v, ok := s.Migrations.Next(version); !ok { 67 | return 0, &os.PathError{fmt.Sprintf("next for version %v", version), s.Url, os.ErrNotExist} 68 | } else { 69 | return v, nil 70 | } 71 | } 72 | 73 | func (s *Stub) ReadUp(version uint) (r io.ReadCloser, identifier string, err error) { 74 | if m, ok := s.Migrations.Up(version); ok { 75 | return ioutil.NopCloser(bytes.NewBufferString(m.Identifier)), fmt.Sprintf("%v.up.stub", version), nil 76 | } 77 | return nil, "", &os.PathError{fmt.Sprintf("read up version %v", version), s.Url, os.ErrNotExist} 78 | } 79 | 80 | func (s *Stub) ReadDown(version uint) (r io.ReadCloser, identifier string, err error) { 81 | if m, ok := s.Migrations.Down(version); ok { 82 | return ioutil.NopCloser(bytes.NewBufferString(m.Identifier)), fmt.Sprintf("%v.down.stub", version), nil 83 | } 84 | return nil, "", &os.PathError{fmt.Sprintf("read down version %v", version), s.Url, os.ErrNotExist} 85 | } 86 | -------------------------------------------------------------------------------- /source/stub/stub_test.go: -------------------------------------------------------------------------------- 1 | package stub 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/mattes/migrate/source" 7 | st "github.com/mattes/migrate/source/testing" 8 | ) 9 | 10 | func Test(t *testing.T) { 11 | s := &Stub{} 12 | d, err := s.Open("") 13 | if err != nil { 14 | t.Fatal(err) 15 | } 16 | 17 | m := source.NewMigrations() 18 | m.Append(&source.Migration{Version: 1, Direction: source.Up}) 19 | m.Append(&source.Migration{Version: 1, Direction: source.Down}) 20 | m.Append(&source.Migration{Version: 3, Direction: source.Up}) 21 | m.Append(&source.Migration{Version: 4, Direction: source.Up}) 22 | m.Append(&source.Migration{Version: 4, Direction: source.Down}) 23 | m.Append(&source.Migration{Version: 5, Direction: source.Down}) 24 | m.Append(&source.Migration{Version: 7, Direction: source.Up}) 25 | m.Append(&source.Migration{Version: 7, Direction: source.Down}) 26 | 27 | d.(*Stub).Migrations = m 28 | 29 | st.Test(t, d) 30 | } 31 | -------------------------------------------------------------------------------- /source/testing/testing.go: -------------------------------------------------------------------------------- 1 | // Package testing has the source tests. 2 | // All source drivers must pass the Test function. 3 | // This lives in it's own package so it stays a test dependency. 4 | package testing 5 | 6 | import ( 7 | "os" 8 | "testing" 9 | 10 | "github.com/mattes/migrate/source" 11 | ) 12 | 13 | // Test runs tests against source implementations. 14 | // It assumes that the driver tests has access to the following migrations: 15 | // 16 | // u = up migration, d = down migration, n = version 17 | // | 1 | - | 3 | 4 | 5 | - | 7 | 18 | // | u d | - | u | u d | d | - | u d | 19 | // 20 | // See source/stub/stub_test.go or source/file/file_test.go for an example. 21 | func Test(t *testing.T, d source.Driver) { 22 | TestFirst(t, d) 23 | TestPrev(t, d) 24 | TestNext(t, d) 25 | TestReadUp(t, d) 26 | TestReadDown(t, d) 27 | } 28 | 29 | func TestFirst(t *testing.T, d source.Driver) { 30 | version, err := d.First() 31 | if err != nil { 32 | t.Fatalf("First: expected err to be nil, got %v", err) 33 | } 34 | if version != 1 { 35 | t.Errorf("First: expected 1, got %v", version) 36 | } 37 | } 38 | 39 | func TestPrev(t *testing.T, d source.Driver) { 40 | tt := []struct { 41 | version uint 42 | expectErr error 43 | expectPrevVersion uint 44 | }{ 45 | {version: 0, expectErr: os.ErrNotExist}, 46 | {version: 1, expectErr: os.ErrNotExist}, 47 | {version: 2, expectErr: os.ErrNotExist}, 48 | {version: 3, expectErr: nil, expectPrevVersion: 1}, 49 | {version: 4, expectErr: nil, expectPrevVersion: 3}, 50 | {version: 5, expectErr: nil, expectPrevVersion: 4}, 51 | {version: 6, expectErr: os.ErrNotExist}, 52 | {version: 7, expectErr: nil, expectPrevVersion: 5}, 53 | {version: 8, expectErr: os.ErrNotExist}, 54 | {version: 9, expectErr: os.ErrNotExist}, 55 | } 56 | 57 | for i, v := range tt { 58 | pv, err := d.Prev(v.version) 59 | if (v.expectErr == os.ErrNotExist && !os.IsNotExist(err)) && v.expectErr != err { 60 | t.Errorf("Prev: expected %v, got %v, in %v", v.expectErr, err, i) 61 | } 62 | if err == nil && v.expectPrevVersion != pv { 63 | t.Errorf("Prev: expected %v, got %v, in %v", v.expectPrevVersion, pv, i) 64 | } 65 | } 66 | } 67 | 68 | func TestNext(t *testing.T, d source.Driver) { 69 | tt := []struct { 70 | version uint 71 | expectErr error 72 | expectNextVersion uint 73 | }{ 74 | {version: 0, expectErr: os.ErrNotExist}, 75 | {version: 1, expectErr: nil, expectNextVersion: 3}, 76 | {version: 2, expectErr: os.ErrNotExist}, 77 | {version: 3, expectErr: nil, expectNextVersion: 4}, 78 | {version: 4, expectErr: nil, expectNextVersion: 5}, 79 | {version: 5, expectErr: nil, expectNextVersion: 7}, 80 | {version: 6, expectErr: os.ErrNotExist}, 81 | {version: 7, expectErr: os.ErrNotExist}, 82 | {version: 8, expectErr: os.ErrNotExist}, 83 | {version: 9, expectErr: os.ErrNotExist}, 84 | } 85 | 86 | for i, v := range tt { 87 | nv, err := d.Next(v.version) 88 | if (v.expectErr == os.ErrNotExist && !os.IsNotExist(err)) && v.expectErr != err { 89 | t.Errorf("Next: expected %v, got %v, in %v", v.expectErr, err, i) 90 | } 91 | if err == nil && v.expectNextVersion != nv { 92 | t.Errorf("Next: expected %v, got %v, in %v", v.expectNextVersion, nv, i) 93 | } 94 | } 95 | } 96 | 97 | func TestReadUp(t *testing.T, d source.Driver) { 98 | tt := []struct { 99 | version uint 100 | expectErr error 101 | expectUp bool 102 | }{ 103 | {version: 0, expectErr: os.ErrNotExist}, 104 | {version: 1, expectErr: nil, expectUp: true}, 105 | {version: 2, expectErr: os.ErrNotExist}, 106 | {version: 3, expectErr: nil, expectUp: true}, 107 | {version: 4, expectErr: nil, expectUp: true}, 108 | {version: 5, expectErr: os.ErrNotExist}, 109 | {version: 6, expectErr: os.ErrNotExist}, 110 | {version: 7, expectErr: nil, expectUp: true}, 111 | {version: 8, expectErr: os.ErrNotExist}, 112 | } 113 | 114 | for i, v := range tt { 115 | up, identifier, err := d.ReadUp(v.version) 116 | if (v.expectErr == os.ErrNotExist && !os.IsNotExist(err)) || 117 | (v.expectErr != os.ErrNotExist && err != v.expectErr) { 118 | t.Errorf("expected %v, got %v, in %v", v.expectErr, err, i) 119 | 120 | } else if err == nil { 121 | if len(identifier) == 0 { 122 | t.Errorf("expected identifier not to be empty, in %v", i) 123 | } 124 | 125 | if v.expectUp == true && up == nil { 126 | t.Errorf("expected up not to be nil, in %v", i) 127 | } else if v.expectUp == false && up != nil { 128 | t.Errorf("expected up to be nil, got %v, in %v", up, i) 129 | } 130 | } 131 | } 132 | } 133 | 134 | func TestReadDown(t *testing.T, d source.Driver) { 135 | tt := []struct { 136 | version uint 137 | expectErr error 138 | expectDown bool 139 | }{ 140 | {version: 0, expectErr: os.ErrNotExist}, 141 | {version: 1, expectErr: nil, expectDown: true}, 142 | {version: 2, expectErr: os.ErrNotExist}, 143 | {version: 3, expectErr: os.ErrNotExist}, 144 | {version: 4, expectErr: nil, expectDown: true}, 145 | {version: 5, expectErr: nil, expectDown: true}, 146 | {version: 6, expectErr: os.ErrNotExist}, 147 | {version: 7, expectErr: nil, expectDown: true}, 148 | {version: 8, expectErr: os.ErrNotExist}, 149 | } 150 | 151 | for i, v := range tt { 152 | down, identifier, err := d.ReadDown(v.version) 153 | if (v.expectErr == os.ErrNotExist && !os.IsNotExist(err)) || 154 | (v.expectErr != os.ErrNotExist && err != v.expectErr) { 155 | t.Errorf("expected %v, got %v, in %v", v.expectErr, err, i) 156 | 157 | } else if err == nil { 158 | if len(identifier) == 0 { 159 | t.Errorf("expected identifier not to be empty, in %v", i) 160 | } 161 | 162 | if v.expectDown == true && down == nil { 163 | t.Errorf("expected down not to be nil, in %v", i) 164 | } else if v.expectDown == false && down != nil { 165 | t.Errorf("expected down to be nil, got %v, in %v", down, i) 166 | } 167 | } 168 | } 169 | } 170 | -------------------------------------------------------------------------------- /testing/docker.go: -------------------------------------------------------------------------------- 1 | // Package testing is used in driver tests. 2 | package testing 3 | 4 | import ( 5 | "bufio" 6 | "context" 7 | "encoding/json" 8 | "fmt" 9 | "io" 10 | "math/rand" 11 | "strconv" 12 | "strings" 13 | "testing" 14 | "time" 15 | dockertypes "github.com/docker/docker/api/types" 16 | dockercontainer "github.com/docker/docker/api/types/container" 17 | dockernetwork "github.com/docker/docker/api/types/network" 18 | dockerclient "github.com/docker/docker/client" 19 | ) 20 | 21 | func NewDockerContainer(t testing.TB, image string, env []string, cmd []string) (*DockerContainer, error) { 22 | c, err := dockerclient.NewEnvClient() 23 | if err != nil { 24 | return nil, err 25 | } 26 | 27 | if cmd == nil { 28 | cmd = make([]string, 0) 29 | } 30 | 31 | contr := &DockerContainer{ 32 | t: t, 33 | client: c, 34 | ImageName: image, 35 | ENV: env, 36 | Cmd: cmd, 37 | } 38 | 39 | if err := contr.PullImage(); err != nil { 40 | return nil, err 41 | } 42 | 43 | if err := contr.Start(); err != nil { 44 | return nil, err 45 | } 46 | 47 | return contr, nil 48 | } 49 | 50 | // DockerContainer implements Instance interface 51 | type DockerContainer struct { 52 | t testing.TB 53 | client *dockerclient.Client 54 | ImageName string 55 | ENV []string 56 | Cmd []string 57 | ContainerId string 58 | ContainerName string 59 | ContainerJSON dockertypes.ContainerJSON 60 | containerInspected bool 61 | keepForDebugging bool 62 | } 63 | 64 | func (d *DockerContainer) PullImage() error { 65 | d.t.Logf("Docker: Pull image %v", d.ImageName) 66 | r, err := d.client.ImagePull(context.Background(), d.ImageName, dockertypes.ImagePullOptions{}) 67 | if err != nil { 68 | return err 69 | } 70 | defer r.Close() 71 | 72 | // read output and log relevant lines 73 | bf := bufio.NewScanner(r) 74 | for bf.Scan() { 75 | var resp dockerImagePullOutput 76 | if err := json.Unmarshal(bf.Bytes(), &resp); err != nil { 77 | return err 78 | } 79 | if strings.HasPrefix(resp.Status, "Status: ") { 80 | d.t.Logf("Docker: %v", resp.Status) 81 | } 82 | } 83 | return bf.Err() 84 | } 85 | 86 | func (d *DockerContainer) Start() error { 87 | containerName := fmt.Sprintf("migrate_test_%v", pseudoRandStr(10)) 88 | 89 | // create container first 90 | resp, err := d.client.ContainerCreate(context.Background(), 91 | &dockercontainer.Config{ 92 | Image: d.ImageName, 93 | Labels: map[string]string{"migrate_test": "true"}, 94 | Env: d.ENV, 95 | Cmd: d.Cmd, 96 | }, 97 | &dockercontainer.HostConfig{ 98 | PublishAllPorts: true, 99 | }, 100 | &dockernetwork.NetworkingConfig{}, 101 | containerName) 102 | if err != nil { 103 | return err 104 | } 105 | 106 | d.ContainerId = resp.ID 107 | d.ContainerName = containerName 108 | 109 | // then start it 110 | if err := d.client.ContainerStart(context.Background(), resp.ID, dockertypes.ContainerStartOptions{}); err != nil { 111 | return err 112 | } 113 | 114 | d.t.Logf("Docker: Started container %v (%v) for image %v listening at %v:%v", resp.ID[0:12], containerName, d.ImageName, d.Host(), d.Port()) 115 | for _, v := range resp.Warnings { 116 | d.t.Logf("Docker: Warning: %v", v) 117 | } 118 | return nil 119 | } 120 | 121 | func (d *DockerContainer) KeepForDebugging() { 122 | d.keepForDebugging = true 123 | } 124 | 125 | func (d *DockerContainer) Remove() error { 126 | if d.keepForDebugging { 127 | return nil 128 | } 129 | 130 | if len(d.ContainerId) == 0 { 131 | return fmt.Errorf("missing containerId") 132 | } 133 | if err := d.client.ContainerRemove(context.Background(), d.ContainerId, 134 | dockertypes.ContainerRemoveOptions{ 135 | Force: true, 136 | }); err != nil { 137 | d.t.Log(err) 138 | return err 139 | } 140 | d.t.Logf("Docker: Removed %v", d.ContainerName) 141 | return nil 142 | } 143 | 144 | func (d *DockerContainer) Inspect() error { 145 | if len(d.ContainerId) == 0 { 146 | return fmt.Errorf("missing containerId") 147 | } 148 | resp, err := d.client.ContainerInspect(context.Background(), d.ContainerId) 149 | if err != nil { 150 | return err 151 | } 152 | 153 | d.ContainerJSON = resp 154 | d.containerInspected = true 155 | return nil 156 | } 157 | 158 | func (d *DockerContainer) Logs() (io.ReadCloser, error) { 159 | if len(d.ContainerId) == 0 { 160 | return nil, fmt.Errorf("missing containerId") 161 | } 162 | 163 | return d.client.ContainerLogs(context.Background(), d.ContainerId, dockertypes.ContainerLogsOptions{ 164 | ShowStdout: true, 165 | ShowStderr: true, 166 | }) 167 | } 168 | 169 | func (d *DockerContainer) portMapping(selectFirst bool, cPort int) (containerPort uint, hostIP string, hostPort uint, err error) { 170 | if !d.containerInspected { 171 | if err := d.Inspect(); err != nil { 172 | d.t.Fatal(err) 173 | } 174 | } 175 | 176 | for port, bindings := range d.ContainerJSON.NetworkSettings.Ports { 177 | if !selectFirst && port.Int() != cPort { 178 | // Skip ahead until we find the port we want 179 | continue 180 | } 181 | for _, binding := range bindings { 182 | 183 | hostPortUint, err := strconv.ParseUint(binding.HostPort, 10, 64) 184 | if err != nil { 185 | return 0, "", 0, err 186 | } 187 | 188 | return uint(port.Int()), binding.HostIP, uint(hostPortUint), nil 189 | } 190 | } 191 | 192 | if selectFirst { 193 | return 0, "", 0, fmt.Errorf("no port binding") 194 | } else { 195 | return 0, "", 0, fmt.Errorf("specified port not bound") 196 | } 197 | } 198 | 199 | func (d *DockerContainer) Host() string { 200 | _, hostIP, _, err := d.portMapping(true, -1) 201 | if err != nil { 202 | d.t.Fatal(err) 203 | } 204 | 205 | if hostIP == "0.0.0.0" { 206 | return "127.0.0.1" 207 | } else { 208 | return hostIP 209 | } 210 | } 211 | 212 | func (d *DockerContainer) Port() uint { 213 | _, _, port, err := d.portMapping(true, -1) 214 | if err != nil { 215 | d.t.Fatal(err) 216 | } 217 | return port 218 | } 219 | 220 | func (d *DockerContainer) PortFor(cPort int) uint { 221 | _, _, port, err := d.portMapping(false, cPort) 222 | if err != nil { 223 | d.t.Fatal(err) 224 | } 225 | return port 226 | } 227 | 228 | func (d *DockerContainer) NetworkSettings() dockertypes.NetworkSettings { 229 | netSettings := d.ContainerJSON.NetworkSettings 230 | return *netSettings 231 | } 232 | 233 | type dockerImagePullOutput struct { 234 | Status string `json:"status"` 235 | ProgressDetails struct { 236 | Current int `json:"current"` 237 | Total int `json:"total"` 238 | } `json:"progressDetail"` 239 | Id string `json:"id"` 240 | Progress string `json:"progress"` 241 | } 242 | 243 | func init() { 244 | rand.Seed(time.Now().UnixNano()) 245 | } 246 | 247 | func pseudoRandStr(n int) string { 248 | var letterRunes = []rune("abcdefghijklmnopqrstuvwxyz0123456789") 249 | b := make([]rune, n) 250 | for i := range b { 251 | b[i] = letterRunes[rand.Intn(len(letterRunes))] 252 | } 253 | return string(b) 254 | } 255 | -------------------------------------------------------------------------------- /testing/testing.go: -------------------------------------------------------------------------------- 1 | package testing 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "strconv" 7 | "testing" 8 | "time" 9 | 10 | dockertypes "github.com/docker/docker/api/types" 11 | ) 12 | 13 | type IsReadyFunc func(Instance) bool 14 | 15 | type TestFunc func(*testing.T, Instance) 16 | 17 | type Version struct { 18 | Image string 19 | ENV []string 20 | Cmd []string 21 | } 22 | 23 | func ParallelTest(t *testing.T, versions []Version, readyFn IsReadyFunc, testFn TestFunc) { 24 | delay, err := strconv.Atoi(os.Getenv("MIGRATE_TEST_CONTAINER_BOOT_DELAY")) 25 | if err != nil { 26 | delay = 0 27 | } 28 | 29 | for i, version := range versions { 30 | version := version // capture range variable, see https://goo.gl/60w3p2 31 | 32 | // Only test against one version in short mode 33 | // TODO: order is random, maybe always pick first version instead? 34 | if i > 0 && testing.Short() { 35 | t.Logf("Skipping %v in short mode", version) 36 | 37 | } else { 38 | t.Run(version.Image, func(t *testing.T) { 39 | t.Parallel() 40 | 41 | // create new container 42 | container, err := NewDockerContainer(t, version.Image, version.ENV, version.Cmd) 43 | if err != nil { 44 | t.Fatalf("%v\n%s", err, containerLogs(t, container)) 45 | } 46 | 47 | // make sure to remove container once done 48 | defer container.Remove() 49 | 50 | // wait until database is ready 51 | tick := time.Tick(1000 * time.Millisecond) 52 | timeout := time.After(time.Duration(delay + 60) * time.Second) 53 | outer: 54 | for { 55 | select { 56 | case <-tick: 57 | if readyFn(container) { 58 | break outer 59 | } 60 | 61 | case <-timeout: 62 | t.Fatalf("Docker: Container not ready, timeout for %v.\n%s", version, containerLogs(t, container)) 63 | } 64 | } 65 | 66 | time.Sleep(time.Duration(int64(delay)) * time.Second) 67 | 68 | // we can now run the tests 69 | testFn(t, container) 70 | }) 71 | } 72 | } 73 | } 74 | 75 | func containerLogs(t *testing.T, c *DockerContainer) []byte { 76 | r, err := c.Logs() 77 | if err != nil { 78 | t.Error("%v", err) 79 | return nil 80 | } 81 | defer r.Close() 82 | b, err := ioutil.ReadAll(r) 83 | if err != nil { 84 | t.Error("%v", err) 85 | return nil 86 | } 87 | return b 88 | } 89 | 90 | type Instance interface { 91 | Host() string 92 | Port() uint 93 | PortFor(int) uint 94 | NetworkSettings() dockertypes.NetworkSettings 95 | KeepForDebugging() 96 | } 97 | -------------------------------------------------------------------------------- /testing/testing_test.go: -------------------------------------------------------------------------------- 1 | package testing 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func ExampleParallelTest(t *testing.T) { 8 | var isReady = func(i Instance) bool { 9 | // Return true if Instance is ready to run tests. 10 | // Don't block here though. 11 | return true 12 | } 13 | 14 | // t is *testing.T coming from parent Test(t *testing.T) 15 | ParallelTest(t, []Version{{Image: "docker_image:9.6"}}, isReady, 16 | func(t *testing.T, i Instance) { 17 | // Run your test/s ... 18 | t.Fatal("...") 19 | }) 20 | } 21 | -------------------------------------------------------------------------------- /util.go: -------------------------------------------------------------------------------- 1 | package migrate 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "io" 7 | nurl "net/url" 8 | "strings" 9 | "time" 10 | ) 11 | 12 | // MultiError holds multiple errors. 13 | type MultiError struct { 14 | Errs []error 15 | } 16 | 17 | // NewMultiError returns an error type holding multiple errors. 18 | func NewMultiError(errs ...error) MultiError { 19 | compactErrs := make([]error, 0) 20 | for _, e := range errs { 21 | if e != nil { 22 | compactErrs = append(compactErrs, e) 23 | } 24 | } 25 | return MultiError{compactErrs} 26 | } 27 | 28 | // Error implements error. Mulitple errors are concatenated with 'and's. 29 | func (m MultiError) Error() string { 30 | var strs = make([]string, 0) 31 | for _, e := range m.Errs { 32 | if len(e.Error()) > 0 { 33 | strs = append(strs, e.Error()) 34 | } 35 | } 36 | return strings.Join(strs, " and ") 37 | } 38 | 39 | // suint safely converts int to uint 40 | // see https://goo.gl/wEcqof 41 | // see https://goo.gl/pai7Dr 42 | func suint(n int) uint { 43 | if n < 0 { 44 | panic(fmt.Sprintf("suint(%v) expects input >= 0", n)) 45 | } 46 | return uint(n) 47 | } 48 | 49 | // newSlowReader turns an io.ReadCloser into a slow io.ReadCloser. 50 | // Use this to simulate a slow internet connection. 51 | func newSlowReader(r io.ReadCloser) io.ReadCloser { 52 | return &slowReader{ 53 | rx: r, 54 | reader: bufio.NewReader(r), 55 | } 56 | } 57 | 58 | type slowReader struct { 59 | rx io.ReadCloser 60 | reader *bufio.Reader 61 | } 62 | 63 | func (b *slowReader) Read(p []byte) (n int, err error) { 64 | time.Sleep(10 * time.Millisecond) 65 | c, err := b.reader.ReadByte() 66 | if err != nil { 67 | return 0, err 68 | } else { 69 | copy(p, []byte{c}) 70 | return 1, nil 71 | } 72 | } 73 | 74 | func (b *slowReader) Close() error { 75 | return b.rx.Close() 76 | } 77 | 78 | var errNoScheme = fmt.Errorf("no scheme") 79 | 80 | // schemeFromUrl returns the scheme from a URL string 81 | func schemeFromUrl(url string) (string, error) { 82 | u, err := nurl.Parse(url) 83 | if err != nil { 84 | return "", err 85 | } 86 | 87 | if len(u.Scheme) == 0 { 88 | return "", errNoScheme 89 | } 90 | 91 | return u.Scheme, nil 92 | } 93 | 94 | // FilterCustomQuery filters all query values starting with `x-` 95 | func FilterCustomQuery(u *nurl.URL) *nurl.URL { 96 | ux := *u 97 | vx := make(nurl.Values) 98 | for k, v := range ux.Query() { 99 | if len(k) <= 1 || (len(k) > 1 && k[0:2] != "x-") { 100 | vx[k] = v 101 | } 102 | } 103 | ux.RawQuery = vx.Encode() 104 | return &ux 105 | } 106 | -------------------------------------------------------------------------------- /util_test.go: -------------------------------------------------------------------------------- 1 | package migrate 2 | 3 | import ( 4 | nurl "net/url" 5 | "testing" 6 | ) 7 | 8 | func TestSuintPanicsWithNegativeInput(t *testing.T) { 9 | defer func() { 10 | if r := recover(); r == nil { 11 | t.Fatal("expected suint to panic for -1") 12 | } 13 | }() 14 | suint(-1) 15 | } 16 | 17 | func TestSuint(t *testing.T) { 18 | if u := suint(0); u != 0 { 19 | t.Fatalf("expected 0, got %v", u) 20 | } 21 | } 22 | 23 | func TestFilterCustomQuery(t *testing.T) { 24 | n, err := nurl.Parse("foo://host?a=b&x-custom=foo&c=d") 25 | if err != nil { 26 | t.Fatal(err) 27 | } 28 | nx := FilterCustomQuery(n).Query() 29 | if nx.Get("x-custom") != "" { 30 | t.Fatalf("didn't expect x-custom") 31 | } 32 | } 33 | --------------------------------------------------------------------------------