├── .circleci └── config.yml ├── .github └── FUNDING.yml ├── .gitignore ├── .travis.yml ├── CHANGES.md ├── README.md ├── bin └── kaocha ├── deps.edn ├── migrate.png ├── project.clj ├── release.cljs ├── src └── migratus │ ├── cli.clj │ ├── core.clj │ ├── database.clj │ ├── migration │ ├── edn.clj │ └── sql.clj │ ├── migrations.clj │ ├── properties.clj │ ├── protocols.clj │ └── utils.clj ├── test ├── migrations-bad-type │ └── 20170328124600-bad-type.foo ├── migrations-duplicate-name │ ├── 20170328130700-dup-name-1.up.sql │ └── 20170328130700-dup-name-2.up.sql ├── migrations-duplicate-type │ ├── 20170328125100-duplicate-type.edn │ └── 20170328125100-duplicate-type.up.sql ├── migrations-edn-args │ └── 20220827210100-say-hello-with-args.edn ├── migrations-edn │ └── 20170330142700-say-hello.edn ├── migrations-intentionally-broken-no-tx │ ├── 20120827170200-multiple-statements-broken.down.sql │ └── 20120827170200-multiple-statements-broken.up.sql ├── migrations-intentionally-broken │ ├── 20120827170200-multiple-statements-broken.down.sql │ └── 20120827170200-multiple-statements-broken.up.sql ├── migrations-jar │ ├── init-test.jar │ └── migrations.jar ├── migrations-no-tx │ ├── 20111202110600-create-foo-table.down.sql │ └── 20111202110600-create-foo-table.up.sql ├── migrations-parse │ ├── 20241012170200-create-quux-table.down.sql │ └── 20241012170200-create-quux-table.up.sql ├── migrations-postgres │ ├── 20220820030400-create-table-quux.down.sql │ ├── 20220820030400-create-table-quux.up.sql │ └── init.sql ├── migrations-with-props │ ├── 20111202110600-create-foo-table.down.sql │ ├── 20111202110600-create-foo-table.up.sql │ └── 20111202110600-create-schema.up.sql ├── migrations │ ├── 20111202110600-create-foo-table.down.sql │ ├── 20111202110600-create-foo-table.up.sql │ ├── 20111202110600-create-foo-table.up.sql~ │ ├── 20111202113000-create-bar-table.down.sql │ ├── 20111202113000-create-bar-table.up.sql │ ├── 20120827170200-multiple-statements.down.sql │ ├── 20120827170200-multiple-statements.up.sql │ ├── bogus.txt │ └── init.sql ├── migrations1 │ ├── 20220604110500-create-foo1-table.down.sql │ ├── 20220604110500-create-foo1-table.up.sql │ ├── 20220604113000-create-bar1-table.down.sql │ └── 20220604113000-create-bar1-table.up.sql ├── migrations2 │ ├── 20220604111500-create-foo2-table.down.sql │ ├── 20220604111500-create-foo2-table.up.sql │ ├── 20220604113500-create-bar2-table.down.sql │ └── 20220604113500-create-bar2-table.up.sql └── migratus │ ├── logger.clj │ ├── mock.clj │ ├── test │ ├── core.clj │ ├── database.clj │ ├── migration │ │ ├── edn.clj │ │ ├── edn │ │ │ ├── test_script.clj │ │ │ └── test_script_args.clj │ │ ├── edn_with_args.clj │ │ └── sql.clj │ ├── migrations.clj │ └── utils.clj │ └── testcontainers │ └── postgres.clj └── tests.edn /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | jobs: 3 | build: 4 | parameters: 5 | clojure-tag: 6 | type: string 7 | machine: 8 | image: ubuntu-2204:edge 9 | working_directory: /home/circleci/migratus 10 | steps: 11 | - checkout 12 | - restore_cache: 13 | keys: 14 | - v1-dependency-jars-{{ checksum "project.clj" }} 15 | - v1-dependency-jars 16 | - run: 17 | name: run tests 18 | command: | 19 | whoami 20 | echo $PWD 21 | docker run --rm -u root -v $PWD:/home/circleci/migratus -v /var/run/docker.sock:/var/run/docker.sock -v /home/circleci/.m2:/home/circleci/.m2 -v /home/circleci/.lein:/home/circleci/.lein -w=/home/circleci/migratus cimg/clojure:<< parameters.clojure-tag >> bin/kaocha 22 | ls -lah 23 | # Change permissions for migratus 24 | sudo chown -R circleci:circleci /home/circleci/migratus 25 | # Make sure .m2 and .lein have also right permissions 26 | sudo chown -R circleci:circleci /home/circleci/ 27 | ls -lah 28 | 29 | - store_test_results: 30 | path: /home/circleci/migratus/target/test-reports/ 31 | - save_cache: 32 | key: v1-dependency-jars-{{ checksum "project.clj" }} 33 | paths: 34 | - /home/circleci/.m2 35 | - /home/circleci/.lein 36 | 37 | workflows: 38 | build: 39 | jobs: 40 | - build: 41 | matrix: 42 | parameters: 43 | clojure-tag: ["1.10-openjdk-8.0", "1.11-openjdk-8.0", "1.11-openjdk-11.0", "1.11-openjdk-17.0"] 44 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: yogthos 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | pom.xml 2 | *jar 3 | /lib/ 4 | /classes/ 5 | /target/ 6 | .lein-failures 7 | .lein-deps-sum 8 | .lein-repl-history 9 | .nrepl-port 10 | *.log 11 | .idea 12 | migratus.iml 13 | *.db 14 | 15 | .clj-kondo/.cache 16 | .lsp/.cache 17 | .portal 18 | .cpcache 19 | .calva/output-window/ 20 | .clj-kondo/ -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: clojure 2 | lein: lein 3 | jdk: 4 | - openjdk6 5 | - openjdk7 6 | - oraclejdk7 7 | -------------------------------------------------------------------------------- /CHANGES.md: -------------------------------------------------------------------------------- 1 | ### 1.6.3 2 | * add tools.cli to project.clj 3 | * 4 | ### 1.6.2 5 | * add dependency for clojure.data.json, bump up next-jdbc to 1.3.955 6 | 7 | ### 1.6.1 8 | * [Add functionality for squashing migrations](https://github.com/yogthos/migratus/pull/271) 9 | 10 | ### 1.6.0 11 | * [Warn on multiple statements without delimiter](https://github.com/yogthos/migratus/pull/270) 12 | 13 | ### 1.5.9 14 | * [Fix bug in sanitizing in-line comments](https://github.com/yogthos/migratus/pull/269) 15 | 16 | ### 1.5.8 17 | * fix for resolving the migration path from a jar on Windows 18 | 19 | ### 1.5.7 20 | * add exclusion for `parse-long` in `migratus.cli` 21 | 22 | ### 1.5.6 23 | * [Fix spec problem for next.jdbc.sql/insert! and next.jdbc.sql/delete! ](https://github.com/yogthos/migratus/pull/262) 24 | 25 | ### 1.5.5 26 | * [allow migrations when next.jdbc.transaction/*nested-tx* is set to :prohibit](https://github.com/yogthos/migratus/pull/261) 27 | 28 | ### 1.5.4 29 | * [handle connnection in no-tx mode](https://github.com/yogthos/migratus/pull/254) 30 | * [allow externally managed connections](https://github.com/yogthos/migratus/pull/256) 31 | 32 | ### 1.5.3 33 | * [support for listing migration logs](https://github.com/yogthos/migratus/pull/251) 34 | 35 | ### 1.5.2 36 | * [CLI options](https://github.com/yogthos/migratus/pull/244) 37 | * [logging improvements](https://github.com/yogthos/migratus/pull/245) 38 | 39 | ### 1.5.1 40 | fixed handling lazy for in migration creation 41 | 42 | ### 1.5.0 43 | return the names when creating migrations with `migratus.core/create` 44 | 45 | ### 1.4.9 46 | [Fix error if any migration lacked an applied date](https://github.com/yogthos/migratus/pull/237) 47 | 48 | ### 1.4.8 49 | [Re-throw connection error](https://github.com/yogthos/migratus/pull/236) 50 | 51 | ### 1.4.7 52 | [update rollback behavior to roll back last applied migration when no arguments provided](https://github.com/yogthos/migratus/issues/199) 53 | 54 | ### 1.4.6 55 | 56 | [ability to exclude scripts based on globs](https://github.com/yogthos/migratus/pull/232) 57 | 58 | ### 1.4.5 59 | [remove sanitation of migration table name](https://github.com/yogthos/migratus/pull/231) 60 | 61 | ### 1.4.0 62 | 63 | new feature: Run basic tests against PostgreSQL using testcontainers 64 | 65 | new feature: Circle CI matrix runner - run against multiple clojure and jdk versions 66 | 67 | new feature: Circle CI junit reports, run tests with kaocha 68 | 69 | bug fix: [Pass :connection and :datasource through :db](https://github.com/yogthos/migratus/issues/181) 70 | 71 | enhancement: [Upgrade to next.jdbc 1.2.790 - Can pass {:connection-uri ...}](https://github.com/yogthos/migratus/issues/221) 72 | 73 | ### 1.3.8 74 | 75 | new feature: [Provide deps.edn and kaocha test runner](https://github.com/yogthos/migratus/pull/212) 76 | 77 | new feature: [Port migratus to next.jdbc](https://github.com/yogthos/migratus/pull/214) 78 | 79 | ### 1.3.7 80 | 81 | new feature: [Multi migration dirs](https://github.com/yogthos/migratus/pull/210) 82 | 83 | ### 1.3.6 84 | 85 | feature: [:transaction? config flag to toggle whether migrations happen within a transaction](https://github.com/yogthos/migratus/pull/209) 86 | 87 | ### 1.3.5 88 | 89 | new fearture: [rollback-until-just-after function](https://github.com/yogthos/migratus/pull/201) 90 | 91 | ### 1.3.4 92 | 93 | new feature: [property substitution](https://github.com/yogthos/migratus/pull/198) 94 | 95 | ### 1.3.3 96 | 97 | type hint for the data source, updated dependencies 98 | 99 | ### 1.3.2 100 | 101 | [Don't log value of :connection-uri in db-spec](https://github.com/yogthos/migratus/pull/193) 102 | 103 | ### 1.3.1 104 | 105 | [skip logging connection-uri on connection failure to avoid logging passwords](https://github.com/yogthos/migratus/pull/192) 106 | 107 | ### 1.3.0 108 | 109 | [Wrap modify-sql-fn to support returning a sequence](https://github.com/yogthos/migratus/pull/187) 110 | 111 | ### 1.2.9 112 | 113 | [Fix DB connection leak in select-migrations](https://github.com/yogthos/migratus/pull/186) 114 | 115 | ### 1.2.8 116 | 117 | fix for checking whether table exists when using the latest pg driver 118 | 119 | ### 1.2.7 120 | 121 | allow subfolders inside the migrations folder [PR 176](https://github.com/yogthos/migratus/pull/176) 122 | 123 | ### 1.2.6 124 | 125 | improved error reporting for invalid migation file names 126 | 127 | ### 1.2.5 128 | 129 | fixed error for transactional mode. 130 | 131 | ### 1.2.4 132 | 133 | [support for passing in an existing connection](https://github.com/yogthos/migratus/pull/172) 134 | 135 | ### 1.2.3 136 | 137 | censor passworrd in logs [PR 166](https://github.com/yogthos/migratus/pull/166) 138 | 139 | ### 1.2.2 140 | 141 | fix doc strings, [file system lookup for pending migrations](https://github.com/yogthos/migratus/commit/6bd8948b452a4ba909e1f978f7a33422e47b3d9e) 142 | 143 | ### 1.2.1 144 | 145 | type hints for compiling with Graal 146 | 147 | ### 1.1.1 148 | 149 | [pr](https://github.com/yogthos/migratus/pull/151) that adds the optional `-- expect` sanity check in migrations 150 | [pr](https://github.com/yogthos/migratus/pull/150) that adds `:tx-handles-ddl?` flag that skips the automatic down that occurs on exception 151 | 152 | ### 1.1.0 153 | 154 | - switched to use `db-do-commands` when applying migrations to address [issue 149](https://github.com/yogthos/migratus/issues/149) 155 | 156 | ### 1.0.9 157 | 158 | [PR 144](https://github.com/yogthos/migratus/pull/144) removed \n in SQL to also allow windows line terminators. 159 | 160 | ### 1.0.8 161 | 162 | alter migration function to return nil if successful, `:ignore` or `:failure` when migrations are incomplete. 163 | Add support for Thread cancellation during migrations. 164 | Tests added for backout. 165 | 166 | ### 1.0.7 167 | 168 | Update dependency on `org.clojure/tools.logging` to 0.4.1 169 | Update dependency on `org.clojure/java.jdbc` to 0.7.7 170 | Fix issue with handling directories that have spaces. 171 | 172 | ### 1.0.6 173 | 174 | search Context classloader as fall back to system class loader for migration directory discovery 175 | 176 | ### 1.0.5 177 | ### 1.0.4 178 | 179 | updated `migratus.migrations/timestamp` to use UTC 180 | 181 | ### 1.0.3 182 | 183 | updated `pending-list` function to use `log/debug` as well as return names of the migrations as a vector. 184 | 185 | ### 0.9.2 186 | 187 | ### 0.9.2 188 | 189 | Changedd `datetime` to `timestamp` as it's supported by more databases. 190 | 191 | ### 0.9.1 192 | 193 | #### features 194 | 195 | As of version 0.9.1 Migratus writes a human-readable description, and timestamp when the migration was applied. 196 | This is a breaking change, as the schema for the migration table has changed. Users upgrading from pervious versions 197 | need the following additional columns in the migrations table: 198 | 199 | ```clojure 200 | [:applied "timestamp" "" ""] 201 | [:description "VARCHAR(1024)" "" ""] 202 | ``` 203 | 204 | or 205 | 206 | ```sql 207 | ALTER TABLE migratus.schema_migrations ADD COLUMN description varchar(1024); 208 | --;; 209 | ALTER TABLE migratus.schema_migrations ADD COLUMN applied timestamp with time zone; 210 | ``` 211 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Migratus 2 | 3 | [![CircleCI](https://dl.circleci.com/status-badge/img/gh/yogthos/migratus/tree/master.svg?style=svg)](https://dl.circleci.com/status-badge/redirect/gh/yogthos/migratus/tree/master) 4 | 5 | ![MIGRATE ALL THE THINGS!](https://cdn.rawgit.com/yogthos/migratus/master/migrate.png) 6 | 7 | A general migration framework, with implementations for migrations as SQL 8 | scripts or general Clojure code. 9 | 10 | Designed to be compatible with a git based work flow where multiple topic 11 | branches may exist simultaneously, and be merged into a master branch in 12 | unpredictable order. 13 | 14 | This is accomplished two ways: 15 | 16 | 1. Migration ids are not assumed to be incremented integers. It is recommended that they be timestamps (e.g. '20111202091200'). 17 | 2. Migrations are considered for completion independently. 18 | 19 | Using a 14 digit timestamp will accommodate migrations granularity to a second, 20 | reducing the chance of collisions for a distributed team. 21 | 22 | In contrast, using a single global version for a store and incremented 23 | integers for migration versions, it is possible for a higher numbered 24 | migration to get merged to master and deployed before a lower numbered 25 | migration, in which case the lower numbered migration would never get run, 26 | unless it is renumbered. 27 | 28 | Migratus does not use a single global version for a store. It considers each 29 | migration independently, and runs all uncompleted migrations in sorted order. 30 | 31 | ## Quick Start 32 | 33 | - add the Migratus dependency: 34 | 35 | [![Clojars Project](https://img.shields.io/clojars/v/migratus.svg)](https://clojars.org/migratus) 36 | [![Open Source Helpers](https://www.codetriage.com/yogthos/migratus/badges/users.svg)](https://www.codetriage.com/yogthos/migratus) 37 | 38 | - Add the following code to 39 | `resources/migrations/20111206154000-create-foo-table.up.sql` 40 | 41 | `CREATE TABLE IF NOT EXISTS foo(id BIGINT);` 42 | 43 | - Add the following code to 44 | `resources/migrations/20111206154000-create-foo-table.down.sql` 45 | 46 | `DROP TABLE IF EXISTS foo;` 47 | 48 | ### Multiple Statements 49 | 50 | If you would like to run multiple statements in your migration, then 51 | separate them with `--;;`. For example: 52 | 53 | ```sql 54 | CREATE TABLE IF NOT EXISTS quux(id bigint, name varchar(255)); 55 | --;; 56 | CREATE INDEX quux_name on quux(name); 57 | ``` 58 | 59 | This is necessary because JDBC does not have a method that allows you to 60 | send multiple SQL commands for execution. Migratus will split your 61 | commands, and attempt to execute them inside of a transaction. 62 | 63 | Note that some databases, such as MySQL, do not support transactional DDL 64 | commands. If you're working with such a database then it will not be able 65 | to rollback all the DDL statements that were applied in case a statement 66 | fails. 67 | 68 | ### Disabling transactions 69 | 70 | Migratus attempts to run migrations within a transaction by default. 71 | However, some databases do not support transactional DDL statements. 72 | Transactions can be disabled by adding the following line at the start 73 | of the migration file: 74 | 75 | ```sql 76 | -- :disable-transaction 77 | ``` 78 | 79 | ### Running Functions in Migrations 80 | 81 | Functions inside migrations may need to be additionally wrapped, a PostgreSQL example would look as follows: 82 | 83 | ```sql 84 | DO $func$ 85 | BEGIN 86 | PERFORM schema_name.function_name('foo', 10); 87 | END;$func$; 88 | ``` 89 | 90 | ### Supporting `use` statements 91 | 92 | To run migrations against several different databases (in MySQL, or "schemas" in Postgres, etc.), with embedded `use` statements in your migrations, specify the database in your migration-table-name in the connections, i.e. `database_name.table_name` not `table_name`. 93 | 94 | ### Property substitution 95 | 96 | Migratus supports property substitution where migration files can contain placeholders with the format of `${property.name}`, these placeholders will be replaced with values found in the environment as a result of calling `(System/getenv)`. 97 | 98 | Shell variables will be normalized into Java properties style by being lower cased and with `_` being transformed into `.`, e.g: 99 | 100 | ``` 101 | FOO_BAR - > foo.bar 102 | ``` 103 | 104 | This feature is enabled when the `:properties` flag is set in the configuration. 105 | 106 | Migratus will look for the following default properties: 107 | 108 | * `migratus.schema` 109 | * `migratus.user` 110 | * `migratus.database` 111 | * `migratus.timestamp` (defaults to the value of `(.getTime (java.util.Date.))`) 112 | 113 | Additional property can be specified using the `:env` key or by providing a map of custom properties using the `:env` key: 114 | 115 | ```clojure 116 | {:store :database 117 | :properties {:env ["database.table"] 118 | :map {:database {:user "bob"}}} 119 | :db {:dbtype "h2" 120 | :dbname "site.db"}} 121 | ``` 122 | 123 | For example, given the following template: 124 | 125 | ```sql 126 | GRANT SELECT,INSERT ON ${database.table} TO ${database.user}; 127 | ``` 128 | 129 | The environment variable associated with the `database.table` key will replace `${database.table}` tag in the template, while `{:database {:user "bob"}}` will replace `${database.user}` with `"bob"`. 130 | 131 | ### Setup 132 | 133 | - Add Migratus as a dependency to your `project.clj` 134 | ```clojure 135 | :dependencies [[migratus ]] 136 | ``` 137 | 138 | Next, create a namespace to manage the migrations: 139 | 140 | ```clojure 141 | (ns my-migrations 142 | (:require [migratus.core :as migratus])) 143 | 144 | (def config {:store :database 145 | :migration-dir "migrations/" 146 | :init-script "init.sql" ;script should be located in the :migration-dir path 147 | ;defaults to true, some databases do not support 148 | ;schema initialization in a transaction 149 | :init-in-transaction? false 150 | :migration-table-name "foo_bar" 151 | :db {:dbtype "h2" 152 | :dbname "site.db"}}) 153 | 154 | ;initialize the database using the 'init.sql' script 155 | (migratus/init config) 156 | 157 | ;apply pending migrations 158 | (migratus/migrate config) 159 | 160 | ;rollback the migration with the latest timestamp 161 | (migratus/rollback config) 162 | 163 | ;bring up migrations matching the ids 164 | (migratus/up config 20111206154000) 165 | 166 | ;bring down migrations matching the ids 167 | (migratus/down config 20111206154000) 168 | ``` 169 | 170 | #### Alternative setup 171 | 172 | It is possible to pass a `java.sql.Connection` or `javax.sql.DataSource` in place of a db spec map, e.g: 173 | 174 | ```clojure 175 | (ns my-migrations 176 | (:require [next.jdbc :as jdbc])) 177 | 178 | (def connection (jdbc/get-connection 179 | {:dbtype "h2" 180 | :dbname "site.db"})) 181 | 182 | (def config {:db {:connection connection}}) 183 | 184 | ;; With next.jdbc >= 1.2.790 you can use {:connection-uri ...} format (as well as raw {:datasource ...} without :user/:password). 185 | (def config {:db {:connection-uri ...}}) 186 | 187 | ;; Migratus will close the connection by default 188 | ;; providing :managed-connection? hint allows managing the state of the connection externally 189 | ;; in case you wish to reuse the connection for other purposes 190 | (def config {:connection conn :managed-connection? true}) 191 | 192 | ``` 193 | 194 | ```clojure 195 | (ns my-migrations 196 | (:require [hikari-cp.core :as hk])) 197 | ;; Hikari: https://github.com/tomekw/hikari-cp 198 | 199 | (def datasource-options {:adapter "h2" 200 | :url "jdbc:h2:./site.db"}) 201 | 202 | (def config {:db {:datasource (hk/make-datasource datasource-options)}}) 203 | ``` 204 | 205 | #### Running as native image (Postgres only) 206 | 207 | [PGMig](https://github.com/leafclick/pgmig) is a standalone tool built with migratus that's compiled as a standalone GraalVM native image executable. 208 | 209 | ### Generate migration files 210 | 211 | Migratus also provides a convenience function for creating migration files: 212 | 213 | ```clojure 214 | (migratus/create config "create-user") 215 | ;; minimal config needed to call create while specifying the destination path 216 | (migratus/create {:migration-dir "migrations"} "create-user") 217 | ``` 218 | 219 | This will result with up/down migration files being created prefixed with the current timestamp, e.g: 220 | 221 | ``` 222 | 20150701134958-create-user.up.sql 223 | 20150701134958-create-user.down.sql 224 | ``` 225 | 226 | ## Code-based Migrations 227 | 228 | Application developers often encounter situations where migrations cannot be easily expressed as a SQL script. For instance: 229 | 230 | - Executing programmatically-generated DDL statements 231 | (e.g. updating the schema of a dynamically-sharded table). 232 | - Transferring data between database servers. 233 | - Backfilling existing records with information that must be 234 | retrieved from an external system. 235 | 236 | A common approach in these scenarios is to write one-off scripts which an admin must manually apply for each instance of the application, but issues arise if a script is not run or run multiple times. 237 | 238 | Migratus addresses this problem by providing support for code-based migrations. You can write a migration as a Clojure function, and Migratus will ensure that it's run exactly once for each instance of the application. 239 | 240 | ### Defining a code-based migration 241 | 242 | Create a code-based migration by adding a `.edn` file to your migrations directory that contains the namespace and up/down functions to run, e.g. `resources/migrations/20170331141500-import-users.edn`: 243 | 244 | ```clojure 245 | {:ns app.migrations.import-users 246 | :up-fn migrate-up 247 | :down-fn migrate-down 248 | :transaction? true} 249 | ``` 250 | 251 | Migrations will run within a transaction by default, set `transaction?` to `false` to disable transactions. 252 | 253 | Then, in `src/app/migrations/import_users.clj`: 254 | 255 | ```clojure 256 | (ns app.migrations.import-users) 257 | 258 | (defn migrate-up [config] 259 | ;; do stuff here 260 | ) 261 | 262 | (defn migrate-down [config] 263 | ;; maybe undo stuff here 264 | ) 265 | ``` 266 | 267 | - The up and down migration functions should both accept a single 268 | parameter, which is the config map passed to Migratus (so your 269 | migrations can be configurable). 270 | - You can omit the up or down migration by setting `:up-fn` or 271 | `down-fn` to `nil` in the EDN file. 272 | - The `:up-fn` and `:down-fn` entries can optionally be a vector containing the 273 | migration function followed by additional args to be passed after 274 | the config map, e.g. `{..., :up-fn [migrate-up "arg1" :arg2], ...}`. 275 | 276 | ### Generate code-based migration files 277 | 278 | The `migratus.core/create` function accepts an optional type parameter, which you can pass as `:edn` to create a new migration file. 279 | 280 | ```clojure 281 | (migratus/create config "import-users" :edn) 282 | ``` 283 | 284 | ### Mixing SQL and code-based migrations 285 | 286 | You can include both SQL and code-based migrations in the same migrations directory, in which case they will be run intermixed in the order defined by their timestamps and their status stored in the same table in the migrations database. This way if there are dependencies between your SQL and code-based migrations, you can be assured that they'll run in the correct order. 287 | 288 | ## Quick Start with Leiningen 289 | 290 | Migratus provides a Leiningen plugin: 291 | 292 | - Add migratus-lein as a plugin in addition to the Migratus dependency: 293 | 294 | [![Clojars Project](https://img.shields.io/clojars/v/migratus-lein.svg)](https://clojars.org/migratus-lein) 295 | 296 | - Add the following key and value to your project.clj: 297 | 298 | ```clojure 299 | :migratus {:store :database 300 | :migration-dir "migrations" 301 | :db {:dbtype "mysql" 302 | :dbname "//localhost/migratus" 303 | :user "root" 304 | :password ""}} 305 | ``` 306 | 307 | To apply pending migrations: 308 | 309 | - Run `lein migratus migrate` 310 | 311 | To rollback the migration with the last creation timestamp: 312 | 313 | - Run `lein migratus rollback` 314 | 315 | Then follow the rest of the above instructions. 316 | 317 | ## Configuration 318 | 319 | Migratus is configured via a configuration map that you pass in as its first parameter. The `:store` key describes the type of store against which migrations should be run. All other keys/values in the configuration map are store specific. 320 | 321 | ### Databases 322 | 323 | To run migrations against a database use a :store of :database, and specify the database connection configuration in the :db key of the configuration map. 324 | 325 | * `:migration-dir` - directory where migration files are found 326 | * `:exclude-scripts` - a collection of script name globs that will be excluded from migrations 327 | * `:db` - One of `java.sql.Connection` or `javax.sql.DataSource` instance or a `next.jdbc` database spec. See next.jdbc docs for the version you are using: https://cljdoc.org/d/com.github.seancorfield/next.jdbc/1.2.780/api/next.jdbc#get-datasource 328 | * `:command-separator` - the separator will be used to split the commands within each transaction when specified 329 | * `:expect-results?` - allows comparing migration query results using the `-- expect n` comment 330 | * `:tx-handles-ddl?` - skips the automatic down that occurs on exception 331 | * `:init-script` - string pointing to a script that should be run when the database is initialized 332 | * `:init-in-transaction?` - defaults to true, but some databases do not support schema initialization in a transaction 333 | * `:migration-table-name` - string specifying a custom name for the migration table, defaults to `schema_migrations` 334 | 335 | #### example configurations 336 | 337 | ```clojure 338 | {:store :database 339 | :migration-dir "migrations" 340 | :exclude-scripts ["*.clj"] 341 | :db {:dbtype "mysql" 342 | :dbname "migratus" 343 | :user "root" 344 | :password ""}} 345 | ``` 346 | 347 | The `:migration-dir` key specifies the directory on the classpath in which to find SQL migration files. Each file should be named with the following pattern `[id]-[name].[direction].sql` where id is a unique integer `id` (ideally it should be a timestamp) for the migration, name is some human readable description of the migration, and direction is either `up` or `down`. 348 | 349 | When the `expect-results?` key is set in the config, an assertion can be added to the migrations to check that the expected number of rows was updated: 350 | 351 | ```sql 352 | -- expect 17;; 353 | update foobar set thing = 'c' where thing = 'a'; 354 | 355 | --;; 356 | 357 | -- expect 1;; 358 | delete from foobar where thing = 'c'; 359 | ``` 360 | 361 | If Migratus is trying to run either the up or down migration and it does not exist, then an Exception will be thrown. 362 | 363 | See test/migrations in this repository for an example of how database migrations work. 364 | 365 | ### Modify sql fn 366 | 367 | If you want to do some processing of the sql before it gets executed, you can provide a `:modify-sql-fn` in the config data structure to do so. 368 | It expects a sql-string and can return either a modified sql-string or a sequence of sql-strings. 369 | This is intended for use with http://2ndquadrant.com/en/resources/pglogical/ and similar systems, where DDL statements need to be executed via an extension-provided function. 370 | 371 | ## Usage 372 | Migratus can be used programmatically by calling one of the following functions: 373 | 374 | | Function | Description | 375 | |-----------------------------------------|--------------------------| 376 | | `migratus.core/init` | Runs a script to initialize the database, e.g: create a new schema. | 377 | | `migratus.core/create` | Create a new migration with the current date. | 378 | | `migratus.core/migrate` | Run `up` for any migrations that have not been run. Returns `nil` if successful, `:ignore` if the table is reserved. Supports thread cancellation. | 379 | | `migratus.core/rollback` | Run `down` for the last migration that was run. | 380 | | `migratus.core/rollback-until-just-after` | Run `down` all migrations after `migration-id`. This only considers completed migrations, and will not migrate up. | 381 | | `migratus.core/up` | Run `up` for the specified migration ids. Will skip any migration that is already up. | 382 | | `migratus.core/down` | Run `down` for the specified migration ids. Will skip any migration that is already down. 383 | | `migratus.core/reset` | Reset the database by down-ing all migrations successfully applied, then up-ing all migrations. 384 | | `migratus.core/pending-list` | Returns a list of pending migrations. | 385 | | `migratus.core/migrate-until-just-before` | Run `up` for for any pending migrations which precede the given migration id (good for testing migrations). | 386 | | `migratus.core/squashing-list` | Takes from-id and to-id as inputs (both inclusive) and returns the list of migrations to be squashed. 387 | | `migratus.core/create-squash` | Reads files within the specified id range and generates a new squashed migration file. It removes the original migration files and creates a new one with the last ID from the range and a user-provided name. 388 | | `migratus.core/squash-between` | Updates the migration table by marking the old IDs as squashed and replacing them with the new ID. No actual migration is applied, assuming they were previously applied. 389 | 390 | See the docstrings of each function for more details. 391 | 392 | Migratus can also be used from leiningen if you add it as a plugin dependency. 393 | 394 | ```clojure 395 | :plugins [[migratus-lein ]] 396 | ``` 397 | 398 | And add a configuration :migratus key to your `project.clj`. 399 | 400 | ```clojure 401 | :migratus {:store :database 402 | :migration-dir "migrations" 403 | :db {:dbtype "mysql" 404 | :dbname "migratus" 405 | :user "root" 406 | :password ""}} 407 | ``` 408 | 409 | You can then run the following tasks: 410 | 411 | | Task | Description | 412 | |-----------------------------|--------------------------------------------------------------------------------------------| 413 | | lein migratus create | Create a new migration with the current date. | 414 | | lein migratus migrate | Run 'up' for any migrations that have not been run. | 415 | | lein migratus rollback | Run 'down' for the last migration that was run. | 416 | | lein migratus up & ids | Run 'up' for the specified migration ids. Will skip any migration that is already up. | 417 | | lein migratus down & ids | Run 'down' for the specified migration ids. Will skip any migration that is already down. | 418 | | lein migratus reset | Run 'down' for all migrations that have been run, and 'up' for all migrations. | 419 | | lein migratus pending | Run 'pending-list' to get all pending migrations. | 420 | 421 | ## Quickstart with native Clojure projects 422 | 423 | See [clj-migratus](https://github.com/paulbutcher/clj-migratus) for more information. 424 | 425 | ## Usage 426 | 427 | Add the following to your `deps.edn`: 428 | 429 | ``` 430 | :aliases {:migrate {:extra-deps {com.github.paulbutcher/clj-migratus {:git/tag "v1.0.0" 431 | :git/sha "67d0fe5"}} 432 | :main-opts ["-m" "clj-migratus"]}} 433 | ``` 434 | 435 | Create a [Migratus configuration](https://github.com/yogthos/migratus#configuration) file. This can either be `migratus.edn`: 436 | 437 | ``` 438 | {:store :database 439 | :migration-dir "migrations" 440 | :db {:dbtype "mysql" 441 | :dbname "migratus" 442 | :user "root" 443 | :password ""}} 444 | ``` 445 | 446 | Or (recommended) `migratus.clj`, allowing credentials to be taken from the environment: 447 | 448 | ``` 449 | {:store :database 450 | :db {:jdbcUrl (get (System/getenv) "JDBC_DATABASE_URL")}} 451 | ``` 452 | 453 | Then run, for example: 454 | 455 | ``` 456 | $ clj -M:migrate init 457 | $ clj -M:migrate migrate 458 | $ clj -M:migrate create create-user-table 459 | ``` 460 | 461 | See [Migratus Usage](https://github.com/yogthos/migratus#usage) for documentation on each command. 462 | 463 | ## Working on migratus itself 464 | 465 | You can use either `lein` or `clj` for now as it has both project definitions. 466 | 467 | Run tests with kaocha: 468 | 469 | ``` 470 | # https://cljdoc.org/d/lambdaisland/kaocha/1.68.1059/doc/4-running-kaocha-cli 471 | 472 | bin/kaocha --test-help 473 | 474 | bin/kaocha --fail-fast 475 | 476 | bin/kaocha --fail-fast --focus migratus.test.migration.sql/test-run-sql-migrations 477 | 478 | # Run only integration tests - defined in tests.edn 479 | bin/kaocha testcontainers 480 | ``` 481 | 482 | ## License 483 | 484 | Copyright © 2016 Paul Stadig, Dmitri Sotnikov 485 | 486 | Licensed under the Apache License, Version 2.0. 487 | -------------------------------------------------------------------------------- /bin/kaocha: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | clojure -J-Dclojure.main.report=stderr -Sforce -M:test-runner:dev "$@" 4 | -------------------------------------------------------------------------------- /deps.edn: -------------------------------------------------------------------------------- 1 | {:paths ["src"] 2 | :description "MIGRATE ALL THE THINGS!" 3 | :url "http://github.com/yogthos/migratus" 4 | :license {:name "Apache License, Version 2.0" 5 | :url "http://www.apache.org/licenses/LICENSE-2.0.html" 6 | :distribution :repo} 7 | :deps {com.github.seancorfield/next.jdbc {:mvn/version "1.2.790"} 8 | org.clojure/tools.logging {:mvn/version "1.1.0"} 9 | org.clojure/tools.cli {:mvn/version "1.0.219"} 10 | org.clojure/data.json {:mvn/version "2.5.0"}} 11 | :aliases {:clojure1.10 {:extra-deps 12 | {org.clojure/clojure {:mvn/version "1.10.1"}}} 13 | :clojure1.11 {:extra-deps 14 | {org.clojure/clojure {:mvn/version "1.11.1"}}} 15 | :dev {:extra-paths ["test"] 16 | :extra-deps 17 | {jar-migrations/jar-migrations {:mvn/version "1.0.0"} 18 | ch.qos.logback/logback-classic {:mvn/version "1.2.3"} 19 | clj-test-containers/clj-test-containers {:mvn/version "0.7.1"} 20 | com.h2database/h2 {:mvn/version "2.1.214"} 21 | hikari-cp/hikari-cp {:mvn/version "2.13.0"} 22 | org.clojure/tools.trace {:mvn/version "0.7.11"} 23 | org.postgresql/postgresql {:mvn/version "42.2.5"}}} 24 | :test-runner {:extra-paths ["test"] 25 | :extra-deps 26 | {lambdaisland/kaocha {:mvn/version "1.66.1034"} 27 | lambdaisland/kaocha-cloverage {:mvn/version "1.0.75"} 28 | lambdaisland/kaocha-junit-xml {:mvn/version "0.0.76"} 29 | orchestra/orchestra {:mvn/version "2021.01.01-1"}} 30 | :main-opts ["-m" "kaocha.runner" "--reporter" "kaocha.report/documentation"]}}} 31 | -------------------------------------------------------------------------------- /migrate.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yogthos/migratus/b495090fb8123b6d5ff5744eb1bce1a147d6ff67/migrate.png -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (defproject migratus "1.6.4" 2 | :description "MIGRATE ALL THE THINGS!" 3 | :url "http://github.com/yogthos/migratus" 4 | :license {:name "Apache License, Version 2.0" 5 | :url "http://www.apache.org/licenses/LICENSE-2.0.html" 6 | :distribution :repo} 7 | :aliases {"test!" ["do" "clean," "test"]} 8 | :dependencies [[com.github.seancorfield/next.jdbc "1.3.955"] 9 | [org.clojure/clojure "1.10.1"] 10 | [org.clojure/tools.logging "1.1.0"] 11 | [org.clojure/tools.cli "1.0.219"] 12 | [org.clojure/data.json "2.5.0"]] 13 | :profiles {:dev {:dependencies [[jar-migrations "1.0.0"] 14 | [ch.qos.logback/logback-classic "1.2.3"] 15 | [clj-test-containers/clj-test-containers "0.7.1"] 16 | [com.h2database/h2 "2.1.214"] 17 | [hikari-cp/hikari-cp "2.13.0"] 18 | [org.clojure/tools.trace "0.7.11"] 19 | [org.postgresql/postgresql "42.2.5"]]}}) 20 | -------------------------------------------------------------------------------- /release.cljs: -------------------------------------------------------------------------------- 1 | (ns release.core 2 | (:require ["fs" :as fs] 3 | [clojure.edn :as edn] 4 | [clojure.string :refer [replace-first split]])) 5 | 6 | (def exec (.-execSync (js/require "child_process"))) 7 | 8 | (defn bump-version [v] 9 | (let [[x y z] (map js/parseInt (split v #"\."))] 10 | (cond 11 | (= 9 y z) (str (inc x) ".0.0") 12 | (= 9 z) (str x "." (inc y) ".0") 13 | :else (str x "." y "." (inc z))))) 14 | 15 | (defn write-version [project version] 16 | (fs/writeFileSync 17 | "project.clj" 18 | (replace-first project #"\d+\.\d+\.\d+" version))) 19 | 20 | (defn increment-version [] 21 | (let [project (->> "project.clj" (fs/readFileSync) (str)) 22 | version (->> project (edn/read-string) (vec) (drop 2) first bump-version)] 23 | (write-version project version) 24 | version)) 25 | 26 | (defn run [command] 27 | ( println (str (exec command)))) 28 | 29 | (let [version (increment-version)] 30 | (println "releasing version:" version) 31 | (run (str "git commit -a -m \"release version " version "\"")) 32 | (run "git push") 33 | (run (str "git tag -a v" version " -m \"release " version "\"" )) 34 | (run "git push --tags")) 35 | -------------------------------------------------------------------------------- /src/migratus/cli.clj: -------------------------------------------------------------------------------- 1 | (ns migratus.cli 2 | (:refer-clojure :exclude [parse-long]) 3 | (:require [clojure.data.json :as json] 4 | [clojure.core :as core] 5 | [clojure.java.io :as io] 6 | [clojure.string :as str] 7 | [clojure.tools.cli :refer [parse-opts]] 8 | [clojure.tools.logging :as log] 9 | [migratus.core :as migratus]) 10 | (:import [java.time ZoneId ZoneOffset] 11 | [java.time.format DateTimeFormatter] 12 | [java.util.logging 13 | ConsoleHandler 14 | Formatter 15 | LogRecord 16 | Logger 17 | SimpleFormatter])) 18 | 19 | ;; needed fro Clojure 1.10 compatibility 20 | (defn parse-long [s] 21 | (Long/valueOf s)) 22 | 23 | (defn validate-format [s] 24 | (boolean (some (set (list s)) #{"plain" "edn" "json"}))) 25 | 26 | (def global-cli-options 27 | [[nil "--config NAME" "Configuration file name" :default "migratus.edn"] 28 | ["-v" nil "Verbosity level; may be specified multiple times to increase value" 29 | :id :verbosity 30 | :default 0 31 | :update-fn inc] 32 | ["-h" "--help"]]) 33 | 34 | (def migrate-cli-options 35 | [[nil "--until-just-before MIGRATION-ID" "Run all migrations preceding migration-id. This is useful when testing that a migration behaves as expected on fixture data. This only considers uncompleted migrations, and will not migrate down."] 36 | ["-h" "--help"]]) 37 | 38 | (def rollback-cli-options 39 | [[nil "--until-just-after MIGRATION-ID" "Migrate down all migrations after migration-id. This only considers completed migrations, and will not migrate up."] 40 | ["-h" "--help"]]) 41 | 42 | (def list-cli-options 43 | [[nil "--available" "List all migrations, applied and non applied"] 44 | [nil "--pending" "List pending migrations"] 45 | [nil "--applied" "List applied migrations"] 46 | [nil "--format FORMAT" "Option to print in plain text (default), edn or json" :default "plain" 47 | :validate [#(validate-format %) "Unsupported format. Valid options: plain (default), edn, json."]] 48 | ["-h" "--help"]]) 49 | 50 | (defn usage [options-summary] 51 | (->> ["Usage: migratus action [options]" 52 | "" 53 | "Actions:" 54 | " init" 55 | " create" 56 | " migrate" 57 | " reset" 58 | " rollback" 59 | " up" 60 | " down" 61 | " list" 62 | "" 63 | "options:" 64 | options-summary] 65 | (str/join \newline))) 66 | 67 | (defn error-msg [errors] 68 | (binding [*out* *err*] 69 | (println "The following errors occurred while parsing your command:\n\n" 70 | (str/join \newline errors)))) 71 | 72 | (defn no-match-message 73 | "No matching clause message info" 74 | [arguments summary] 75 | (binding [*out* *err*] 76 | (println "Migratus API does not support this action(s) : " arguments "\n\n" 77 | (str/join (usage summary))))) 78 | 79 | (defn run-migrate [cfg args] 80 | (let [{:keys [options arguments errors summary]} (parse-opts args migrate-cli-options :in-order true) 81 | rest-args (rest arguments)] 82 | 83 | (cond 84 | errors (error-msg errors) 85 | (:until-just-before options) 86 | (do (log/debug "configuration is: \n" cfg "\n" 87 | "arguments:" rest-args) 88 | (migratus/migrate-until-just-before cfg rest-args)) 89 | (empty? args) 90 | (do (log/debug "calling (migrate cfg)" cfg) 91 | (migratus/migrate cfg)) 92 | :else (no-match-message args summary)))) 93 | 94 | (defn run-rollback [cfg args] 95 | (let [{:keys [options arguments errors summary]} (parse-opts args rollback-cli-options :in-order true) 96 | rest-args (rest arguments)] 97 | 98 | (cond 99 | errors (error-msg errors) 100 | 101 | (:until-just-after options) 102 | (do (log/debug "configuration is: \n" cfg "\n" 103 | "args:" rest-args) 104 | (migratus/rollback-until-just-after cfg rest-args)) 105 | 106 | (empty? args) 107 | (do (log/debug "configuration is: \n" cfg) 108 | (migratus/rollback cfg)) 109 | 110 | :else (no-match-message args summary)))) 111 | 112 | (defn util-date-to-local-datetime [util-date] 113 | (when (some? util-date) 114 | (let [instant (.toInstant util-date) 115 | zone-id (ZoneId/systemDefault) 116 | local-datetime (.atZone instant zone-id)] 117 | local-datetime))) 118 | 119 | (defn parse-migration-applied-date [m] 120 | (let [{:keys [id name applied]} m 121 | local-date (when (some? applied) 122 | (-> 123 | (util-date-to-local-datetime applied) 124 | (.format DateTimeFormatter/ISO_LOCAL_DATE_TIME)))] 125 | {:id id :name name :applied local-date})) 126 | 127 | (defn parsed-migrations-data [cfg] 128 | (let [all-migrations (migratus/all-migrations cfg)] 129 | (map parse-migration-applied-date all-migrations))) 130 | 131 | (defn pending-migrations [cfg] 132 | (let [keep-pending-migs (fn [mig] (nil? (:applied mig)))] 133 | (filter keep-pending-migs (parsed-migrations-data cfg)))) 134 | 135 | (defn applied-migrations [cfg] 136 | (let [keep-applied-migs (fn [mig] (not= nil (:applied mig)))] 137 | (filter keep-applied-migs (parsed-migrations-data cfg)))) 138 | 139 | (defn col-width 140 | "Set column width for CLI table" 141 | [n] 142 | (apply str (repeat n "-"))) 143 | 144 | (defn table-line [n] 145 | (let [str (str "%-" n "s")] 146 | (core/format str, (col-width n)))) 147 | 148 | (defn format-mig-data [m] 149 | (let [{:keys [id name applied]} m 150 | applied? (if (nil? applied) 151 | "pending" 152 | applied) 153 | fmt-str "%1$-15s | %2$-22s | %3$-20s"] 154 | (println (core/format fmt-str, id, name, applied?)))) 155 | 156 | (defn format-pending-mig-data [m] 157 | (let [{:keys [id name]} m 158 | fmt-str "%1$-15s| %2$-22s%3$s"] 159 | (println (core/format fmt-str, id, name, )))) 160 | 161 | (defn mig-print-fmt [data & format-opts] 162 | (let [pending? (:pending format-opts)] 163 | (if pending? 164 | (do (println (table-line 43)) 165 | (println (core/format "%-15s%-24s", 166 | "MIGRATION-ID" "| NAME")) 167 | (println (table-line 41)) 168 | (doseq [d data] (format-pending-mig-data d))) 169 | (do (println (table-line 67)) 170 | (println (core/format "%-16s%-25s%-22s", 171 | "MIGRATION-ID" "| NAME" "| APPLIED")) 172 | (println (table-line 67)) 173 | (doseq [d data] (format-mig-data d)))))) 174 | 175 | (defn cli-print-migs! [data f & format-opts] 176 | (case f 177 | "plain" (mig-print-fmt data format-opts) 178 | "edn" (println data) 179 | "json" (println (json/write-str data)) 180 | nil)) 181 | 182 | (defn list-pending-migrations [migs format] 183 | (cli-print-migs! migs format {:pending true})) 184 | 185 | (defn run-list [cfg args] 186 | (let [{:keys [options errors summary]} (parse-opts args list-cli-options :in-order true) 187 | {:keys [available pending applied]} options 188 | {f :format} options] 189 | (cond 190 | errors (error-msg errors) 191 | applied (let [applied-migs (applied-migrations cfg)] 192 | (cli-print-migs! applied-migs f)) 193 | pending (let [pending-migs (pending-migrations cfg)] 194 | (list-pending-migrations pending-migs f)) 195 | available (let [available-migs (parsed-migrations-data cfg)] 196 | (cli-print-migs! available-migs f)) 197 | (or (empty? args) f) (let [pending-migs (pending-migrations cfg)] 198 | (list-pending-migrations pending-migs f)) 199 | :else (no-match-message args summary)))) 200 | 201 | (defn simple-formatter 202 | "Clojure bridge for java.util.logging.SimpleFormatter. 203 | Can register a clojure fn as a logger formatter. 204 | 205 | * format-fn - clojure fn that receives the record to send to logging." 206 | (^SimpleFormatter [format-fn] 207 | (proxy [SimpleFormatter] [] 208 | (format [record] 209 | (format-fn record))))) 210 | 211 | (defn format-log-record 212 | "Format jul logger record." 213 | (^String [^LogRecord record] 214 | (let [fmt "%5$s" 215 | instant (.getInstant record) 216 | date (-> instant (.atZone ZoneOffset/UTC)) 217 | level (.getLevel record) 218 | src (.getSourceClassName record) 219 | msg (.getMessage record) 220 | thr (.getThrown record) 221 | logger (.getLoggerName record)] 222 | (core/format fmt date src logger level msg thr)))) 223 | 224 | (defn verbose-log-level [v] 225 | (case v 226 | 0 java.util.logging.Level/INFO ;; :info 227 | 1 java.util.logging.Level/FINE ;; :debug 228 | java.util.logging.Level/FINEST)) ;; :trace 229 | 230 | (defn set-logger-format 231 | "Configure JUL logger to use a custom log formatter. 232 | 233 | * formatter - instance of java.util.logging.Formatter" 234 | ([verbosity] 235 | (set-logger-format verbosity (simple-formatter format-log-record))) 236 | ([verbosity ^Formatter formatter] 237 | (let [main-logger (doto (Logger/getLogger "") 238 | (.setUseParentHandlers false) 239 | (.setLevel (verbose-log-level verbosity))) 240 | handler (doto (ConsoleHandler.) 241 | (.setFormatter formatter) 242 | (.setLevel (verbose-log-level verbosity))) 243 | handlers (.getHandlers main-logger)] 244 | (doseq [h handlers] 245 | (.removeHandler main-logger h)) 246 | (.addHandler main-logger handler)))) 247 | 248 | (defn load-config! 249 | "Returns the content of config file as a clojure map datastructure" 250 | [^String config] 251 | (let [config-path (.getAbsolutePath (io/file config))] 252 | (try 253 | (read-string (slurp config-path)) 254 | (catch java.io.FileNotFoundException e 255 | (binding [*out* *err*] 256 | (println "Missing config file" (.getMessage e) 257 | "\nYou can use --config path_to_file to specify a path to config file")))))) 258 | 259 | (defn up [cfg args] 260 | (if (empty? args) 261 | (binding [*out* *err*] 262 | (println "To run action up you must provide a migration-id as a parameter: 263 | up ")) 264 | (->> args 265 | (map #(parse-long %)) 266 | (apply migratus/up cfg)))) 267 | 268 | (defn down [cfg args] 269 | (if (empty? args) 270 | (binding [*out* *err*] 271 | (println "To run action down you must provide a migration-id as a parameter: 272 | down ")) 273 | (->> args 274 | (map #(parse-long %)) 275 | (apply migratus/down cfg)))) 276 | 277 | (defn -main [& args] 278 | (let [{:keys [options arguments _errors summary]} (parse-opts args global-cli-options :in-order true) 279 | config (:config options) 280 | verbosity (:verbosity options) 281 | cfg (load-config! config) 282 | action (first arguments) 283 | rest-args (rest arguments)] 284 | (set-logger-format verbosity) 285 | (cond 286 | (:help options) (usage summary) 287 | (nil? (:config options)) (error-msg "No config provided \n --config [file-name]>") 288 | :else (case action 289 | "init" (migratus/init cfg) 290 | "create" (migratus/create cfg (second arguments)) 291 | "migrate" (run-migrate cfg rest-args) 292 | "rollback" (run-rollback cfg rest-args) 293 | "reset" (migratus/reset cfg) 294 | "up" (up cfg rest-args) 295 | "down" (down cfg rest-args) 296 | "list" (run-list cfg rest-args) 297 | (no-match-message arguments summary))))) 298 | 299 | -------------------------------------------------------------------------------- /src/migratus/core.clj: -------------------------------------------------------------------------------- 1 | ;;;; Copyright © 2011 Paul Stadig 2 | ;;;; 3 | ;;;; Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | ;;;; use this file except in compliance with the License. You may obtain a copy 5 | ;;;; of the License at 6 | ;;;; 7 | ;;;; http://www.apache.org/licenses/LICENSE-2.0 8 | ;;;; 9 | ;;;; Unless required by applicable law or agreed to in writing, software 10 | ;;;; distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | ;;;; WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | ;;;; License for the specific language governing permissions and limitations 13 | ;;;; under the License. 14 | (ns migratus.core 15 | (:require 16 | [clojure.set :as set] 17 | [clojure.string :as str] 18 | [clojure.tools.logging :as log] 19 | [migratus.migrations :as mig] 20 | [migratus.protocols :as proto] 21 | migratus.database 22 | [next.jdbc.transaction :as jdbc-tx])) 23 | 24 | (defmacro ^{:private true} assert-args 25 | [& pairs] 26 | `(do (when-not ~(first pairs) 27 | (throw (IllegalArgumentException. 28 | (str (first ~'&form) " requires " ~(second pairs) " in " ~'*ns* ":" (:line (meta ~'&form)))))) 29 | ~(let [more (nnext pairs)] 30 | (when more 31 | (list* `assert-args more))))) 32 | 33 | (defmacro with-store 34 | "bindings => name init 35 | Evaluates body in a try expression with name bound to the value 36 | of the init, and (proto/connect name) called before body, and a 37 | finally clause that calls (proto/disconnect name)." 38 | ([bindings & body] 39 | (assert-args 40 | (vector? bindings) "a vector for its binding" 41 | (= 2 (count bindings)) "exactly 2 forms in binding vector" 42 | (symbol? (bindings 0)) "only Symbols in bindings") 43 | (let [form (bindings 0) init (bindings 1)] 44 | `(let [~form ~init] 45 | (try 46 | (proto/connect ~form) 47 | ~@body 48 | (finally 49 | (proto/disconnect ~form))))))) 50 | 51 | (defn run [store ids command] 52 | (binding [jdbc-tx/*nested-tx* :ignore] 53 | (try 54 | (log/info "Starting migrations") 55 | (proto/connect store) 56 | (command store ids) 57 | (catch java.sql.BatchUpdateException e 58 | (throw (or (.getNextException e) e))) 59 | (finally 60 | (log/info "Ending migrations") 61 | (proto/disconnect store))))) 62 | 63 | (defn require-plugin [{:keys [store]}] 64 | (when-not store 65 | (throw (Exception. "Store is not configured"))) 66 | (let [plugin (symbol (str "migratus." (name store)))] 67 | (require plugin))) 68 | 69 | (defn completed-migrations [config store] 70 | (let [completed? (set (proto/completed-ids store))] 71 | (filter (comp completed? proto/id) (mig/list-migrations config)))) 72 | 73 | (defn gather-migrations 74 | "Returns a list of all migrations from migration dir and db 75 | with enriched data: 76 | - date and time when was applied; 77 | - migration type (:sql or :edn); 78 | - description;" 79 | [config store] 80 | (let [completed-migrations (vec (proto/completed store)) 81 | available-migrations (->> (mig/list-migrations config) 82 | (map (fn [mig] (assoc mig :mig-type (proto/migration-type mig))))) 83 | merged-migrations-data (apply merge completed-migrations available-migrations) 84 | grouped-migrations-by-id (group-by :id merged-migrations-data) 85 | unify-mig-values (fn [[_ v]] (apply merge v))] 86 | (map unify-mig-values grouped-migrations-by-id))) 87 | 88 | (defn all-migrations [config] 89 | (with-store 90 | [store (proto/make-store config)] 91 | (->> store 92 | (gather-migrations config) 93 | (map (fn [e] {:id (:id e) :name (:name e) :applied (:applied e)}))))) 94 | 95 | (defn migrations-between 96 | "Returns a list of migrations between from(inclusive) and to(inclusive)." 97 | [config from to] 98 | (with-store 99 | [store (proto/make-store config)] 100 | (->> store 101 | (gather-migrations config) 102 | (filter (fn [e] 103 | (and (>= (:id e) from) 104 | (<= (:id e) to)))) 105 | (sort-by :id)))) 106 | 107 | (defn uncompleted-migrations 108 | "Returns a list of uncompleted migrations. 109 | Fetch list of applied migrations from db and existing migrations from migrations dir." 110 | [config store] 111 | (let [completed? (set (proto/completed-ids store))] 112 | (remove (comp completed? proto/id) (mig/list-migrations config)))) 113 | 114 | (defn migration-name [migration] 115 | (str (proto/id migration) "-" (proto/name migration))) 116 | 117 | (defn- up* [store migration] 118 | (log/info "Up" (migration-name migration)) 119 | (proto/migrate-up store migration)) 120 | 121 | (defn- migrate-up* [store migrations] 122 | (let [migrations (sort-by proto/id migrations)] 123 | (when (seq migrations) 124 | (log/info "Running up for" (pr-str (vec (map proto/id migrations)))) 125 | (loop [[migration & more] migrations] 126 | (when migration 127 | (when (Thread/interrupted) 128 | (log/info "Thread cancellation detected. Stopping migration.") 129 | (throw (InterruptedException. "Migration interrupted by thread cancellation."))) 130 | (case (up* store migration) 131 | :success (recur more) 132 | :ignore (do 133 | (log/info "Migration reserved by another instance. Ignoring.") 134 | :ignore) 135 | (do 136 | (log/error "Stopping:" (migration-name migration) "failed to migrate") 137 | :failure))))))) 138 | 139 | (defn- migrate* [config store _] 140 | (let [migrations (->> store 141 | (uncompleted-migrations config) 142 | (sort-by proto/id))] 143 | (migrate-up* store migrations))) 144 | 145 | (defn migrate 146 | "Bring up any migrations that are not completed. 147 | Returns nil if successful, :ignore if the table is reserved, :failure otherwise. 148 | Supports thread cancellation." 149 | [config] 150 | (run (proto/make-store config) nil (partial migrate* config))) 151 | 152 | (defn- run-up [config store ids] 153 | (let [completed (set (proto/completed-ids store)) 154 | ids (set/difference (set ids) completed) 155 | migrations (filter (comp ids proto/id) (mig/list-migrations config))] 156 | (migrate-up* store migrations))) 157 | 158 | (defn up 159 | "Bring up the migrations identified by ids. 160 | Any migrations that are already complete will be skipped." 161 | [config & ids] 162 | (run (proto/make-store config) ids (partial run-up config))) 163 | 164 | (defn- run-down [config store ids] 165 | (let [completed (set (proto/completed-ids store)) 166 | ids (set/intersection (set ids) completed) 167 | migrations (filter (comp ids proto/id) 168 | (mig/list-migrations config)) 169 | migrations (reverse (sort-by proto/id migrations))] 170 | (when (seq migrations) 171 | (log/info "Running down for" (pr-str (vec (map proto/id migrations)))) 172 | (doseq [migration migrations] 173 | (log/info "Down" (migration-name migration)) 174 | (proto/migrate-down store migration))))) 175 | 176 | (defn down 177 | "Bring down the migrations identified by ids. 178 | Any migrations that are not completed will be skipped." 179 | [config & ids] 180 | (run (proto/make-store config) ids (partial run-down config))) 181 | 182 | (defn- rollback* [config store _] 183 | (run-down 184 | config 185 | store 186 | (->> (proto/completed-ids store) 187 | first 188 | vector))) 189 | 190 | (defn- reset* [config store _] 191 | (run-down config store (->> (proto/completed-ids store) sort))) 192 | 193 | (defn rollback 194 | "Rollback the last migration that was successfully applied." 195 | [config] 196 | (run (proto/make-store config) nil (partial rollback* config))) 197 | 198 | (defn reset 199 | "Reset the database by down-ing all migrations successfully 200 | applied, then up-ing all migratinos." 201 | [config] 202 | (run (proto/make-store config) nil (partial reset* config)) 203 | (migrate config)) 204 | 205 | (defn init 206 | "Initialize the data store" 207 | [config & [name]] 208 | (proto/init (proto/make-store config))) 209 | 210 | (defn create 211 | "Create a new migration with the current date" 212 | [config & [name type]] 213 | (mig/create config name (or type :sql))) 214 | 215 | (defn create-squash 216 | "Delete all migrations between from and to, 217 | squash them into a single migration with the given name and last applied migration date." 218 | [config & [from-id to-id name]] 219 | (let [migrations (migrations-between config from-id to-id) 220 | ups (str/join "\n--;;\n" (map :up migrations)) 221 | downs (str/join "\n--;;\n" (reverse (map :down migrations))) 222 | id (:id (last migrations))] 223 | (when (not (every? #(= (:mig-type %) :sql) migrations)) 224 | (throw (IllegalArgumentException. "All migrations must be of the same type."))) 225 | (doseq [migration migrations] 226 | (when (not (:applied migration)) 227 | (throw (IllegalArgumentException. (str "Migration " (:id migration) " is not applied. Apply it first.")))) 228 | (mig/destroy config (:name migration))) 229 | (mig/create-squash config id name :sql ups downs))) 230 | 231 | (defn destroy 232 | "Destroy migration" 233 | [config & [name]] 234 | (mig/destroy config name)) 235 | 236 | (defn select-migrations 237 | "List pairs of id and name for migrations selected by the selection-fn." 238 | [config selection-fn] 239 | (with-store [store (proto/make-store config)] 240 | (->> store 241 | (selection-fn config) 242 | (mapv (juxt proto/id proto/name))))) 243 | 244 | (defn completed-list 245 | "List completed migrations" 246 | [config] 247 | (let [migrations (select-migrations config completed-migrations)] 248 | (log/debug (apply str "You have " (count migrations) " completed migrations:\n" 249 | (str/join "\n" migrations))) 250 | (mapv second migrations))) 251 | 252 | (defn pending-list 253 | "List pending migrations" 254 | [config] 255 | (let [migrations (select-migrations config uncompleted-migrations)] 256 | (log/debug (apply str "You have " (count migrations) " pending migrations:\n" 257 | (str/join "\n" migrations))) 258 | (mapv second migrations))) 259 | 260 | (defn squashing-list 261 | "List to be squashed migrations" 262 | [config from to] 263 | (let [migrations (migrations-between config from to)] 264 | (log/debug (apply str "You have " (count migrations) " migrations to be squashed:\n" 265 | (str/join "\n" migrations))) 266 | (doseq [migration migrations] 267 | (when (not (:applied migration)) 268 | (throw (IllegalArgumentException. (str "Migration " (:id migration) " is not applied. Apply it first."))))) 269 | (mapv :name migrations))) 270 | 271 | (defn squash-between 272 | "Squash a batch of migrations into a single migration" 273 | [config from-id to-id name] 274 | (with-store [store (proto/make-store config)] 275 | (let [completed-ids (->> (proto/completed-ids store) 276 | (filter (fn [mig-id] 277 | (and (>= mig-id from-id) 278 | (<= mig-id to-id)))) 279 | (sort >))] 280 | (log/debug (apply str "You have " (count completed-ids) " migrations to be squashed:\n" 281 | (str/join "\n" completed-ids))) 282 | (proto/squash store completed-ids name)))) 283 | 284 | (defn migrate-until-just-before 285 | "Run all migrations preceding migration-id. This is useful when testing that a 286 | migration behaves as expected on fixture data. This only considers uncompleted 287 | migrations, and will not migrate down." 288 | [config migration-id] 289 | (with-store [store (proto/make-store config)] 290 | (->> (uncompleted-migrations config store) 291 | (map proto/id) 292 | distinct 293 | sort 294 | (take-while #(< % migration-id)) 295 | (apply up config)))) 296 | 297 | (defn rollback-until-just-after 298 | "Migrate down all migrations after migration-id. This only considers completed 299 | migrations, and will not migrate up." 300 | [config migration-id] 301 | (with-store [store (proto/make-store config)] 302 | (->> (completed-migrations config store) 303 | (map proto/id) 304 | distinct 305 | sort 306 | reverse 307 | (take-while #(> % migration-id)) 308 | (apply down config)))) 309 | -------------------------------------------------------------------------------- /src/migratus/database.clj: -------------------------------------------------------------------------------- 1 | ;;;; Copyright © 2011 Paul Stadig 2 | ;;;; 3 | ;;;; Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | ;;;; use this file except in compliance with the License. You may obtain a copy 5 | ;;;; of the License at 6 | ;;;; 7 | ;;;; http://www.apache.org/licenses/LICENSE-2.0 8 | ;;;; 9 | ;;;; Unless required by applicable law or agreed to in writing, software 10 | ;;;; distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | ;;;; WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | ;;;; License for the specific language governing permissions and limitations 13 | ;;;; under the License. 14 | (ns migratus.database 15 | (:require [clojure.java.io :as io] 16 | [clojure.tools.logging :as log] 17 | [clojure.string :as str] 18 | [migratus.migration.sql :as sql-mig] 19 | [migratus.properties :as props] 20 | [migratus.protocols :as proto] 21 | [migratus.utils :as utils] 22 | [next.jdbc :as jdbc] 23 | [next.jdbc.result-set :as rs] 24 | [next.jdbc.sql :as sql]) 25 | (:import java.io.File 26 | [java.sql Connection SQLException] 27 | [javax.sql DataSource] 28 | [java.util.jar JarEntry JarFile])) 29 | 30 | (def default-migrations-table "schema_migrations") 31 | 32 | (defn migration-table-name 33 | "Makes migration table name available from config." 34 | [config] 35 | (:migration-table-name config default-migrations-table)) 36 | 37 | (defn connection-or-spec 38 | "Migration code from java.jdbc to next.jdbc . 39 | java.jdbc accepts a spec that contains a ^java.sql.Connection as :connection. 40 | Return :connection or the db spec." 41 | [db] 42 | (let [conn (:connection db)] 43 | (if conn conn db))) 44 | 45 | (def reserved-id -1) 46 | 47 | (defn mark-reserved [db table-name] 48 | (boolean 49 | (try 50 | (sql/insert! (connection-or-spec db) (keyword table-name) {:id reserved-id} {:return-keys false}) 51 | (catch Exception e 52 | (log/infof "mark-reserved failed %s" (ex-message e)))))) 53 | 54 | (defn mark-unreserved [db table-name] 55 | (sql/delete! (connection-or-spec db) (keyword table-name) ["id=?" reserved-id])) 56 | 57 | (defn complete? [db table-name id] 58 | (first (sql/query (connection-or-spec db) 59 | [(str "SELECT * from " table-name " WHERE id=?") id]))) 60 | 61 | (defn complete-all? [db table-name ids] 62 | (when (seq ids) 63 | (not-empty 64 | (sql/query (connection-or-spec db) 65 | [(str "SELECT * FROM " table-name " WHERE id IN (" 66 | (str/join "," ids) 67 | ")")])))) 68 | 69 | (defn mark-complete [db table-name description id] 70 | (log/debug "marking" id "complete") 71 | (sql/insert! (connection-or-spec db) 72 | (keyword table-name) 73 | {:id id 74 | :applied (java.sql.Timestamp. (.getTime (java.util.Date.))) 75 | :description description})) 76 | 77 | (defn mark-not-complete [db table-name id] 78 | (log/debug "marking" id "not complete") 79 | (sql/delete! (connection-or-spec db) table-name ["id=?" id])) 80 | 81 | (defn mark-not-complete-all [db table-name ids] 82 | (log/debug "marking" ids "not complete") 83 | (when (seq ids) 84 | (jdbc/execute! db [(str "DELETE FROM " table-name " WHERE id IN (" 85 | (str/join "," ids) 86 | ")")]))) 87 | 88 | (defn migrate-up* [db {:keys [tx-handles-ddl?] :as config} {:keys [name] :as migration}] 89 | (let [id (proto/id migration) 90 | table-name (migration-table-name config)] 91 | (if (mark-reserved db table-name) 92 | (try 93 | (when-not (complete? db table-name id) 94 | (proto/up migration (assoc config :conn db)) 95 | (mark-complete db table-name name id) 96 | :success) 97 | (catch Throwable up-e 98 | (if tx-handles-ddl? 99 | (log/error (format "Migration %s failed because %s" name (.getMessage up-e))) 100 | (do 101 | (log/error (format "Migration %s failed because %s backing out" name (.getMessage up-e))) 102 | (try 103 | (proto/down migration (assoc config :conn db)) 104 | (catch Throwable down-e 105 | (log/debug down-e (format "As expected, one of the statements failed in %s while backing out the migration" name)))))) 106 | (throw up-e)) 107 | (finally 108 | (mark-unreserved db table-name))) 109 | :ignore))) 110 | 111 | (defn migrate-down* [db config migration] 112 | (let [id (proto/id migration) 113 | table-name (migration-table-name config)] 114 | (if (mark-reserved db table-name) 115 | (try 116 | (when (complete? db table-name id) 117 | (proto/down migration (assoc config :conn db)) 118 | (mark-not-complete db table-name id) 119 | :success) 120 | (finally 121 | (mark-unreserved db table-name))) 122 | :ignore))) 123 | 124 | (defn squash* [db config ids name] 125 | (let [table-name (migration-table-name config)] 126 | (if (mark-reserved db table-name) 127 | (try 128 | (when (complete-all? db table-name ids) 129 | (mark-not-complete-all db table-name ids) 130 | (mark-complete db table-name name (first ids)) 131 | :success) 132 | (finally 133 | (mark-unreserved db table-name))) 134 | :ignore))) 135 | 136 | (defn find-init-script-file [migration-dir init-script-name] 137 | (first 138 | (filter (fn [^File f] (and (.isFile f) (= (.getName f) init-script-name))) 139 | (file-seq migration-dir)))) 140 | 141 | (defn find-init-script-resource [migration-dir ^JarFile jar init-script-name] 142 | (let [init-script-path (utils/normalize-path 143 | (.getPath (io/file migration-dir init-script-name)))] 144 | (->> (.entries jar) 145 | (enumeration-seq) 146 | (filter (fn [^JarEntry entry] 147 | (.endsWith (.getName entry) init-script-path))) 148 | (first) 149 | (.getInputStream jar)))) 150 | 151 | (defn find-init-script [dir init-script-name] 152 | (let [dir (utils/ensure-trailing-slash dir)] 153 | (when-let [migration-dir (utils/find-migration-dir dir)] 154 | (if (instance? File migration-dir) 155 | (find-init-script-file migration-dir init-script-name) 156 | (find-init-script-resource dir migration-dir init-script-name))))) 157 | 158 | (defn connection-from-datasource [ds] 159 | (try (.getConnection ^DataSource ds) 160 | (catch Exception e 161 | (log/error e (str "Error getting DB connection from source" ds)) 162 | (throw e)))) 163 | 164 | (defn connect* 165 | "Connects to the store - SQL database in this case. 166 | Accepts a ^java.sql.Connection, ^javax.sql.DataSource or a db spec." 167 | [db] 168 | (assert (map? db) "db must be a map") 169 | (let [^Connection conn 170 | (cond 171 | (:connection db) (let [c (:connection db)] 172 | (assert (instance? Connection c) "c is not a Connection") 173 | c) 174 | (:datasource db) (let [ds (:datasource db)] 175 | (assert (instance? DataSource ds) "ds is not a DataSource") 176 | (connection-from-datasource ds)) 177 | :else (try 178 | ;; @ieugen: We can set auto-commit here as next.jdbc supports it. 179 | ;; But I guess we need to conside the case when we get a Connection directly 180 | (jdbc/get-connection db) 181 | (catch Exception e 182 | (log/error e (str "Error creating DB connection for " 183 | (utils/censor-password db))) 184 | (throw e))))] 185 | ;; Mutate Connection: set autocommit to false is necessary for transactional mode 186 | ;; and must be enabled for non transactional mode 187 | (if (:transaction? db) 188 | (.setAutoCommit conn false) 189 | (.setAutoCommit conn true)) 190 | {:connection conn})) 191 | 192 | (defn disconnect* [db config] 193 | (when-let [^Connection conn (:connection db)] 194 | (when-not (or (.isClosed conn) (:managed-connection? (:db config))) 195 | (.close conn)))) 196 | 197 | (defn completed-ids* [db table-name] 198 | (let [t-con (connection-or-spec db)] 199 | (->> (sql/query t-con 200 | [(str "select id, applied from " table-name " where id != " reserved-id)] 201 | {:builder-fn rs/as-unqualified-lower-maps}) 202 | (sort-by :applied #(compare %2 %1)) 203 | (map :id) 204 | (doall)))) 205 | 206 | (defn completed* [db table-name] 207 | (let [t-con (connection-or-spec db)] 208 | (->> (sql/query t-con 209 | [(str "select * from " table-name " where id != " reserved-id)] 210 | {:builder-fn rs/as-unqualified-lower-maps}) 211 | (sort-by :applied #(compare %2 %1)) 212 | (vec)))) 213 | 214 | (defn table-exists? 215 | "Checks whether the migrations table exists, by attempting to select from 216 | it. Note that this appears to be the only truly portable way to determine 217 | whether the table exists in a schema which the `db` configuration will find 218 | via a `SELECT FROM` or `INSERT INTO` the table. (In particular, note that 219 | attempting to find the table in the database meta-data as exposed by the JDBC 220 | driver does *not* tell you whether the table is on the current schema search 221 | path.)" 222 | [db table-name] 223 | (try 224 | ;; TODO: @ieugen: do we need :rollback-only here ? 225 | (let [db (connection-or-spec db)] 226 | (sql/query db [(str "SELECT 1 FROM " table-name)])) 227 | true 228 | (catch SQLException _ 229 | false))) 230 | 231 | (defn migration-table-up-to-date? 232 | [db table-name] 233 | (jdbc/with-transaction [t-con (connection-or-spec db)] 234 | (try 235 | (sql/query t-con [(str "SELECT applied,description FROM " table-name)]) 236 | true 237 | (catch SQLException _ 238 | false)))) 239 | 240 | (defn datetime-backend? 241 | "Checks whether the underlying backend requires the applied column to be 242 | of type datetime instead of timestamp." 243 | [db] 244 | (let [^Connection conn (:connection db) 245 | db-name (.. conn getMetaData getDatabaseProductName)] 246 | ;; TODO: @ieugen: we could leverage honeysql here but it might be a heavy extra dependency?! 247 | (if (= "Microsoft SQL Server" db-name) 248 | "DATETIME" 249 | "TIMESTAMP"))) 250 | 251 | (defn create-migration-table! 252 | "Creates the schema for the migration table via t-con in db in table-name" 253 | [db modify-sql-fn table-name] 254 | (log/info "creating migration table" (str "'" table-name "'")) 255 | (let [timestamp-column-type (datetime-backend? db)] 256 | (jdbc/with-transaction [t-con (connection-or-spec db)] 257 | (jdbc/execute! 258 | t-con 259 | (modify-sql-fn 260 | (str "CREATE TABLE " table-name 261 | " (id BIGINT UNIQUE NOT NULL, applied " timestamp-column-type 262 | ", description VARCHAR(1024) )")))))) 263 | 264 | (defn update-migration-table! 265 | "Updates the schema for the migration table via t-con in db in table-name" 266 | [db modify-sql-fn table-name] 267 | (log/info "updating migration table" (str "'" table-name "'")) 268 | (jdbc/with-transaction [t-con (connection-or-spec db)] 269 | (jdbc/execute-batch! 270 | t-con 271 | [(modify-sql-fn 272 | [(str "ALTER TABLE " table-name " ADD COLUMN description varchar(1024)") 273 | (str "ALTER TABLE " table-name " ADD COLUMN applied timestamp")])]))) 274 | 275 | 276 | (defn init-schema! [db table-name modify-sql-fn] 277 | ;; Note: the table-exists? *has* to be done in its own top-level 278 | ;; transaction. It can't be run in the same transaction as other code, because 279 | ;; if the table doesn't exist, then the error it raises internally in 280 | ;; detecting that will (on Postgres, at least) mark the transaction as 281 | ;; rollback only. That is, the act of detecting that it is necessary to create 282 | ;; the table renders the current transaction unusable for that purpose. I 283 | ;; blame Heisenberg. 284 | (or (table-exists? db table-name) 285 | (create-migration-table! db modify-sql-fn table-name)) 286 | (or (migration-table-up-to-date? db table-name) 287 | (update-migration-table! db modify-sql-fn table-name))) 288 | 289 | (defn run-init-script! [init-script-name init-script conn modify-sql-fn transaction?] 290 | (try 291 | (log/info "running initialization script '" init-script-name "'") 292 | (log/trace "\n" init-script "\n") 293 | ;; TODO: @ieugen Why was db-do-prepared used here ? 294 | ;; Do we need to care about `transaction?` in next.jdbc ? 295 | (if transaction? 296 | (jdbc/execute! conn (modify-sql-fn init-script)) 297 | (jdbc/execute! conn (modify-sql-fn init-script) {})) 298 | (catch Throwable t 299 | (log/error t "failed to initialize the database with:\n" init-script "\n") 300 | (throw t)))) 301 | 302 | (defn inject-properties [init-script properties] 303 | (if properties 304 | (props/inject-properties properties init-script) 305 | init-script)) 306 | 307 | (defn init-db! [db migration-dir init-script-name modify-sql-fn transaction? properties] 308 | (if-let [init-script (some-> (find-init-script migration-dir init-script-name) 309 | slurp 310 | (inject-properties properties))] 311 | (if transaction? 312 | (jdbc/with-transaction [t-con (connection-or-spec db)] 313 | (run-init-script! init-script-name init-script t-con modify-sql-fn transaction?)) 314 | (run-init-script! init-script-name init-script (connection-or-spec db) modify-sql-fn transaction?)) 315 | (log/error "could not locate the initialization script '" init-script-name "'"))) 316 | 317 | (defrecord Database [connection config] 318 | proto/Store 319 | (config [this] config) 320 | (init [this] 321 | (let [conn (connect* (assoc (:db config) :transaction? (:init-in-transaction? config)))] 322 | (try 323 | (init-db! conn 324 | (utils/get-migration-dir config) 325 | (utils/get-init-script config) 326 | (sql-mig/wrap-modify-sql-fn (:modify-sql-fn config)) 327 | (get config :init-in-transaction? true) 328 | (props/load-properties config)) 329 | (finally 330 | (disconnect* conn config))))) 331 | (completed-ids [this] 332 | (completed-ids* @connection (migration-table-name config))) 333 | (completed [this] 334 | (completed* @connection (migration-table-name config))) 335 | (migrate-up [this migration] 336 | (log/info "Connection is " @connection 337 | "Config is" (update config :db utils/censor-password)) 338 | (if (proto/tx? migration :up) 339 | (jdbc/with-transaction [t-con (connection-or-spec @connection)] 340 | (migrate-up* t-con config migration)) 341 | (migrate-up* (:db config) config migration))) 342 | (migrate-down [this migration] 343 | (log/info "Connection is " @connection 344 | "Config is" (update config :db utils/censor-password)) 345 | (if (proto/tx? migration :down) 346 | (jdbc/with-transaction [t-con (connection-or-spec @connection)] 347 | (migrate-down* t-con config migration)) 348 | (migrate-down* (:db config) config migration))) 349 | (squash [this ids name] 350 | (log/info "Connection is " @connection 351 | "Config is" (update config :db utils/censor-password)) 352 | (jdbc/with-transaction [t-con (connection-or-spec @connection)] 353 | (squash* t-con config ids name))) 354 | (connect [this] 355 | (reset! connection (connect* (:db config))) 356 | (init-schema! @connection 357 | (migration-table-name config) 358 | (sql-mig/wrap-modify-sql-fn (:modify-sql-fn config)))) 359 | (disconnect [this] 360 | (disconnect* @connection config) 361 | (reset! connection nil))) 362 | 363 | (defmethod proto/make-store :database 364 | [config] 365 | (->Database (atom nil) config)) 366 | -------------------------------------------------------------------------------- /src/migratus/migration/edn.clj: -------------------------------------------------------------------------------- 1 | (ns migratus.migration.edn 2 | "Support for EDN migration files that specify clojure code migrations." 3 | (:require 4 | [clojure.edn :as edn] 5 | [migratus.protocols :as proto])) 6 | 7 | ;; up-fn and down-fn here are actually vars; invoking them as fns will deref 8 | ;; them and invoke the fn bound by the var. 9 | (defrecord EdnMigration [id name up-fn down-fn transaction? up-args down-args] 10 | proto/Migration 11 | (id [this] id) 12 | (migration-type [this] :edn) 13 | (name [this] name) 14 | (tx? [this direction] (if (nil? transaction?) true transaction?)) 15 | (up [this config] 16 | (when up-fn 17 | (apply up-fn config up-args))) 18 | (down [this config] 19 | (when down-fn 20 | (apply down-fn config down-args)))) 21 | 22 | (defn to-sym 23 | "Converts x to a non-namespaced symbol, throwing if x is namespaced" 24 | [x] 25 | (let [validate #(if (and (instance? clojure.lang.Named %) 26 | (namespace %)) 27 | (throw (IllegalArgumentException. 28 | (str "Namespaced symbol not allowed: " %))) 29 | %)] 30 | ;; validate on input to catch namespaced symbols/keywords, and on output 31 | ;; to catch strings that parse as a namespaced symbol e.g. "foo/bar" 32 | (some-> x validate name symbol validate))) 33 | 34 | (defn resolve-fn 35 | "Basically ns-resolve with some error-checking" 36 | [mig-name mig-ns fn-name] 37 | (when fn-name 38 | (or (ns-resolve mig-ns (to-sym fn-name)) 39 | (throw (IllegalArgumentException. 40 | (format "Unable to resolve %s/%s for migration %s" 41 | mig-ns fn-name mig-name)))))) 42 | 43 | (defmethod proto/make-migration* :edn 44 | [_ mig-id mig-name payload config] 45 | (let [{:keys [ns up-fn down-fn transaction?] 46 | :or {up-fn "up" down-fn "down"}} (edn/read-string payload) 47 | mig-ns (to-sym ns) 48 | [up-fn & up-args] (cond-> up-fn (not (coll? up-fn)) vector) 49 | [down-fn & down-args] (cond-> down-fn (not (coll? down-fn)) vector)] 50 | (when-not mig-ns 51 | (throw (IllegalArgumentException. 52 | (format "Invalid migration %s: no namespace" mig-name)))) 53 | (require mig-ns) 54 | (->EdnMigration mig-id mig-name 55 | (resolve-fn mig-name mig-ns up-fn) 56 | (resolve-fn mig-name mig-ns down-fn) 57 | transaction? 58 | up-args 59 | down-args))) 60 | 61 | (defmethod proto/get-extension* :edn 62 | [_] 63 | "edn") 64 | 65 | (defmethod proto/migration-files* :edn 66 | [x migration-name] 67 | [(str migration-name "." (proto/get-extension* x))]) 68 | 69 | (defmethod proto/squash-migration-files* :edn 70 | [x migration-dir migration-name ups downs] 71 | (throw (Exception. "EDN migrations not implemented"))) 72 | -------------------------------------------------------------------------------- /src/migratus/migration/sql.clj: -------------------------------------------------------------------------------- 1 | (ns migratus.migration.sql 2 | (:require [next.jdbc :as jdbc] 3 | [next.jdbc.prepare :as prepare] 4 | [clojure.string :as str] 5 | [clojure.tools.logging :as log] 6 | [migratus.protocols :as proto] 7 | [clojure.java.io :as io]) 8 | (:import 9 | (java.sql Connection 10 | SQLException) 11 | java.util.regex.Pattern)) 12 | 13 | (def ^Pattern sep (Pattern/compile "^.*--;;.*\r?\n" Pattern/MULTILINE)) 14 | (def ^Pattern sql-comment (Pattern/compile "--.*" Pattern/MULTILINE)) 15 | (def ^Pattern sql-comment-without-expect (Pattern/compile "--((?! *expect).)*$" Pattern/MULTILINE)) 16 | (def ^Pattern empty-line (Pattern/compile "^[ ]+" Pattern/MULTILINE)) 17 | 18 | (defn use-tx? [sql] 19 | (not (str/starts-with? sql "-- :disable-transaction"))) 20 | 21 | (defn sanitize [command expect-results?] 22 | (-> command 23 | (clojure.string/replace (if expect-results? sql-comment-without-expect sql-comment) "") 24 | (clojure.string/replace empty-line ""))) 25 | 26 | (defn split-commands [commands expect-results?] 27 | (->> (.split sep commands) 28 | (map #(sanitize % expect-results?)) 29 | (remove empty?) 30 | (not-empty))) 31 | 32 | (defn check-expectations [result c] 33 | (let [[_full-str expect-str command] (re-matches #"(?sm).*\s*-- expect (.*);;\n+(.*)" c)] 34 | (assert expect-str (str "No expectation on command: " c)) 35 | (let [expected (some-> expect-str Long/parseLong) 36 | actual (some-> result first) 37 | different? (not= actual expected) 38 | message (format "%s %d" 39 | (some-> command (clojure.string/split #"\s+" 2) first clojure.string/upper-case) 40 | actual)] 41 | (if different? 42 | (log/error message "Expected" expected) 43 | (log/info message))))) 44 | 45 | (defn wrap-modify-sql-fn [old-modify-fn] 46 | (fn [sql] 47 | (let [modify-fn (or old-modify-fn identity) 48 | result (modify-fn sql)] 49 | (if (string? result) 50 | [result] 51 | result)))) 52 | 53 | (defn parse-commands-sql [{:keys [command-separator]} commands] 54 | (when (and (nil? command-separator) (> (count (re-seq #"(?m).+?;" commands)) 1)) 55 | (log/warn "Mismatch between number of SQL statements and '--;;' separators. Please ensure each statement is separated by '--;;'.")) 56 | (if command-separator 57 | (->> 58 | (str/split commands (re-pattern command-separator)) 59 | (map str/trim) 60 | (remove empty?)) 61 | commands)) 62 | 63 | (defn do-commands 64 | "Adapt db-do-commands to jdbc 65 | https://cljdoc.org/d/com.github.seancorfield/next.jdbc/1.2.780/doc/migration-from-clojure-java-jdbc" 66 | [connectable commands] 67 | (cond 68 | (instance? Connection connectable) 69 | (with-open [stmt (prepare/statement connectable)] 70 | ;; We test for (string? commands) because migratus.test.migrations.sql tests fails otherwise. 71 | ;; Perhaps it is a bug in migratus.test.mock implementation ?! 72 | (if (string? commands) 73 | (run! #(.addBatch stmt %) [commands]) 74 | (run! #(.addBatch stmt %) commands)) 75 | (into [] (.executeBatch stmt))) 76 | (:connection connectable) 77 | (do-commands (:connection connectable) commands) 78 | :else 79 | (with-open [conn (jdbc/get-connection connectable)] 80 | (do-commands conn commands)))) 81 | 82 | (defn execute-command [config t-con expect-results? commands] 83 | (log/trace "executing" commands) 84 | (cond-> 85 | (try 86 | (do-commands t-con (parse-commands-sql config commands)) 87 | (catch SQLException e 88 | (log/error (format "failed to execute command:\n %s" commands)) 89 | (loop [e e] 90 | (if-let [next-e (.getNextException e)] 91 | (recur next-e) 92 | (log/error (.getMessage e)))) 93 | (throw e)) 94 | (catch Throwable t 95 | (log/error (format "failed to execute command:\n %s\nFailure: %s" commands (.getMessage t))) 96 | (throw t))) 97 | expect-results? (check-expectations commands))) 98 | 99 | (defn- run-sql* 100 | [config conn expect-results? commands direction] 101 | (log/debug "found" (count commands) (name direction) "migrations") 102 | (doseq [c commands] 103 | (execute-command config conn expect-results? c))) 104 | 105 | (defn run-sql 106 | [{:keys [conn db modify-sql-fn expect-results?] :as config} sql direction] 107 | (when-let [commands (mapcat (wrap-modify-sql-fn modify-sql-fn) (split-commands sql expect-results?))] 108 | (if (use-tx? sql) 109 | (jdbc/with-transaction [t-con (or conn db)] 110 | (run-sql* config t-con expect-results? commands direction)) 111 | (run-sql* config (or conn db) expect-results? commands direction)))) 112 | 113 | (defrecord SqlMigration [id name up down] 114 | proto/Migration 115 | (id [_this] 116 | id) 117 | (migration-type [_this] :sql) 118 | (name [_this] 119 | name) 120 | (tx? [this direction] 121 | (if-let [sql (get this direction)] 122 | (use-tx? sql) 123 | (throw (Exception. (format "SQL %s commands not found for %d" direction id))))) 124 | (up [_this config] 125 | (if up 126 | (run-sql config up :up) 127 | (throw (Exception. (format "Up commands not found for %d" id))))) 128 | (down [_this config] 129 | (if down 130 | (run-sql config down :down) 131 | (throw (Exception. (format "Down commands not found for %d" id)))))) 132 | 133 | (defmethod proto/make-migration* :sql 134 | [_ mig-id mig-name payload _config] 135 | (->SqlMigration mig-id mig-name (:up payload) (:down payload))) 136 | 137 | 138 | (defmethod proto/get-extension* :sql 139 | [_] 140 | "sql") 141 | 142 | (defmethod proto/migration-files* :sql 143 | [x migration-name] 144 | (let [ext (proto/get-extension* x)] 145 | [(str migration-name ".up." ext) 146 | (str migration-name ".down." ext)])) 147 | 148 | (defmethod proto/squash-migration-files* :sql 149 | [x migration-dir migration-name ups downs] 150 | (doall 151 | (for [[mig-file ^String sql] (map vector (proto/migration-files* x migration-name) [ups downs])] 152 | (let [file (io/file migration-dir mig-file)] 153 | (.createNewFile file) 154 | (with-open [writer (java.io.BufferedWriter. (java.io.FileWriter. file))] 155 | (.write writer sql)) 156 | (.getName (io/file migration-dir mig-file)))))) 157 | -------------------------------------------------------------------------------- /src/migratus/migrations.clj: -------------------------------------------------------------------------------- 1 | (ns migratus.migrations 2 | (:require 3 | [clojure.java.io :as io] 4 | [clojure.string :as str] 5 | [clojure.tools.logging :as log] 6 | migratus.migration.edn 7 | migratus.migration.sql 8 | [migratus.properties :as props] 9 | [migratus.protocols :as proto] 10 | [migratus.utils :as utils]) 11 | (:import 12 | [java.io File StringWriter] 13 | [java.util Date TimeZone] 14 | [java.util.jar JarEntry JarFile] 15 | java.text.SimpleDateFormat 16 | java.util.regex.Pattern)) 17 | 18 | (defn ->kebab-case [s] 19 | (-> (reduce 20 | (fn [s c] 21 | (if (and 22 | (not-empty s) 23 | (Character/isLowerCase (char (last s))) 24 | (Character/isUpperCase (char c))) 25 | (str s "-" c) 26 | (str s c))) 27 | "" s) 28 | (str/replace #"[\s]+" "-") 29 | (.replaceAll "_" "-") 30 | (.toLowerCase))) 31 | 32 | (defn- timestamp [] 33 | (let [fmt (doto (SimpleDateFormat. "yyyyMMddHHmmss ") 34 | (.setTimeZone (TimeZone/getTimeZone "UTC")))] 35 | (.format fmt (Date.)))) 36 | 37 | (defn parse-migration-id [id] 38 | (try 39 | (Long/parseLong id) 40 | (catch Exception e 41 | (log/error e (str "failed to parse migration id: " id))))) 42 | 43 | (def migration-file-pattern #"^(\d+)-([^\.]+)((?:\.[^\.]+)+)$") 44 | 45 | (defn valid-extension? 46 | "Returns true if file-name extension matches one of the file extensions supported 47 | by all migration protocols/multimethods implemented" 48 | [file-name] 49 | (-> (->> (proto/get-all-supported-extensions) 50 | (interpose "|") 51 | (apply str) 52 | (format "^.*?\\.(%s)$")) 53 | re-pattern 54 | (re-matches file-name) 55 | boolean)) 56 | 57 | (defn parse-name [file-name] 58 | (when (valid-extension? file-name) 59 | (let [[id name ext] (next (re-matches migration-file-pattern file-name)) 60 | migration-type (remove empty? (some-> ext (str/split #"\.")))] 61 | (when (and id name (< 0 (count migration-type) 3)) 62 | [id name migration-type])))) 63 | 64 | (defn warn-on-invalid-migration [file-name] 65 | (log/warn (str "skipping: '" file-name "'") 66 | "migrations must match pattern:" 67 | (str migration-file-pattern))) 68 | 69 | (defn migration-map 70 | [[id name exts] content properties] 71 | (assoc-in {} 72 | (concat [id name] (map keyword (reverse exts))) 73 | (if properties 74 | (props/inject-properties properties content) 75 | content))) 76 | 77 | (defn find-migration-files [migration-dir exclude-scripts properties] 78 | (log/debug "Looking for migrations in" migration-dir) 79 | (->> (for [f (filter (fn [^File f] (.isFile f)) 80 | (file-seq migration-dir)) 81 | :let [file-name (.getName ^File f)] 82 | :when (not (utils/script-excluded? file-name migration-dir exclude-scripts))] 83 | (if-let [mig (parse-name file-name)] 84 | (migration-map mig (slurp f) properties) 85 | (warn-on-invalid-migration file-name))) 86 | (remove nil?))) 87 | 88 | 89 | (defn find-migration-resources [dir jar exclude-scripts properties] 90 | (log/debug "Looking for migrations in" dir jar) 91 | (->> (for [entry (enumeration-seq (.entries ^JarFile jar)) 92 | :when (.matches (.getName ^JarEntry entry) 93 | (str "^" (Pattern/quote dir) ".+")) 94 | :let [entry-name (.replaceAll (.getName ^JarEntry entry) dir "") 95 | last-slash-index (str/last-index-of entry-name "/") 96 | file-name (subs entry-name (if-not last-slash-index 0 97 | (+ 1 last-slash-index)))] 98 | :when (not (utils/script-excluded? file-name jar exclude-scripts))] 99 | (if-let [mig (parse-name file-name)] 100 | (let [w (StringWriter.)] 101 | (io/copy (.getInputStream ^JarFile jar entry) w) 102 | (migration-map mig (.toString w) properties)) 103 | (warn-on-invalid-migration file-name))) 104 | (remove nil?))) 105 | 106 | (defn read-migrations [dir exclude-scripts properties] 107 | (when-let [migration-dir (utils/find-migration-dir dir)] 108 | (if (instance? File migration-dir) 109 | (find-migration-files migration-dir exclude-scripts properties) 110 | (find-migration-resources dir migration-dir exclude-scripts properties)))) 111 | 112 | (defn find-migrations* 113 | [dir exclude-scripts properties] 114 | (->> (read-migrations (utils/ensure-trailing-slash dir) exclude-scripts properties) 115 | (apply utils/deep-merge))) 116 | 117 | (defn find-migrations 118 | [dir exclude-scripts properties] 119 | (let [dirs (if (string? dir) [dir] dir) 120 | fm (fn [d] (find-migrations* d exclude-scripts properties))] 121 | (into {} (map fm) dirs))) 122 | 123 | (defn find-or-create-migration-dir 124 | ([dir] (find-or-create-migration-dir utils/default-migration-parent dir)) 125 | ([parent-dir dir] 126 | (if-let [migration-dir (utils/find-migration-dir dir)] 127 | migration-dir 128 | 129 | ;; Couldn't find the migration dir, create it 130 | (let [new-migration-dir (io/file parent-dir dir)] 131 | (io/make-parents new-migration-dir ".") 132 | new-migration-dir)))) 133 | 134 | (defn make-migration 135 | "Constructs a Migration instance from the merged migration file maps collected 136 | by find-migrations. Expected structure for `mig` is: 137 | {`migration-name` {`migration-type` payload}} where the structure of `payload` 138 | is specific to the migration type." 139 | [config id mig] 140 | (if-let [id (parse-migration-id id)] 141 | (if (= 1 (count mig)) 142 | (let [[mig-name mig'] (first mig)] 143 | (if (= 1 (count mig')) 144 | (let [[mig-type payload] (first mig')] 145 | (proto/make-migration* mig-type id mig-name payload config)) 146 | (throw (Exception. 147 | (format 148 | "Multiple migration types specified for migration %d %s" 149 | id (pr-str (map name (keys mig')))))))) 150 | (throw (Exception. (format "Multiple migrations with id %d %s" 151 | id (pr-str (keys mig)))))) 152 | (throw (Exception. (str "Invalid migration id: " id))))) 153 | 154 | (defn list-migrations [config] 155 | (doall 156 | (for [[id mig] (find-migrations (utils/get-migration-dir config) 157 | (utils/get-exclude-scripts config) 158 | (props/load-properties config))] 159 | (make-migration config id mig)))) 160 | 161 | (defn create [config name migration-type] 162 | (let [migration-dir (find-or-create-migration-dir 163 | (utils/get-parent-migration-dir config) 164 | (utils/get-migration-dir config)) 165 | migration-name (->kebab-case (str (timestamp) name))] 166 | (doall 167 | (for [mig-file (proto/migration-files* migration-type migration-name)] 168 | (let [file (io/file migration-dir mig-file)] 169 | (.createNewFile file) 170 | (.getName (io/file migration-dir mig-file))))))) 171 | 172 | (defn create-squash [config id name migration-type ups downs] 173 | (let [migration-dir (find-or-create-migration-dir 174 | (utils/get-parent-migration-dir config) 175 | (utils/get-migration-dir config)) 176 | migration-name (->kebab-case (str id "-" name))] 177 | (proto/squash-migration-files* migration-type migration-dir migration-name ups downs))) 178 | 179 | (defn destroy [config name] 180 | (let [migration-dir (utils/find-migration-dir 181 | (utils/get-migration-dir config)) 182 | migration-name (->kebab-case name) 183 | pattern (re-pattern (str "[\\d]*-" migration-name "\\..*")) 184 | migrations (file-seq migration-dir)] 185 | (doseq [f (filter #(re-find pattern (.getName ^File %)) migrations)] 186 | (.delete ^File f)))) 187 | -------------------------------------------------------------------------------- /src/migratus/properties.clj: -------------------------------------------------------------------------------- 1 | (ns migratus.properties 2 | (:require 3 | [clojure.string :as s] 4 | [clojure.tools.logging :as log]) 5 | (:import 6 | java.util.Date)) 7 | 8 | (def ^:private default-properties 9 | #{"migratus.schema" "migratus.user" "migratus.database"}) 10 | 11 | (defn read-system-env [] 12 | (reduce 13 | (fn [m [k v]] 14 | (assoc m (-> (s/lower-case k) (s/replace "_" ".")) v)) 15 | {} 16 | (System/getenv))) 17 | 18 | (defn inject-properties [properties text] 19 | (let [text-with-props (reduce 20 | (fn [^String text [^String k v]] 21 | (.replace text k (str v))) 22 | text 23 | properties)] 24 | (doseq [x (re-seq #"\$\{[a-zA-Z0-9\-_\.]+}" text-with-props)] 25 | (log/warn "no property found for key:" x)) 26 | text-with-props)) 27 | 28 | (defn system-properties 29 | "Read system properties, accepts an optional collection of strings 30 | specifying additional property names" 31 | [property-names] 32 | (let [props (read-system-env)] 33 | (reduce 34 | (fn [m k] 35 | (if-let [v (get props k)] 36 | (assoc m (str "${" k "}") v) 37 | m)) 38 | {"${migratus.timestamp}" (.getTime (Date.))} 39 | (into default-properties property-names)))) 40 | 41 | (defn map->props 42 | ([m] (map->props {} nil m)) 43 | ([props path m] 44 | (reduce 45 | (fn [m [k v]] 46 | (let [path (if path (str path "." (name k)) (name k))] 47 | (if (map? v) 48 | (map->props m path v) 49 | (assoc m (str "${" path "}") v)))) 50 | props 51 | m))) 52 | 53 | (defn load-properties [{{:keys [env map]} :properties :as opts}] 54 | (when (map? (:properties opts)) 55 | (merge (system-properties env) (map->props map)))) 56 | -------------------------------------------------------------------------------- /src/migratus/protocols.clj: -------------------------------------------------------------------------------- 1 | ;;;; Copyright © 2011 Paul Stadig 2 | ;;;; 3 | ;;;; Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | ;;;; use this file except in compliance with the License. You may obtain a copy 5 | ;;;; of the License at 6 | ;;;; 7 | ;;;; http://www.apache.org/licenses/LICENSE-2.0 8 | ;;;; 9 | ;;;; Unless required by applicable law or agreed to in writing, software 10 | ;;;; distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | ;;;; WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | ;;;; License for the specific language governing permissions and limitations 13 | ;;;; under the License. 14 | (ns migratus.protocols 15 | (:refer-clojure :exclude [name])) 16 | 17 | (defprotocol Migration 18 | (id [this] "Id of this migration.") 19 | (migration-type [this] "Type of this migration.") 20 | (name [this] "Name of this migration") 21 | (tx? [this direction] "Whether this migration should run in a transaction.") 22 | (up [this config] "Bring this migration up.") 23 | (down [this config] "Bring this migration down.")) 24 | 25 | (defprotocol Store 26 | (config [this]) 27 | (init [this] 28 | "Initialize the data store.") 29 | (completed-ids [this] 30 | "Seq of ids of completed migrations in descending order of applied 31 | date.") 32 | (completed [this] 33 | "Seq of applied migrations in descending order of applied 34 | date.") 35 | (migrate-up [this migration] 36 | "Run and record an up migration") 37 | (migrate-down [this migration] 38 | "Run and record a down migration") 39 | (squash [this ids name] 40 | "Squash a batch of migrations into a single migration") 41 | (connect [this] 42 | "Opens resources necessary to run migrations against the store.") 43 | (disconnect [this] 44 | "Frees resources necessary to run migrations against the store.")) 45 | 46 | (defmulti make-store :store) 47 | 48 | (defmulti make-migration* 49 | "Dispatcher to create migrations based on filename extension. To add support 50 | for a new migration filename type, add a new defmethod for this." 51 | (fn [mig-type mig-id mig-name payload config] 52 | mig-type)) 53 | 54 | (defmethod make-migration* :default 55 | [mig-type mig-id mig-name payload config] 56 | (throw (Exception. (format "Unknown type '%s' for migration %d" 57 | (clojure.core/name mig-type) mig-id)))) 58 | 59 | (defmulti migration-files* 60 | "Dispatcher to get a list of filenames to create when creating new migrations" 61 | (fn [mig-type migration-name] 62 | mig-type)) 63 | 64 | (defmethod migration-files* :default 65 | [mig-type migration-name] 66 | (throw (Exception. (format "Unknown migration type '%s'" 67 | (clojure.core/name mig-type))))) 68 | 69 | 70 | (defmulti get-extension* 71 | "Dispatcher to get the supported file extension for this migration" 72 | (fn [mig-type] 73 | mig-type)) 74 | 75 | (defmethod get-extension* :default 76 | [mig-type] 77 | (throw (Exception. (format "Unknown migration type '%s'" 78 | (clojure.core/name mig-type))))) 79 | 80 | 81 | (defn get-all-supported-extensions 82 | "Returns a seq of all the file extensions supported by all migration protocols" 83 | [] 84 | (for [[k v] (methods get-extension*) 85 | :when (-> k (= :default) not)] 86 | (v k))) 87 | 88 | (defmulti squash-migration-files* 89 | "Dispatcher to read a list of files and squash them into a single migration file" 90 | (fn [mig-type migration-dir migration-name ups downs] 91 | mig-type)) 92 | 93 | (defmethod squash-migration-files* :default 94 | [mig-type migration-dir migration-name ups downs] 95 | (throw (Exception. (format "Unknown migration type '%s'" 96 | (clojure.core/name mig-type))))) 97 | -------------------------------------------------------------------------------- /src/migratus/utils.clj: -------------------------------------------------------------------------------- 1 | (ns migratus.utils 2 | (:require 3 | [clojure.string :as str] 4 | [clojure.java.io :as io]) 5 | (:import 6 | java.io.File 7 | java.util.jar.JarFile 8 | [java.util Map] 9 | [java.net URL URLDecoder URI] 10 | [java.nio.file FileSystem FileSystems FileSystemNotFoundException])) 11 | 12 | (def default-migration-parent "resources/") 13 | (def default-migration-dir "migrations") 14 | (def default-init-script-name "init.sql") 15 | 16 | (defn get-parent-migration-dir 17 | "Gets the :parent-migration-dir from config, or default if missing." 18 | [config] 19 | (get config :parent-migration-dir default-migration-parent)) 20 | 21 | (defn get-migration-dir 22 | "Gets the :migration-dir from config, or default if missing." 23 | [config] 24 | (get config :migration-dir default-migration-dir)) 25 | 26 | (defn get-init-script 27 | "Gets the :init-script from config, or default if missing." 28 | [config] 29 | (get config :init-script default-init-script-name)) 30 | 31 | (defn get-exclude-scripts 32 | "Returns a set of script name globs to exclude when finding migrations" 33 | [config] 34 | (into #{(get-init-script config)} 35 | (get config :exclude-scripts))) 36 | 37 | (defn resolve-uri 38 | ^URI [^JarFile migration-dir] 39 | (try 40 | (URI. (str "jar:file:" (.getName migration-dir))) 41 | (catch java.net.URISyntaxException _ 42 | (URI. (str "jar:" (-> migration-dir .getName io/file .toURI)))))) 43 | 44 | (defn script-excluded? 45 | "Returns true if the script should be excluded according 46 | to the collection of globs in exclude-scripts." 47 | [script migration-dir exclude-scripts] 48 | (when (seq exclude-scripts) 49 | (let [^FileSystem fs (if (instance? File migration-dir) 50 | (.getFileSystem (.toPath ^File migration-dir)) 51 | (let [uri (resolve-uri migration-dir) 52 | ^Map env {}] 53 | (try 54 | (FileSystems/getFileSystem uri) 55 | (catch FileSystemNotFoundException _ 56 | (FileSystems/newFileSystem uri env))))) 57 | path (.getPath fs script (make-array String 0))] 58 | (some #(.matches (.getPathMatcher fs (str "glob:" %)) path) 59 | exclude-scripts)))) 60 | 61 | (defn ensure-trailing-slash 62 | "Put a trailing slash on the dirname if not present" 63 | [dir] 64 | (if (not= (last dir) \/) 65 | (str dir "/") 66 | dir)) 67 | 68 | (defn jar-name 69 | ^String [^String s] 70 | (some-> s 71 | (str/replace "+" "%2B") 72 | (URLDecoder/decode "UTF-8") 73 | (.split "!/") 74 | ^String (first) 75 | (.replaceFirst "file:" ""))) 76 | 77 | (defn jar-file [^URL url] 78 | (some-> url 79 | (.getFile) 80 | (jar-name) 81 | (JarFile.))) 82 | 83 | (defn find-migration-dir 84 | "Finds the given directory on the classpath. For backward 85 | compatibility, tries the System ClassLoader first, but falls back to 86 | using the Context ClassLoader like Clojure's compiler. 87 | If classloaders return nothing try to find it on a filesystem." 88 | ([^String dir] 89 | (or (find-migration-dir (ClassLoader/getSystemClassLoader) default-migration-parent dir) 90 | (-> (Thread/currentThread) 91 | (.getContextClassLoader) 92 | (find-migration-dir default-migration-parent dir)))) 93 | ([^ClassLoader class-loader ^String parent-dir ^String dir] 94 | (if-let [^URL url (.getResource class-loader dir)] 95 | (if (= "jar" (.getProtocol url)) 96 | (jar-file url) 97 | (File. (URLDecoder/decode (.getFile url) "UTF-8"))) 98 | (let [migration-dir (io/file parent-dir dir)] 99 | (if (.exists migration-dir) 100 | migration-dir 101 | (let [no-implicit-parent-dir (io/file dir)] 102 | (when (.exists no-implicit-parent-dir) 103 | no-implicit-parent-dir))))))) 104 | 105 | (defn deep-merge 106 | "Merge keys at all nested levels of the maps." 107 | [& maps] 108 | (apply merge-with deep-merge maps)) 109 | 110 | (defn recursive-delete 111 | "Delete a file, including all children if it's a directory" 112 | [^File f] 113 | (when (.exists f) 114 | (if (.isDirectory f) 115 | (doseq [child (.listFiles f)] 116 | (recursive-delete child)) 117 | (.delete f)))) 118 | 119 | (defn normalize-path 120 | "Replace backslashes with forwardslashes" 121 | [^String s] 122 | (str/replace s "\\" "/")) 123 | 124 | (defmulti censor-password class) 125 | 126 | (defmethod censor-password String [uri] 127 | (if (empty? uri) 128 | "" 129 | "uri-censored")) 130 | 131 | (defmethod censor-password :default 132 | [{:keys [password connection-uri] :as db-spec}] 133 | (let [password-map 134 | (if (empty? password) 135 | nil 136 | ;; Show only first character of password if given db-spec has password 137 | {:password 138 | (str (subs password 0 (min 1 (count password))) 139 | "")}) 140 | uri-map 141 | (if (empty? connection-uri) 142 | nil 143 | ;; Censor entire uri instead of trying to parse out and replace only a possible password parameter 144 | {:connection-uri "uri-censored"})] 145 | (merge db-spec password-map uri-map))) 146 | -------------------------------------------------------------------------------- /test/migrations-bad-type/20170328124600-bad-type.foo: -------------------------------------------------------------------------------- 1 | -- Doesn't matter what goes in here, it won't get run. 2 | -------------------------------------------------------------------------------- /test/migrations-duplicate-name/20170328130700-dup-name-1.up.sql: -------------------------------------------------------------------------------- 1 | -- Doesn't matter what goes in here, it won't get run. 2 | -------------------------------------------------------------------------------- /test/migrations-duplicate-name/20170328130700-dup-name-2.up.sql: -------------------------------------------------------------------------------- 1 | -- Doesn't matter what goes in here, it won't get run. 2 | -------------------------------------------------------------------------------- /test/migrations-duplicate-type/20170328125100-duplicate-type.edn: -------------------------------------------------------------------------------- 1 | {:namespace "doesnt.matter"} 2 | -------------------------------------------------------------------------------- /test/migrations-duplicate-type/20170328125100-duplicate-type.up.sql: -------------------------------------------------------------------------------- 1 | -- Doesn't matter what goes in here, it won't get run. 2 | -------------------------------------------------------------------------------- /test/migrations-edn-args/20220827210100-say-hello-with-args.edn: -------------------------------------------------------------------------------- 1 | {:ns migratus.test.migration.edn.test-script-args 2 | :up-fn [migrate-up "hello-with-args.txt" "Hello, world, with args!"] 3 | :down-fn [migrate-down "hello-with-args.txt"] 4 | :transaction? true} 5 | -------------------------------------------------------------------------------- /test/migrations-edn/20170330142700-say-hello.edn: -------------------------------------------------------------------------------- 1 | {:ns migratus.test.migration.edn.test-script 2 | :up-fn migrate-up 3 | :down-fn migrate-down 4 | :transaction? true} 5 | -------------------------------------------------------------------------------- /test/migrations-intentionally-broken-no-tx/20120827170200-multiple-statements-broken.down.sql: -------------------------------------------------------------------------------- 1 | -- :disable-transaction 2 | DROP TABLE quux2; 3 | --;; 4 | DROP TABLE quux3; 5 | -------------------------------------------------------------------------------- /test/migrations-intentionally-broken-no-tx/20120827170200-multiple-statements-broken.up.sql: -------------------------------------------------------------------------------- 1 | -- :disable-transaction 2 | 3 | CREATE TABLE 4 | quux2 5 | (id bigint, 6 | name varchar(255)); 7 | 8 | --;; 9 | THIS IS MOST DEFINITELY NOT VALID SQL; 10 | 11 | --;; 12 | 13 | CREATE TABLE 14 | quux3 15 | (id bigint, 16 | name varchar(255)); 17 | -------------------------------------------------------------------------------- /test/migrations-intentionally-broken/20120827170200-multiple-statements-broken.down.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE quux2; 2 | --;; 3 | DROP TABLE quux3; 4 | -------------------------------------------------------------------------------- /test/migrations-intentionally-broken/20120827170200-multiple-statements-broken.up.sql: -------------------------------------------------------------------------------- 1 | -- this is the first statement 2 | 3 | CREATE TABLE 4 | quux2 5 | (id bigint, 6 | name varchar(255)); 7 | 8 | --;; 9 | THIS IS MOST DEFINITELY NOT VALID SQL; 10 | 11 | --;; 12 | 13 | CREATE TABLE 14 | quux3 15 | (id bigint, 16 | name varchar(255)); 17 | -------------------------------------------------------------------------------- /test/migrations-jar/init-test.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yogthos/migratus/b495090fb8123b6d5ff5744eb1bce1a147d6ff67/test/migrations-jar/init-test.jar -------------------------------------------------------------------------------- /test/migrations-jar/migrations.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yogthos/migratus/b495090fb8123b6d5ff5744eb1bce1a147d6ff67/test/migrations-jar/migrations.jar -------------------------------------------------------------------------------- /test/migrations-no-tx/20111202110600-create-foo-table.down.sql: -------------------------------------------------------------------------------- 1 | -- :disable-transaction 2 | DROP TABLE IF EXISTS foo; 3 | -------------------------------------------------------------------------------- /test/migrations-no-tx/20111202110600-create-foo-table.up.sql: -------------------------------------------------------------------------------- 1 | -- :disable-transaction 2 | CREATE TABLE IF NOT EXISTS foo(id bigint); 3 | -------------------------------------------------------------------------------- /test/migrations-parse/20241012170200-create-quux-table.down.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE quux; 2 | -------------------------------------------------------------------------------- /test/migrations-parse/20241012170200-create-quux-table.up.sql: -------------------------------------------------------------------------------- 1 | -- test comment parsed correctly 2 | CREATE TABLE -- first comment 3 | quux -- second comment; 4 | (id bigint, 5 | name varchar(255)); -- last comment 6 | -- end 7 | -------------------------------------------------------------------------------- /test/migrations-postgres/20220820030400-create-table-quux.down.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS quux; -------------------------------------------------------------------------------- /test/migrations-postgres/20220820030400-create-table-quux.up.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS quux(id bigint, name varchar(255)); 2 | --;; 3 | CREATE INDEX quux_name on quux(name); -------------------------------------------------------------------------------- /test/migrations-postgres/init.sql: -------------------------------------------------------------------------------- 1 | CREATE SCHEMA foo; 2 | CREATE TABLE IF NOT EXISTS foo(id bigint); 3 | -------------------------------------------------------------------------------- /test/migrations-with-props/20111202110600-create-foo-table.down.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS ${migratus.schema}.foo; 2 | -------------------------------------------------------------------------------- /test/migrations-with-props/20111202110600-create-foo-table.up.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS foo(id bigint); 2 | -------------------------------------------------------------------------------- /test/migrations-with-props/20111202110600-create-schema.up.sql: -------------------------------------------------------------------------------- 1 | CREATE SCHEMA ${migratus.schema} 2 | -------------------------------------------------------------------------------- /test/migrations/20111202110600-create-foo-table.down.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS foo; 2 | -------------------------------------------------------------------------------- /test/migrations/20111202110600-create-foo-table.up.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS foo(id bigint); 2 | -------------------------------------------------------------------------------- /test/migrations/20111202110600-create-foo-table.up.sql~: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS foo(id bigint); 2 | -------------------------------------------------------------------------------- /test/migrations/20111202113000-create-bar-table.down.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS bar; 2 | -------------------------------------------------------------------------------- /test/migrations/20111202113000-create-bar-table.up.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS bar(id BIGINT); 2 | -------------------------------------------------------------------------------- /test/migrations/20120827170200-multiple-statements.down.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE quux2; 2 | --;; 3 | DROP TABLE quux; 4 | -------------------------------------------------------------------------------- /test/migrations/20120827170200-multiple-statements.up.sql: -------------------------------------------------------------------------------- 1 | -- this is the first statement 2 | 3 | CREATE TABLE 4 | quux 5 | (id bigint, 6 | name varchar(255)); 7 | 8 | --;; 9 | -- comment for the second statement 10 | 11 | CREATE TABLE quux2(id bigint, name varchar(255)); 12 | -------------------------------------------------------------------------------- /test/migrations/bogus.txt: -------------------------------------------------------------------------------- 1 | This is a bogus migration. 2 | -------------------------------------------------------------------------------- /test/migrations/init.sql: -------------------------------------------------------------------------------- 1 | CREATE SCHEMA foo; 2 | CREATE TABLE IF NOT EXISTS foo(id bigint); 3 | -------------------------------------------------------------------------------- /test/migrations1/20220604110500-create-foo1-table.down.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS foo1; -------------------------------------------------------------------------------- /test/migrations1/20220604110500-create-foo1-table.up.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS foo1(id bigint); -------------------------------------------------------------------------------- /test/migrations1/20220604113000-create-bar1-table.down.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS bar1; -------------------------------------------------------------------------------- /test/migrations1/20220604113000-create-bar1-table.up.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS bar1(id BIGINT); -------------------------------------------------------------------------------- /test/migrations2/20220604111500-create-foo2-table.down.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS foo2; -------------------------------------------------------------------------------- /test/migrations2/20220604111500-create-foo2-table.up.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS foo2(id bigint); -------------------------------------------------------------------------------- /test/migrations2/20220604113500-create-bar2-table.down.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS bar2; -------------------------------------------------------------------------------- /test/migrations2/20220604113500-create-bar2-table.up.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS bar2(id BIGINT); -------------------------------------------------------------------------------- /test/migratus/logger.clj: -------------------------------------------------------------------------------- 1 | (ns migratus.logger 2 | (:import org.slf4j.LoggerFactory 3 | [ch.qos.logback.classic Level Logger])) 4 | 5 | (.setLevel (LoggerFactory/getLogger Logger/ROOT_LOGGER_NAME) Level/ERROR) 6 | 7 | -------------------------------------------------------------------------------- /test/migratus/mock.clj: -------------------------------------------------------------------------------- 1 | ;;;; Copyright © 2011 Paul Stadig 2 | ;;;; 3 | ;;;; Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | ;;;; use this file except in compliance with the License. You may obtain a copy 5 | ;;;; of the License at 6 | ;;;; 7 | ;;;; http://www.apache.org/licenses/LICENSE-2.0 8 | ;;;; 9 | ;;;; Unless required by applicable law or agreed to in writing, software 10 | ;;;; distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | ;;;; WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | ;;;; License for the specific language governing permissions and limitations 13 | ;;;; under the License. 14 | (ns migratus.mock 15 | (:require [migratus.protocols :as proto])) 16 | 17 | (defrecord MockMigration [db id name ups downs] 18 | proto/Migration 19 | (id [this] 20 | id) 21 | (migration-type [this] 22 | :sql) 23 | (name [this] 24 | name) 25 | (up [this config] 26 | (swap! ups conj id) 27 | :success) 28 | (down [this config] 29 | (swap! downs conj id) 30 | :success)) 31 | 32 | (defrecord MockStore [completed-ids config] 33 | proto/Store 34 | (init [this]) 35 | (completed-ids [this] 36 | @completed-ids) 37 | (completed [this] 38 | (map (fn [id] {:id id :applied true}) @completed-ids)) 39 | (migrate-up [this migration] 40 | (proto/up migration config) 41 | (swap! completed-ids conj (proto/id migration)) 42 | :success) 43 | (migrate-down [this migration] 44 | (proto/down migration config) 45 | (swap! completed-ids disj (proto/id migration))) 46 | (connect [this]) 47 | (disconnect [this])) 48 | 49 | (defn make-migration [{:keys [id name ups downs]}] 50 | (MockMigration. nil id name ups downs)) 51 | 52 | (defmethod proto/make-store :mock 53 | [{:keys [completed-ids] :as config}] 54 | (MockStore. completed-ids config)) 55 | -------------------------------------------------------------------------------- /test/migratus/test/core.clj: -------------------------------------------------------------------------------- 1 | ;;;; Copyright © 2011 Paul Stadig 2 | ;;;; 3 | ;;;; Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | ;;;; use this file except in compliance with the License. You may obtain a copy 5 | ;;;; of the License at 6 | ;;;; 7 | ;;;; http://www.apache.org/licenses/LICENSE-2.0 8 | ;;;; 9 | ;;;; Unless required by applicable law or agreed to in writing, software 10 | ;;;; distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | ;;;; WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | ;;;; License for the specific language governing permissions and limitations 13 | ;;;; under the License. 14 | (ns migratus.test.core 15 | (:require [migratus.protocols :as proto] 16 | [migratus.mock :as mock] 17 | [clojure.test :refer :all] 18 | [migratus.core :refer :all] 19 | migratus.logger 20 | [migratus.migrations :as mig] 21 | [migratus.utils :as utils] 22 | [clojure.java.io :as io]) 23 | (:import [migratus.mock MockStore MockMigration])) 24 | 25 | (defn migrations [ups downs] 26 | (for [n (range 4)] 27 | (mock/make-migration 28 | {:id (inc n) :name (str "id-" (inc n)) :ups ups :downs downs}))) 29 | 30 | (deftest test-migrate 31 | (let [ups (atom []) 32 | downs (atom []) 33 | config {:store :mock 34 | :completed-ids (atom #{1 3})}] 35 | (with-redefs [mig/list-migrations (constantly (migrations ups downs))] 36 | (migrate config)) 37 | (is (= [2 4] @ups)) 38 | (is (empty? @downs)))) 39 | 40 | (deftest test-up 41 | (let [ups (atom []) 42 | downs (atom []) 43 | config {:store :mock 44 | :completed-ids (atom #{1 3})}] 45 | (with-redefs [mig/list-migrations (constantly (migrations ups downs))] 46 | (testing "should bring up an uncompleted migration" 47 | (up config 4 2) 48 | (is (= [2 4] @ups)) 49 | (is (empty? @downs))) 50 | (reset! ups []) 51 | (reset! downs []) 52 | (testing "should do nothing for a completed migration" 53 | (up config 1) 54 | (is (empty? @ups)) 55 | (is (empty? @downs)))))) 56 | 57 | (deftest test-down 58 | (let [ups (atom []) 59 | downs (atom []) 60 | config {:store :mock 61 | :completed-ids (atom #{1 3})}] 62 | (with-redefs [mig/list-migrations (constantly (migrations ups downs))] 63 | (testing "should bring down a completed migration" 64 | (down config 1 3) 65 | (is (empty? @ups)) 66 | (is (= [3 1] @downs))) 67 | (reset! ups []) 68 | (reset! downs []) 69 | (testing "should do nothing for an uncompleted migration" 70 | (down config 2) 71 | (is (empty? @ups)) 72 | (is (empty? @downs)))))) 73 | 74 | (defn- migration-exists? [name & [dir]] 75 | (when-let [migrations-dir (utils/find-migration-dir (or dir "migrations"))] 76 | (->> (file-seq migrations-dir) 77 | (map #(.getName %)) 78 | (filter #(.contains % name)) 79 | (not-empty)))) 80 | 81 | (deftest test-create-and-destroy 82 | (let [migration "create-user" 83 | migration-up "create-user.up.sql" 84 | migration-down "create-user.down.sql"] 85 | (testing "should create two migrations" 86 | (create nil migration) 87 | (is (migration-exists? migration-up)) 88 | (is (migration-exists? migration-down))) 89 | (testing "should delete two migrations" 90 | (destroy nil migration) 91 | (is (empty? (migration-exists? migration-up))) 92 | (is (empty? (migration-exists? migration-down)))))) 93 | 94 | (deftest test-create-and-destroy-edn 95 | (let [migration "create-other-user" 96 | migration-edn "create-other-user.edn"] 97 | (testing "should create the migration" 98 | (create nil migration :edn) 99 | (is (migration-exists? migration-edn))) 100 | (testing "should delete the migration" 101 | (destroy nil migration) 102 | (is (empty? (migration-exists? migration-edn)))))) 103 | 104 | (deftest test-create-missing-directory 105 | (let [migration-dir "doesnt_exist" 106 | config {:parent-migration-dir "test" 107 | :migration-dir migration-dir} 108 | migration "create-user" 109 | migration-up "create-user.up.sql" 110 | migration-down "create-user.down.sql"] 111 | ;; Make sure the directory doesn't exist before we start the test 112 | (when (.exists (io/file "test" migration-dir)) 113 | (io/delete-file (io/file "test" migration-dir))) 114 | 115 | (testing "when migration dir doesn't exist, it is created" 116 | (is (nil? (utils/find-migration-dir migration-dir))) 117 | (create config migration) 118 | (is (not (nil? (utils/find-migration-dir migration-dir)))) 119 | (is (migration-exists? migration-up migration-dir)) 120 | (is (migration-exists? migration-down migration-dir))) 121 | 122 | ;; Clean up after ourselves 123 | (when (.exists (io/file "test" migration-dir)) 124 | (destroy config migration) 125 | (io/delete-file (io/file "test" migration-dir))))) 126 | 127 | (deftest test-completed-list 128 | (let [ups (atom []) 129 | downs (atom []) 130 | config {:store :mock 131 | :completed-ids (atom #{1 2 3})}] 132 | (with-redefs [mig/list-migrations (constantly (migrations ups downs))] 133 | (testing "should return the list of completed migrations" 134 | (is (= ["id-1" "id-2" "id-3"] 135 | (migratus.core/completed-list config))))))) 136 | 137 | (deftest test-pending-list 138 | (let [ups (atom []) 139 | downs (atom []) 140 | config {:store :mock 141 | :completed-ids (atom #{1})}] 142 | (with-redefs [mig/list-migrations (constantly (migrations ups downs))] 143 | (testing "should return the list of pending migrations" 144 | (is (= ["id-2" "id-3" "id-4"] 145 | (migratus.core/pending-list config))))))) 146 | 147 | (deftest test-squashing-list 148 | (let [ups (atom []) 149 | downs (atom []) 150 | config {:store :mock 151 | :completed-ids (atom #{1 3})}] 152 | (with-redefs [mig/list-migrations (constantly (migrations ups downs))] 153 | (testing "should throw an exception if the migration is not applied" 154 | (is (thrown? IllegalArgumentException 155 | (migratus.core/squashing-list config 1 3)))) 156 | (testing "should bring up an uncompleted migration" 157 | (up config 4 2) 158 | (is (= [2 4] @ups)) 159 | (is (empty? @downs))) 160 | (testing "should return the list of squashing migrations in order" 161 | (is (= ["id-1" "id-2" "id-3" "id-4"] 162 | (migratus.core/squashing-list config 1 4)))) 163 | (testing "should return the list of squashing migrations within the inclusive range" 164 | (is (= ["id-1" "id-2" "id-3"] 165 | (migratus.core/squashing-list config 1 3))))))) 166 | 167 | (deftest test-select-migrations 168 | (let [ups (atom []) 169 | downs (atom []) 170 | config {:store :mock 171 | :completed-ids (atom #{1 3})}] 172 | (with-redefs [mig/list-migrations (constantly (migrations ups downs))] 173 | (testing "should return the list of [id name] selected migrations" 174 | (is (= [[1 "id-1"] [3 "id-3"]] 175 | (migratus.core/select-migrations config migratus.core/completed-migrations))) 176 | (is (= [[2 "id-2"] [4 "id-4"]] 177 | (migratus.core/select-migrations config migratus.core/uncompleted-migrations))))))) 178 | 179 | (deftest supported-extensions 180 | (testing "All supported extensions show up. 181 | NOTE: when you add a protocol, to migratus core, update this test") 182 | (is (= '("sql" "edn") 183 | (proto/get-all-supported-extensions)))) 184 | -------------------------------------------------------------------------------- /test/migratus/test/database.clj: -------------------------------------------------------------------------------- 1 | ;;;; Copyright © 2011 Paul Stadig 2 | ;;;; 3 | ;;;; Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | ;;;; use this file except in compliance with the License. You may obtain a copy 5 | ;;;; of the License at 6 | ;;;; 7 | ;;;; http://www.apache.org/licenses/LICENSE-2.0 8 | ;;;; 9 | ;;;; Unless required by applicable law or agreed to in writing, software 10 | ;;;; distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | ;;;; WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | ;;;; License for the specific language governing permissions and limitations 13 | ;;;; under the License. 14 | (ns migratus.test.database 15 | (:require [clojure.java.io :as io] 16 | [next.jdbc :as jdbc] 17 | [next.jdbc.result-set :as rs] 18 | [next.jdbc.quoted :as q] 19 | [next.jdbc.sql :as sql] 20 | [migratus.protocols :as proto] 21 | [migratus.core :as core] 22 | [clojure.test :refer :all] 23 | [migratus.database :refer :all :as db] 24 | [clojure.tools.logging :as log] 25 | [migratus.test.migration.edn :as test-edn] 26 | [migratus.test.migration.sql :as test-sql] 27 | [migratus.utils :as utils]) 28 | (:import java.io.File 29 | java.sql.Connection 30 | java.util.jar.JarFile 31 | (java.util.concurrent CancellationException))) 32 | 33 | (def config (merge test-sql/test-config 34 | {:store :database 35 | :command-separator ";" 36 | :migration-table-name "foo_bar"})) 37 | 38 | (defn verify-data [config table-name] 39 | (let [db (connect* (:db config)) 40 | conn (:connection db) 41 | result (sql/query conn 42 | [(str "SELECT * from " table-name)] 43 | {:builder-fn rs/as-unqualified-lower-maps})] 44 | (.close conn) 45 | result)) 46 | 47 | (defn test-with-store [store & commands] 48 | (try 49 | (proto/connect store) 50 | (doseq [cmd commands] 51 | (cmd (proto/config store))) 52 | (finally 53 | (proto/disconnect store)))) 54 | 55 | (use-fixtures :each test-sql/setup-test-db) 56 | 57 | (def db-mem {:dbtype "h2:mem" 58 | :name "mem-db"}) 59 | 60 | (deftest test-connect*-returns-a-connection 61 | (testing "connect* works with a ^java.sql.Connection" 62 | (let [ds (jdbc/get-datasource db-mem)] 63 | (with-open [connection (jdbc/get-connection ds)] 64 | (let [res (db/connect* {:connection connection})] 65 | (is (map? res) "connect* response is a map") 66 | (is (contains? res :connection) "connect* response contains :connection") 67 | (is (instance? Connection (:connection res)) 68 | "connect* response has a ^java.sql.Connection") 69 | (is (= connection (:connection res)) 70 | "connect* response contains the same connection we passed"))))) 71 | 72 | (testing "connect* works with a ^javax.sql.DataSource" 73 | (let [ds (jdbc/get-datasource db-mem) 74 | res (db/connect* {:datasource ds})] 75 | (is (map? res) "connect* response is a map") 76 | (is (contains? res :connection) "connect* response contains :connection") 77 | (is (instance? Connection (:connection res)) 78 | "connect* response has a ^java.sql.Connection"))) 79 | 80 | (testing "connect* works with a db spec" 81 | (let [res (db/connect* db-mem)] 82 | (is (map? res) "connect* response is a map") 83 | (is (contains? res :connection) "connect* response contains :connection") 84 | (is (instance? Connection (:connection res)) 85 | "connect* response has a ^java.sql.Connection")))) 86 | 87 | (deftest test-find-init-script-resource 88 | (testing "finds init.sql under migrations/ in a JAR file" 89 | (let [jar-file (JarFile. "test/migrations-jar/init-test.jar") 90 | init-script (find-init-script-resource "migrations/" jar-file "init.sql")] 91 | (is (not (nil? init-script))) 92 | (is (= "CREATE SCHEMA foo;\n" (slurp init-script)))))) 93 | 94 | (deftest test-make-store 95 | (testing "should create default table name" 96 | (is (not (test-sql/verify-table-exists? 97 | (dissoc config :migration-table-name) default-migrations-table))) 98 | (test-with-store 99 | (proto/make-store (dissoc config :migration-table-name)) 100 | (fn [config] 101 | (is (test-sql/verify-table-exists? config default-migrations-table))))) 102 | (test-sql/reset-db) 103 | (testing "should create schema_migrations table" 104 | (is (not (test-sql/verify-table-exists? config "foo_bar"))) 105 | (test-with-store 106 | (proto/make-store config) 107 | (fn [config] 108 | (is (test-sql/verify-table-exists? config "foo_bar"))))) 109 | (test-sql/reset-db) 110 | (testing "should use complex table name" 111 | (let [table-name "U&\"\\00d6ffnungszeit\"" 112 | config (assoc config :migration-table-name table-name)] 113 | (is (not (test-sql/verify-table-exists? config table-name))) 114 | (test-with-store 115 | (proto/make-store config) 116 | (fn [config] 117 | (is (test-sql/verify-table-exists? config table-name))))))) 118 | 119 | (deftest test-make-store-pass-conn 120 | (testing "should create default table name" 121 | (is (not (test-sql/verify-table-exists? 122 | (dissoc config :migration-table-name) default-migrations-table))) 123 | (test-with-store 124 | (proto/make-store (-> (dissoc config :migration-table-name) 125 | (assoc :db {:connection (jdbc/get-connection (:db config))}))) 126 | (fn [_] 127 | (test-sql/verify-table-exists? (dissoc config :migration-table-name) 128 | default-migrations-table)))) 129 | (test-sql/reset-db)) 130 | 131 | (deftest test-init 132 | (testing "db init" 133 | (let [config (assoc config :init-script "init.sql")] 134 | (test-sql/reset-db) 135 | (let [store (proto/make-store config)] 136 | (proto/init store) 137 | (is (test-sql/verify-table-exists? config "foo"))) 138 | (test-sql/reset-db) 139 | (let [store (proto/make-store (assoc config :init-in-transaction? false))] 140 | (proto/init store) 141 | (is (test-sql/verify-table-exists? config "foo")))))) 142 | 143 | (deftest test-migrate 144 | (is (not (test-sql/verify-table-exists? config "foo"))) 145 | (is (not (test-sql/verify-table-exists? config "bar"))) 146 | (is (not (test-sql/verify-table-exists? config "quux"))) 147 | (is (not (test-sql/verify-table-exists? config "quux2"))) 148 | (core/migrate config) 149 | (is (test-sql/verify-table-exists? config "foo")) 150 | (is (test-sql/verify-table-exists? config "bar")) 151 | (is (test-sql/verify-table-exists? config "quux")) 152 | (is (test-sql/verify-table-exists? config "quux2")) 153 | (core/down config 20111202110600) 154 | (is (not (test-sql/verify-table-exists? config "foo"))) 155 | (is (test-sql/verify-table-exists? config "bar")) 156 | (is (test-sql/verify-table-exists? config "quux")) 157 | (is (test-sql/verify-table-exists? config "quux2")) 158 | (core/migrate config) 159 | (is (test-sql/verify-table-exists? config "foo")) 160 | (is (test-sql/verify-table-exists? config "bar")) 161 | (is (test-sql/verify-table-exists? config "quux")) 162 | (is (test-sql/verify-table-exists? config "quux2")) 163 | (core/down config 20111202110600 20120827170200) 164 | (is (not (test-sql/verify-table-exists? config "foo"))) 165 | (is (test-sql/verify-table-exists? config "bar")) 166 | (is (not (test-sql/verify-table-exists? config "quux"))) 167 | (is (not (test-sql/verify-table-exists? config "quux2"))) 168 | (core/up config 20111202110600 20120827170200) 169 | (is (test-sql/verify-table-exists? config "foo")) 170 | (is (test-sql/verify-table-exists? config "bar")) 171 | (is (test-sql/verify-table-exists? config "quux")) 172 | (is (test-sql/verify-table-exists? config "quux2"))) 173 | 174 | (defn comment-out-bar-statements [sql] 175 | (if (re-find #"CREATE TABLE IF NOT EXISTS bar" sql) 176 | (str "-- " sql) 177 | sql)) 178 | 179 | (defn multi-bar-statements [sql] 180 | (if (re-find #"CREATE TABLE IF NOT EXISTS bar" sql) 181 | ["CREATE TABLE IF NOT EXISTS bar1" 182 | "CREATE TABLE IF NOT EXISTS bar2"] 183 | sql)) 184 | 185 | (deftest test-migrate-with-modify-sql-fn 186 | (is (not (test-sql/verify-table-exists? config "foo"))) 187 | (is (not (test-sql/verify-table-exists? config "bar"))) 188 | (is (not (test-sql/verify-table-exists? config "quux"))) 189 | (is (not (test-sql/verify-table-exists? config "quux2"))) 190 | (core/migrate (assoc config :modify-sql-fn comment-out-bar-statements)) 191 | (is (test-sql/verify-table-exists? config "foo")) 192 | (is (not (test-sql/verify-table-exists? config "bar"))) 193 | (is (test-sql/verify-table-exists? config "quux")) 194 | (is (test-sql/verify-table-exists? config "quux2"))) 195 | 196 | (deftest test-migrate-with-multi-statement-modify-sql-fn 197 | (is (not (test-sql/verify-table-exists? config "foo"))) 198 | (is (not (test-sql/verify-table-exists? config "bar"))) 199 | (is (not (test-sql/verify-table-exists? config "quux"))) 200 | (is (not (test-sql/verify-table-exists? config "quux2"))) 201 | (core/migrate (assoc config :modify-sql-fn multi-bar-statements)) 202 | (is (test-sql/verify-table-exists? config "foo")) 203 | (is (not (test-sql/verify-table-exists? config "bar"))) 204 | (is (test-sql/verify-table-exists? config "bar1")) 205 | (is (test-sql/verify-table-exists? config "bar2")) 206 | (is (test-sql/verify-table-exists? config "quux")) 207 | (is (test-sql/verify-table-exists? config "quux2"))) 208 | 209 | (deftest test-migration-table-creation-is-hooked 210 | (let [hook-called (atom false)] 211 | (core/migrate 212 | (assoc config 213 | :migration-table-name "schema_migrations" 214 | :modify-sql-fn (fn [sql] 215 | (when (re-find #"CREATE TABLE schema_migrations" sql) 216 | (reset! hook-called true)) 217 | sql))) 218 | (is @hook-called))) 219 | 220 | (deftest test-migrate-until-just-before 221 | (is (not (test-sql/verify-table-exists? config "foo"))) 222 | (is (not (test-sql/verify-table-exists? config "bar"))) 223 | (is (not (test-sql/verify-table-exists? config "quux"))) 224 | (is (not (test-sql/verify-table-exists? config "quux2"))) 225 | (core/migrate-until-just-before config 20120827170200) 226 | (is (test-sql/verify-table-exists? config "foo")) 227 | (is (test-sql/verify-table-exists? config "bar")) 228 | (is (not (test-sql/verify-table-exists? config "quux"))) 229 | (is (not (test-sql/verify-table-exists? config "quux2"))) 230 | (core/migrate config) 231 | (is (test-sql/verify-table-exists? config "foo")) 232 | (is (test-sql/verify-table-exists? config "bar")) 233 | (is (test-sql/verify-table-exists? config "quux")) 234 | (is (test-sql/verify-table-exists? config "quux2"))) 235 | 236 | (deftest test-rollback-until-just-after 237 | (core/migrate config) 238 | (is (test-sql/verify-table-exists? config "foo")) 239 | (is (test-sql/verify-table-exists? config "bar")) 240 | (is (test-sql/verify-table-exists? config "quux")) 241 | (is (test-sql/verify-table-exists? config "quux2")) 242 | (core/rollback-until-just-after config 20111202110600) 243 | (is (test-sql/verify-table-exists? config "foo")) 244 | (is (not (test-sql/verify-table-exists? config "bar"))) 245 | (is (not (test-sql/verify-table-exists? config "quux"))) 246 | (is (not (test-sql/verify-table-exists? config "quux2")))) 247 | 248 | 249 | (comment 250 | 251 | (use 'clojure.tools.trace) 252 | (trace-ns migratus.test.migration.sql) 253 | (trace-ns migratus.test.database) 254 | (trace-ns migratus.database) 255 | (trace-ns migratus.migration.sql) 256 | (trace-ns migratus.protocols) 257 | (trace-ns migratus.core) 258 | (trace-ns next.jdbc) 259 | (trace-ns next.jdbc.sql) 260 | (trace-ns next.jdbc.protocols) 261 | 262 | (run-test test-rollback-until-just-after) 263 | (run-test test-squash) 264 | 265 | (core/migrate config) 266 | (db/mark-unreserved (:db config) "foo_bar") 267 | (db/mark-reserved (:db config) "foo_bar") 268 | 269 | (jdbc/execute! (:db config) ["select * from foo_bar"]) 270 | (next.jdbc.sql/insert! (:db config) "foo_bar" {:id -1}) 271 | (jdbc/execute-one! (:db config) ["insert into foo_bar(id) values (?)" -1] {:return-keys false}) 272 | 273 | (jdbc/execute! (:db config) 274 | [(str "CREATE TABLE " (q/ansi "table") 275 | "(id BIGINT UNIQUE NOT NULL, applied TIMESTAMP, 276 | description VARCHAR(1024) )")]) 277 | (run-test test-init) 278 | (run-test test-rollback-until-just-after) 279 | (run-test test-backing-out-bad-migration-no-tx) 280 | 281 | ) 282 | 283 | 284 | (deftest test-migration-ignored-when-already-reserved 285 | (test-with-store 286 | (proto/make-store config) 287 | (fn [{:keys [db migration-table-name] :as config}] 288 | (testing "can only reserve once" 289 | (is (mark-reserved db migration-table-name)) 290 | (is (not (mark-reserved db migration-table-name)))) 291 | (testing "migrations don't run when locked" 292 | (is (not (test-sql/verify-table-exists? config "foo"))) 293 | (is (= :ignore (core/migrate config))) 294 | (is (not (test-sql/verify-table-exists? config "foo")))) 295 | (testing "migrations run once lock is freed" 296 | (mark-unreserved db migration-table-name) 297 | (is (nil? (core/migrate config))) 298 | (is (test-sql/verify-table-exists? config "foo"))) 299 | (testing "rollback migration isn't run when locked" 300 | (is (mark-reserved db migration-table-name)) 301 | (core/down config 20111202110600) 302 | (is (test-sql/verify-table-exists? config "foo"))) 303 | (testing "rollback migration run once lock is freed" 304 | (mark-unreserved db migration-table-name) 305 | (core/down config 20111202110600) 306 | (is (not (test-sql/verify-table-exists? config "foo"))))))) 307 | 308 | (comment 309 | (run-test test-migration-ignored-when-already-reserved) 310 | 311 | ) 312 | 313 | (defn copy-dir 314 | [^File from ^File to] 315 | (when-not (.exists to) 316 | (.mkdirs to)) 317 | (doseq [f (.listFiles from) 318 | :when (.isFile f)] 319 | (io/copy f (io/file to (.getName f))))) 320 | 321 | (deftest test-migration-sql-edn-mixed 322 | (let [migration-dir (io/file "test/migrations-mixed") 323 | test-config (merge config 324 | test-edn/test-config 325 | {:parent-migration-dir "test" 326 | :migration-dir "migrations-mixed"})] 327 | (try 328 | (utils/recursive-delete (io/file test-edn/test-dir)) 329 | (utils/recursive-delete migration-dir) 330 | (copy-dir (io/file "test/migrations") migration-dir) 331 | (copy-dir (io/file "test/migrations-edn") migration-dir) 332 | 333 | (is (not (test-sql/verify-table-exists? test-config "foo"))) 334 | (is (not (test-sql/verify-table-exists? test-config "bar"))) 335 | (is (not (test-sql/verify-table-exists? test-config "quux"))) 336 | (is (not (test-sql/verify-table-exists? test-config "quux2"))) 337 | (is (not (test-edn/test-file-exists?))) 338 | 339 | (core/migrate test-config) 340 | 341 | (is (test-sql/verify-table-exists? test-config "foo")) 342 | (is (test-sql/verify-table-exists? test-config "bar")) 343 | (is (test-sql/verify-table-exists? test-config "quux")) 344 | (is (test-sql/verify-table-exists? test-config "quux2")) 345 | (is (test-edn/test-file-exists?)) 346 | 347 | (finally 348 | (utils/recursive-delete migration-dir))))) 349 | 350 | (deftest test-squashing-list 351 | (core/migrate config) 352 | (let [squashing-list (core/squashing-list config 20111202110600 20241202113000)] 353 | (is (= squashing-list ["create-foo-table" 354 | "create-bar-table" 355 | "multiple-statements"])))) 356 | 357 | (deftest test-create-squash-and-squash-between 358 | (let [migration-dir (io/file "test/migrations-squash") 359 | config (merge config 360 | test-edn/test-config 361 | {:parent-migration-dir "test" 362 | :migration-dir "migrations-squash"})] 363 | (try 364 | (utils/recursive-delete migration-dir) 365 | (copy-dir (io/file "test/migrations1") migration-dir) 366 | (core/migrate config) 367 | (core/create-squash config 20111202110600 20241202113000 "test-squash") 368 | (is (test-sql/verify-table-exists? config "foo1")) 369 | (is (test-sql/verify-table-exists? config "bar1")) 370 | (is (.exists (io/file (str migration-dir "/20220604113000-test-squash.up.sql")))) 371 | (is (.exists (io/file (str migration-dir "/20220604113000-test-squash.down.sql")))) 372 | (core/squash-between config 20111202110600 20241202113000 "test-squash") 373 | (is (test-sql/verify-table-exists? config "foo1")) 374 | (is (test-sql/verify-table-exists? config "bar1")) 375 | (let [from-db (verify-data config (:migration-table-name config))] 376 | (is (= (map #(dissoc % :applied) from-db) 377 | '({:id 20220604113000, 378 | :description "test-squash"})))) 379 | (finally 380 | (utils/recursive-delete migration-dir))))) 381 | 382 | (deftest test-create-squash-with-not-applied-migrations 383 | (let [migration-dir (io/file "test/migrations-squash") 384 | config (merge config 385 | test-edn/test-config 386 | {:parent-migration-dir "test" 387 | :migration-dir "migrations-squash"})] 388 | (try 389 | (utils/recursive-delete migration-dir) 390 | (copy-dir (io/file "test/migrations") migration-dir) 391 | (core/create-squash config 20111202110600 20241202113000 "test-squash") 392 | (catch IllegalArgumentException e 393 | (is (re-find #"Migration 20111202110600 is not applied. Apply it first." (.getMessage e)))) 394 | (finally 395 | (utils/recursive-delete migration-dir))))) 396 | 397 | 398 | 399 | 400 | (comment 401 | 402 | (run-test test-migration-ignored-when-already-reserved) 403 | ) 404 | 405 | 406 | (deftest test-description-and-applied-fields 407 | (core/migrate config) 408 | (let [from-db (verify-data config (:migration-table-name config))] 409 | (testing "descriptions match") 410 | (is (= (map #(dissoc % :applied) from-db) 411 | '({:id 20111202110600, 412 | :description "create-foo-table"} 413 | {:id 20111202113000, 414 | :description "create-bar-table"} 415 | {:id 20120827170200, 416 | :description "multiple-statements"}))) 417 | (testing "applied are timestamps") 418 | (is (every? identity (map #(-> % 419 | :applied 420 | type 421 | (= java.sql.Timestamp)) 422 | from-db))))) 423 | 424 | (defn- test-backing-out* [test-config] 425 | (let [{:keys [db migration-table-name]} test-config] 426 | (testing "should fail") 427 | (is (thrown? Throwable (core/migrate test-config))) 428 | (testing "first statement in migration was backed out because second one failed") 429 | (is (not (test-sql/verify-table-exists? test-config "quux2"))) 430 | (testing "third statement in migration was backed out because second one failed") 431 | (is (not (test-sql/verify-table-exists? test-config "quux3"))) 432 | (testing "migration was not applied") 433 | (is (not (complete? db migration-table-name 20120827170200))) 434 | #_#_(testing "table should be unreserved after migration failure") 435 | (is (false? (mark-reserved db migration-table-name))))) 436 | 437 | (deftest test-backing-out-bad-migration 438 | (log/debug "running backout tests") 439 | (test-backing-out* (assoc config :migration-dir "migrations-intentionally-broken"))) 440 | 441 | (deftest test-backing-out-bad-migration-no-tx 442 | (log/debug "running backout tests without tx") 443 | (test-backing-out* (assoc config :migration-dir "migrations-intentionally-broken-no-tx"))) 444 | 445 | (deftest test-comment-parsing-in-migration 446 | (log/debug "running comment parsing test") 447 | (is (not (test-sql/verify-table-exists? config "quux"))) 448 | (core/migrate (assoc config :migration-dir "migrations-parse")) 449 | (is (test-sql/verify-table-exists? config "quux")) 450 | (core/down config 20241012170200)) 451 | 452 | (deftest test-no-tx-migration 453 | (let [{:keys [db migration-table-name] :as test-config} (assoc config :migration-dir "migrations-no-tx")] 454 | (is (not (test-sql/verify-table-exists? test-config "foo"))) 455 | (core/migrate test-config) 456 | (is (test-sql/verify-table-exists? test-config "foo")) 457 | (core/down test-config 20111202110600) 458 | (is (not (test-sql/verify-table-exists? test-config "foo"))))) 459 | 460 | (deftest test-no-tx-migration-pass-conn 461 | (with-open [conn (jdbc/get-connection (:db config))] 462 | (let [test-config (assoc config 463 | :migration-dir "migrations-no-tx" 464 | :db {:connection conn :managed-connection? true})] 465 | (is (not (test-sql/verify-table-exists? test-config "foo"))) 466 | (core/migrate test-config) 467 | (is (test-sql/verify-table-exists? test-config "foo")) 468 | (core/down test-config 20111202110600) 469 | (is (not (test-sql/verify-table-exists? test-config "foo")))))) 470 | 471 | (deftest test-cancellation-observed 472 | (let [lines-processed (atom 0) 473 | future-instance (atom nil) 474 | future-instance-set (promise) 475 | migration-in-future (future (core/migrate 476 | (assoc config 477 | :migration-table-name "schema_migrations" 478 | :modify-sql-fn (fn [sql] 479 | (when (re-find #"CREATE TABLE schema_migrations" sql) 480 | (deref future-instance-set) 481 | (future-cancel @future-instance)) 482 | (swap! lines-processed inc) 483 | sql))))] 484 | (reset! future-instance migration-in-future) 485 | (deliver future-instance-set true) 486 | (is (thrown? CancellationException @migration-in-future)) 487 | (Thread/sleep 100) 488 | (is (= 1 @lines-processed)))) 489 | -------------------------------------------------------------------------------- /test/migratus/test/migration/edn.clj: -------------------------------------------------------------------------------- 1 | (ns migratus.test.migration.edn 2 | (:require [clojure.java.io :as io] 3 | [clojure.test :refer :all] 4 | [migratus.core :as core] 5 | [migratus.migration.edn :refer :all] 6 | migratus.mock 7 | [migratus.protocols :as proto] 8 | [migratus.utils :as utils]) 9 | (:import java.io.File)) 10 | 11 | (defn unload [ns-sym] 12 | (remove-ns ns-sym) 13 | (dosync 14 | (commute (deref #'clojure.core/*loaded-libs*) disj ns-sym))) 15 | 16 | (def test-namespace 'migratus.test.migration.edn.test-script) 17 | (def test-dir "target/edn-test") 18 | (def test-config {:output-dir test-dir}) 19 | 20 | (defn test-file-exists? [] 21 | (let [f (io/file test-dir "hello.txt")] 22 | (and (.exists f) 23 | (= "Hello, world!" (slurp f))))) 24 | 25 | (use-fixtures :once 26 | (fn [f] 27 | (f) 28 | ;; `lein test` thinks it needs to test this namespace, so make sure 29 | ;; that it exists when we're done 30 | (require test-namespace))) 31 | 32 | (use-fixtures :each 33 | (fn [f] 34 | ;; unload the namespace before each test to ensure that it's loaded 35 | ;; appropriately by the edn-migration code. 36 | (unload test-namespace) 37 | (utils/recursive-delete (io/file test-dir)) 38 | (f))) 39 | 40 | (deftest test-to-sym 41 | (are [x y] (= y (to-sym x)) 42 | nil nil 43 | "aaa" 'aaa 44 | 'aaa 'aaa 45 | :aaa 'aaa) 46 | (are [x] (thrown-with-msg? 47 | IllegalArgumentException 48 | #"Namespaced symbol not allowed" 49 | (to-sym x)) 50 | "aaa/bbb" 51 | 'aaa/bbb 52 | :aaa/bbb 53 | 'a.b.c/def 54 | :a.b.c/def)) 55 | 56 | (deftest test-resolve-fn 57 | (require test-namespace) 58 | (is (var? (resolve-fn "test-mig" test-namespace "migrate-up"))) 59 | (is (var? (resolve-fn "test-mig" test-namespace 'migrate-up))) 60 | (is (var? (resolve-fn "test-mig" test-namespace :migrate-up))) 61 | (is (thrown-with-msg? 62 | IllegalArgumentException 63 | #"Unable to resolve" 64 | (resolve-fn "test-mig" test-namespace "not-a-fn"))) 65 | (is (thrown-with-msg? 66 | IllegalArgumentException 67 | #"Namespaced symbol not allowed" 68 | (resolve-fn "test-mig" test-namespace "clojure.core/map")))) 69 | 70 | (defn edn-mig [content] 71 | (proto/make-migration* :edn 1 "edn-migration" (pr-str content) nil)) 72 | 73 | (deftest test-invalid-migration 74 | (testing "namespace is required" 75 | (is (thrown-with-msg? 76 | IllegalArgumentException 77 | #"Invalid migration .* no namespace" 78 | (edn-mig {})))) 79 | (testing "namespace must exist" 80 | (is (thrown? 81 | Exception 82 | (edn-mig {:ns 'foo.bar.baz})))) 83 | (testing "fn must exist" 84 | (is (thrown-with-msg? 85 | IllegalArgumentException 86 | #"Unable to resolve" 87 | (edn-mig {:ns test-namespace 88 | :up-fn 'not-a-real-fn 89 | :down-fn 'not-a-real-fn}))))) 90 | 91 | (deftest test-edn-migration 92 | (let [mig (edn-mig {:ns test-namespace 93 | :up-fn 'migrate-up 94 | :down-fn 'migrate-down})] 95 | (is (not (test-file-exists?))) 96 | (proto/up mig test-config) 97 | (is (test-file-exists?)) 98 | (proto/down mig test-config) 99 | (is (not (test-file-exists?))))) 100 | 101 | (deftest test-edn-down-optional 102 | (let [mig (edn-mig {:ns test-namespace 103 | :up-fn 'migrate-up 104 | :down-fn nil})] 105 | (is (not (test-file-exists?))) 106 | (proto/up mig test-config) 107 | (is (test-file-exists?)) 108 | (proto/down mig test-config) 109 | (is (test-file-exists?)))) 110 | 111 | (deftest test-run-edn-migrations 112 | (let [config (merge test-config 113 | {:store :mock 114 | :completed-ids (atom #{}) 115 | :migration-dir "migrations-edn"})] 116 | (is (not (test-file-exists?))) 117 | (core/migrate config) 118 | (is (test-file-exists?)) 119 | (core/rollback config) 120 | (is (not (test-file-exists?))))) 121 | -------------------------------------------------------------------------------- /test/migratus/test/migration/edn/test_script.clj: -------------------------------------------------------------------------------- 1 | (ns migratus.test.migration.edn.test-script 2 | (:require [clojure.java.io :as io])) 3 | 4 | (def test-file-name "hello.txt") 5 | 6 | (defn migrate-up [{:keys [output-dir]}] 7 | (.mkdirs (io/file output-dir)) 8 | (spit (io/file output-dir test-file-name) "Hello, world!")) 9 | 10 | (defn migrate-down [config] 11 | (let [f (io/file (:output-dir config) test-file-name)] 12 | (when (.exists f) 13 | (.delete f)))) 14 | -------------------------------------------------------------------------------- /test/migratus/test/migration/edn/test_script_args.clj: -------------------------------------------------------------------------------- 1 | (ns migratus.test.migration.edn.test-script-args 2 | (:require [clojure.java.io :as io])) 3 | 4 | (defn migrate-up [{:keys [output-dir]} filename msg] 5 | (.mkdirs (io/file output-dir)) 6 | (spit (io/file output-dir filename) msg)) 7 | 8 | (defn migrate-down [{:keys [output-dir]} filename] 9 | (let [f (io/file output-dir filename)] 10 | (when (.exists f) 11 | (.delete f)))) 12 | -------------------------------------------------------------------------------- /test/migratus/test/migration/edn_with_args.clj: -------------------------------------------------------------------------------- 1 | (ns migratus.test.migration.edn-with-args 2 | (:require [clojure.java.io :as io] 3 | [clojure.test :refer :all] 4 | [migratus.core :as core] 5 | [migratus.migration.edn :refer :all] 6 | migratus.mock 7 | [migratus.protocols :as proto] 8 | [migratus.utils :as utils]) 9 | (:import java.io.File)) 10 | 11 | (defn unload [ns-sym] 12 | (remove-ns ns-sym) 13 | (dosync 14 | (commute (deref #'clojure.core/*loaded-libs*) disj ns-sym))) 15 | 16 | (def test-namespace 'migratus.test.migration.edn.test-script-args) 17 | (def test-dir "target/edn-args-test") 18 | (def test-config {:output-dir test-dir}) 19 | 20 | (defn test-file-exists? [] 21 | (let [f (io/file test-dir "hello-with-args.txt")] 22 | (and (.exists f) 23 | (= "Hello, world, with args!" (slurp f))))) 24 | 25 | (use-fixtures :once 26 | (fn [f] 27 | (f) 28 | ;; `lein test` thinks it needs to test this namespace, so make sure 29 | ;; that it exists when we're done 30 | (require test-namespace))) 31 | 32 | (use-fixtures :each 33 | (fn [f] 34 | ;; unload the namespace before each test to ensure that it's loaded 35 | ;; appropriately by the edn-migration code. 36 | (unload test-namespace) 37 | (utils/recursive-delete (io/file test-dir)) 38 | (f))) 39 | 40 | (deftest test-run-edn-migrations 41 | (let [config (merge test-config 42 | {:store :mock 43 | :completed-ids (atom #{}) 44 | :migration-dir "migrations-edn-args"})] 45 | (is (not (test-file-exists?))) 46 | (core/migrate config) 47 | (is (test-file-exists?)) 48 | (core/rollback config) 49 | (is (not (test-file-exists?))))) 50 | -------------------------------------------------------------------------------- /test/migratus/test/migration/sql.clj: -------------------------------------------------------------------------------- 1 | (ns migratus.test.migration.sql 2 | (:require [clojure.java.io :as io] 3 | [clojure.test :refer :all] 4 | [migratus.core :as core] 5 | [migratus.database :as db] 6 | [migratus.migration.sql :refer :all] 7 | [next.jdbc :as jdbc] 8 | [next.jdbc.result-set :as rs])) 9 | 10 | (def db-store (str (.getName (io/file ".")) "/site.db")) 11 | 12 | (def db-spec {:dbtype "h2" 13 | :dbname db-store}) 14 | 15 | (def test-config {:migration-dir "migrations/" 16 | :db db-spec}) 17 | 18 | (defn db-tables-and-views 19 | "Fetch tables and views (database metadata) from DB. 20 | Returns a collection of table metadata." 21 | [datasource] 22 | (with-open [con (jdbc/get-connection datasource)] 23 | (-> (.getMetaData con) 24 | (.getTables nil nil nil (into-array ["TABLE" "VIEW"])) 25 | (rs/datafiable-result-set datasource)))) 26 | 27 | (defn reset-db [] 28 | (letfn [(delete [f] 29 | (when (.exists f) 30 | (.delete f)))] 31 | (delete (io/file "site.db.trace.db")) 32 | (delete (io/file "site.db.mv.db")) 33 | (delete (io/file "site.db")))) 34 | 35 | (defn setup-test-db [f] 36 | (reset-db) 37 | (f)) 38 | 39 | (use-fixtures :each setup-test-db) 40 | 41 | (defn verify-table-exists? [config table-name] 42 | (let [db (:db config)] 43 | (db/table-exists? db table-name))) 44 | 45 | (deftest test-run-sql-migrations 46 | (let [config (merge test-config 47 | {:store :mock 48 | :completed-ids (atom #{})})] 49 | 50 | (is (not (verify-table-exists? config "foo"))) 51 | (is (not (verify-table-exists? config "bar"))) 52 | (is (not (verify-table-exists? config "quux"))) 53 | (is (not (verify-table-exists? config "quux2"))) 54 | 55 | (core/migrate config) 56 | 57 | (is (verify-table-exists? config "foo")) 58 | (is (verify-table-exists? config "bar")) 59 | (is (verify-table-exists? config "quux")) 60 | (is (verify-table-exists? config "quux2")) 61 | 62 | (core/rollback config) 63 | 64 | (is (verify-table-exists? config "foo")) 65 | (is (verify-table-exists? config "bar")) 66 | (is (not (verify-table-exists? config "quux"))) 67 | (is (not (verify-table-exists? config "quux2"))))) 68 | 69 | 70 | (comment 71 | (use 'clojure.tools.trace) 72 | 73 | (trace-ns clojure.test) 74 | (trace-ns migratus.test.migration.sql) 75 | (trace-ns migratus.test.database) 76 | (trace-ns migratus.database) 77 | (trace-ns migratus.migration.sql) 78 | (trace-ns migratus.protocols) 79 | (trace-ns migratus.core) 80 | (trace-ns migratus.mock) 81 | (trace-ns next.jdbc) 82 | (trace-ns next.jdbc.sql) 83 | (trace-ns next.jdbc.protocols) 84 | 85 | 86 | (run-test test-run-sql-migrations) 87 | 88 | 0 89 | ) 90 | -------------------------------------------------------------------------------- /test/migratus/test/migrations.clj: -------------------------------------------------------------------------------- 1 | (ns migratus.test.migrations 2 | (:require 3 | [clojure.test :refer [deftest is]] 4 | [migratus.migration.sql :as sql-mig] 5 | [migratus.migrations :as sut] 6 | [migratus.properties :as props] 7 | [migratus.utils :as utils])) 8 | 9 | (deftest test-parse-name 10 | (is (= ["20111202110600" "create-foo-table" ["up" "sql"]] 11 | (sut/parse-name "20111202110600-create-foo-table.up.sql"))) 12 | (is (= ["20111202110600" "create-foo-table" ["down" "sql"]] 13 | (sut/parse-name "20111202110600-create-foo-table.down.sql")))) 14 | 15 | (def multi-stmt-up 16 | (str "-- this is the first statement\n\n" 17 | "CREATE TABLE\nquux\n" 18 | "(id bigint,\n" 19 | " name varchar(255));\n\n" 20 | "--;;\n" 21 | "-- comment for the second statement\n\n" 22 | "CREATE TABLE quux2(id bigint, name varchar(255));\n")) 23 | 24 | (def multi-stmt-down 25 | (str "DROP TABLE quux2;\n" 26 | "--;;\n" 27 | "DROP TABLE quux;\n")) 28 | 29 | (deftest test-properties 30 | (is (nil? (props/load-properties {}))) 31 | (is (number? (get (props/load-properties {:properties {}}) "${migratus.timestamp}"))) 32 | (let [props (props/load-properties 33 | {:properties 34 | {:env ["path"] 35 | :map {:foo "bar" 36 | :baz {:bar "foo"}}}})] 37 | (is (seq (get props "${path}"))) 38 | (is (= "bar" (get props "${foo}"))) 39 | (is (= "foo" (get props "${baz.bar}"))))) 40 | 41 | (deftest test-find-migrations 42 | (let [create-migrations {"20111202113000" 43 | {"create-bar-table" 44 | {:sql 45 | {:up "CREATE TABLE IF NOT EXISTS bar(id BIGINT);\n" 46 | :down "DROP TABLE IF EXISTS bar;\n"}}} 47 | "20111202110600" 48 | {"create-foo-table" 49 | {:sql 50 | {:up "CREATE TABLE IF NOT EXISTS foo(id bigint);\n" 51 | :down "DROP TABLE IF EXISTS foo;\n"}}}} 52 | migrations (assoc create-migrations "20120827170200" 53 | {"multiple-statements" 54 | {:sql 55 | {:up multi-stmt-up 56 | :down multi-stmt-down}}})] 57 | (is (= migrations (sut/find-migrations "migrations" ["init.sql"] nil)) 58 | "single migrations dir") 59 | (is (= create-migrations (sut/find-migrations "migrations" ["init.sql" "*-multiple-*"] nil)) 60 | "single migrations dir with glob exclusions")) 61 | (is (= {"20220604110500" 62 | {"create-foo1-table" 63 | {:sql 64 | {:down "DROP TABLE IF EXISTS foo1;" 65 | :up "CREATE TABLE IF NOT EXISTS foo1(id bigint);"}}} 66 | "20220604113000" 67 | {"create-bar1-table" 68 | {:sql 69 | {:down "DROP TABLE IF EXISTS bar1;" 70 | :up "CREATE TABLE IF NOT EXISTS bar1(id BIGINT);"}}} 71 | "20220604113500" 72 | {"create-bar2-table" 73 | {:sql 74 | {:up "CREATE TABLE IF NOT EXISTS bar2(id BIGINT);" 75 | :down "DROP TABLE IF EXISTS bar2;"}}} 76 | "20220604111500" 77 | {"create-foo2-table" 78 | {:sql 79 | {:down "DROP TABLE IF EXISTS foo2;", 80 | :up "CREATE TABLE IF NOT EXISTS foo2(id bigint);"}}}} 81 | (sut/find-migrations ["migrations1" "migrations2"] [] nil)) 82 | "multiple migrations dirs") 83 | (is (= {"20111202110600" {"create-foo-table" {:sql {:up "CREATE TABLE IF NOT EXISTS foo(id bigint);\n", 84 | :down "DROP TABLE IF EXISTS TEST_SCHEMA.foo;\n"}}, 85 | "create-schema" {:sql {:up "CREATE SCHEMA TEST_SCHEMA\n"}}}} 86 | (sut/find-migrations "migrations-with-props" [] {"${migratus.schema}" "TEST_SCHEMA"})))) 87 | 88 | (deftest test-find-jar-migrations 89 | (let [dir "migrations-in-jar" 90 | url (java.net.URL. (str "jar:file:test/migrations-jar/migrations.jar!/" dir))] 91 | (is (not (nil? (utils/jar-file url)))))) 92 | 93 | (deftest test-list-migrations 94 | (is (= #{(sql-mig/->SqlMigration 95 | 20111202113000 96 | "create-bar-table" 97 | "CREATE TABLE IF NOT EXISTS bar(id BIGINT);\n" 98 | "DROP TABLE IF EXISTS bar;\n") 99 | (sql-mig/->SqlMigration 100 | 20111202110600 101 | "create-foo-table" 102 | "CREATE TABLE IF NOT EXISTS foo(id bigint);\n" 103 | "DROP TABLE IF EXISTS foo;\n") 104 | (sql-mig/->SqlMigration 105 | 20120827170200 106 | "multiple-statements" 107 | multi-stmt-up 108 | multi-stmt-down)} 109 | (set (sut/list-migrations {:migration-dir "migrations"}))))) 110 | 111 | (deftest test-list-migrations-bad-type 112 | (is (empty? 113 | (sut/list-migrations {:migration-dir "migrations-bad-type"})))) 114 | 115 | (deftest test-list-migrations-duplicate-type 116 | (is (thrown-with-msg? 117 | Exception 118 | #"Multiple migration types" 119 | (sut/list-migrations {:migration-dir "migrations-duplicate-type"})))) 120 | 121 | (deftest test-list-migrations-duplicate-name 122 | (is (thrown-with-msg? 123 | Exception 124 | #"Multiple migrations with id" 125 | (sut/list-migrations {:migration-dir "migrations-duplicate-name"})))) 126 | -------------------------------------------------------------------------------- /test/migratus/test/utils.clj: -------------------------------------------------------------------------------- 1 | (ns migratus.test.utils 2 | (:require [clojure.test :refer :all] 3 | [migratus.utils :refer :all])) 4 | 5 | (deftest test-censor-password 6 | (is (= nil (censor-password nil))) 7 | (is (= "" (censor-password ""))) 8 | (is (= {:password nil} (censor-password {:password nil}))) 9 | (is (= {:password "1" :user "user"} 10 | (censor-password {:password "1234" :user "user"}))) 11 | (is (= "uri-censored" 12 | (censor-password 13 | "jdbc:postgresql://fake.example.org/my_dev?user=my_user&password=thisIsNot123ARealPass"))) 14 | (is (= {:connection-uri "uri-censored"} 15 | (censor-password {:connection-uri "jdbc:postgresql://fake.example.org/my_dev?user=my_user&password=thisIsNot123ARealPass"}))) 16 | (is (= {:connection-uri "uri-censored" :password "1" :user "user"} 17 | (censor-password {:password "1234" :user "user" 18 | :connection-uri "jdbc:postgresql://fake.example.org/my_dev?user=my_user&password=thisIsNot123ARealPass"})))) 19 | 20 | (deftest test-jar-name 21 | (is (nil? (jar-name nil))) 22 | (testing "handles file prefix" 23 | (is (= "///tmp/default/clojure-1.10.1.jar" 24 | (jar-name "file:///tmp/default/clojure-1.10.1.jar")))) 25 | (testing "handles '+' in paths" 26 | (is (= "/tmp/default+uberjar/foo.jar" 27 | (jar-name "/tmp/default+uberjar/foo.jar"))))) 28 | -------------------------------------------------------------------------------- /test/migratus/testcontainers/postgres.clj: -------------------------------------------------------------------------------- 1 | (ns migratus.testcontainers.postgres 2 | "Integration tests for postgresql using testcontainers.org" 3 | {:authors ["Eugen Stan"]} 4 | (:require [clj-test-containers.core :as tc] 5 | [clojure.tools.logging :as log] 6 | [clojure.set :as set] 7 | [clojure.test :refer [deftest is testing]] 8 | [migratus.test.migration.sql :as test-sql] 9 | [migratus.core :as migratus] 10 | [next.jdbc :as jdbc] 11 | [next.jdbc.transaction :as jdbc-tx])) 12 | 13 | (def postgres-image (or (System/getenv "MIGRATUS_TESTCONTAINERS_POSTGRES_IMAGE") 14 | "postgres:14")) 15 | 16 | (def pg-container-spec {:image-name postgres-image 17 | :exposed-ports [5432] 18 | :env-vars {"POSTGRES_PASSWORD" "pw"} 19 | :wait-for {:wait-strategy :port}}) 20 | 21 | 22 | (deftest postgres-migrations-test 23 | 24 | (testing "Migrations are applied succesfully in PostgreSQL." 25 | (binding [;; Testing that everything works even if some wrapper code prohibits nested transactions. 26 | jdbc-tx/*nested-tx* :prohibit] 27 | (let [pg-container (tc/create pg-container-spec) 28 | initialized-pg-container (tc/start! pg-container) 29 | meta->table-names #(into #{} (map :pg_class/table_name %))] 30 | (Thread/sleep 1000) 31 | (let [ds (jdbc/get-datasource {:dbtype "postgresql" 32 | :dbname "postgres" 33 | :user "postgres" 34 | :password "pw" 35 | :host (:host initialized-pg-container) 36 | :port (get (:mapped-ports initialized-pg-container) 5432)}) 37 | config {:store :database 38 | :migration-dir "migrations-postgres" 39 | :init-script "init.sql" 40 | :migration-table-name "foo_bar" 41 | :db {:datasource ds}}] 42 | (is (= [] (test-sql/db-tables-and-views ds)) "db is empty before migrations") 43 | 44 | ;; init 45 | (migratus/init config) 46 | (let [db-meta (test-sql/db-tables-and-views ds) 47 | table-names (meta->table-names db-meta)] 48 | (is (= #{"foo"} table-names) "db is initialized")) 49 | 50 | ;; migrate 51 | (migratus/migrate config) 52 | (let [db-meta (test-sql/db-tables-and-views ds) 53 | table-names (meta->table-names db-meta) 54 | expected-tables #{"quux" "foo" "foo_bar"}] 55 | (log/info "Tables are" table-names) 56 | (is (= (count expected-tables) (count db-meta)) 57 | (str "expected table count is ok.")) 58 | 59 | (is (set/subset? expected-tables table-names) 60 | "contains some tables that we expect"))) 61 | 62 | (tc/stop! initialized-pg-container))))) 63 | -------------------------------------------------------------------------------- /tests.edn: -------------------------------------------------------------------------------- 1 | #kaocha/v1 2 | {:tests [{:id :unit 3 | :test-paths ["test"] 4 | :ns-patterns ["migratus\\.test\\..*"]} 5 | {:id :testcontainers 6 | :test-paths ["test"] 7 | :ns-patterns ["migratus\\.testcontainers\\..*"]}] 8 | :plugins [:kaocha.plugin/junit-xml 9 | :kaocha.plugin/cloverage 10 | :kaocha.plugin.alpha/spec-test-check] 11 | :reporter [kaocha.report/documentation 12 | kaocha.report/dots] 13 | :kaocha.plugin.junit-xml/target-file "target/test-reports/junit.xml"} 14 | --------------------------------------------------------------------------------