├── .docker └── clickhouse │ ├── config.xml │ └── users.xml ├── .github └── workflows │ ├── release.yml │ └── tests.yml ├── .gitignore ├── .prettierrc ├── CHANGELOG.md ├── LICENSE ├── README.md ├── docker-compose.yml ├── eslint.config.mjs ├── jestconfig.json ├── package-lock.json ├── package.json ├── src ├── cli.ts ├── migrate.ts ├── sql-parse.ts └── types │ └── cli.d.ts ├── tests ├── cli.test.ts ├── db.test.ts ├── migrations │ ├── bad │ │ └── bad_1.sql │ ├── few │ │ ├── 1_init.sql │ │ └── 2_more.sql │ └── one │ │ └── 1_init.sql └── unit.test.ts └── tsconfig.json /.docker/clickhouse/config.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 8123 5 | 9000 6 | 7 | users.xml 8 | default 9 | default 10 | 11 | 5368709120 12 | 13 | /var/lib/clickhouse/ 14 | /var/lib/clickhouse/tmp/ 15 | /var/lib/clickhouse/user_files/ 16 | /var/lib/clickhouse/access/ 17 | 3 18 | 19 | 20 | debug 21 | /var/log/clickhouse-server/clickhouse-server.log 22 | /var/log/clickhouse-server/clickhouse-server.err.log 23 | 1000M 24 | 10 25 | 1 26 | 27 | 28 | 29 | system 30 | query_log
31 | toYYYYMM(event_date) 32 | 1000 33 | 34 | 35 | 36 |
37 | Access-Control-Allow-Origin 38 | * 39 |
40 |
41 | Access-Control-Allow-Headers 42 | accept, origin, x-requested-with, content-type, authorization 43 |
44 |
45 | Access-Control-Allow-Methods 46 | POST, GET, OPTIONS 47 |
48 |
49 | Access-Control-Max-Age 50 | 86400 51 |
52 | 53 | 54 | -------------------------------------------------------------------------------- /.docker/clickhouse/users.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | random 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | ::/0 15 | 16 | default 17 | default 18 | 1 19 | 20 | 21 | 22 | 23 | 24 | 25 | 3600 26 | 0 27 | 0 28 | 0 29 | 0 30 | 0 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: release-please 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | jobs: 8 | release-please: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: googleapis/release-please-action@v4 12 | id: release 13 | with: 14 | release-type: node 15 | 16 | # publish npm: 17 | - uses: actions/checkout@v4 18 | # a publication only occurs when a new release is created: 19 | if: ${{ steps.release.outputs.release_created }} 20 | - uses: actions/setup-node@v4 21 | with: 22 | node-version: 20 23 | registry-url: 'https://registry.npmjs.org' 24 | if: ${{ steps.release.outputs.release_created }} 25 | - run: npm ci 26 | if: ${{ steps.release.outputs.release_created }} 27 | - run: npm publish 28 | env: 29 | NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}} 30 | if: ${{ steps.release.outputs.release_created }} 31 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: 'tests' 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | branches: 7 | - main 8 | paths-ignore: 9 | - '**/*.md' 10 | 11 | schedule: 12 | - cron: '0 9 * * *' 13 | 14 | jobs: 15 | unit-tests: 16 | runs-on: ubuntu-latest 17 | strategy: 18 | fail-fast: true 19 | matrix: 20 | node: [18, 20, 22] 21 | steps: 22 | - uses: actions/checkout@main 23 | 24 | - name: Setup NodeJS ${{ matrix.node }} 25 | uses: actions/setup-node@v4 26 | with: 27 | node-version: ${{ matrix.node }} 28 | 29 | - name: Install dependencies 30 | run: | 31 | npm ci 32 | 33 | - name: Linting 34 | run: | 35 | npm run lint 36 | 37 | - name: Unit testing 38 | run: | 39 | npm run test 40 | 41 | node-integration-tests-local-single-node: 42 | needs: unit-tests 43 | runs-on: ubuntu-latest 44 | strategy: 45 | fail-fast: true 46 | matrix: 47 | node: [18, 20, 22] 48 | clickhouse: [head, latest] 49 | 50 | steps: 51 | - uses: actions/checkout@main 52 | 53 | - name: Start ClickHouse (version - ${{ matrix.clickhouse }}) using docker-compose 54 | uses: isbang/compose-action@v1.5.1 55 | env: 56 | CLICKHOUSE_VERSION: ${{ matrix.clickhouse }} 57 | with: 58 | compose-file: 'docker-compose.yml' 59 | down-flags: '--volumes' 60 | 61 | - name: Setup NodeJS ${{ matrix.node }} 62 | uses: actions/setup-node@v4 63 | with: 64 | node-version: ${{ matrix.node }} 65 | 66 | - name: Install dependencies 67 | run: | 68 | npm ci 69 | 70 | - name: Build 71 | run: | 72 | npm run build 73 | 74 | - name: Test - apply migrations ./tests/migrations/few 75 | run: | 76 | node ./lib/cli.js migrate --host=http://localhost:8123 --user=default --password='' --db=analytics --migrations-home=./tests/migrations/few 77 | 78 | - name: Verify Migration 79 | run: | 80 | echo "Checking if tables exist..." 81 | response=$(curl -s "http://localhost:8123/?query=SELECT%20groupArray(name)%20FROM%20system.tables%20WHERE%20name%20IN%20('_migrations','events','sessions')%20AND%20database%20=%20'analytics'") 82 | echo "Response: $response" 83 | 84 | expected_tables="['_migrations','events','sessions']" 85 | 86 | if [ "$response" = "$expected_tables" ]; then 87 | echo "The tables were created successfully." 88 | else 89 | echo "Tables were not found!" 90 | echo "Expected: $expected_tables" 91 | echo "Got: $response" 92 | exit 1 93 | fi 94 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | lib 3 | .DS_Store -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "printWidth": 120, 3 | "trailingComma": "all", 4 | "singleQuote": true 5 | } -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## [1.0.4](https://github.com/VVVi/clickhouse-migrations/compare/v1.0.3...v1.0.4) (2025-01-21) 4 | 5 | 6 | ### Features 7 | 8 | * update to eslint 9, update dependencies ([f91cf7d](https://github.com/VVVi/clickhouse-migrations/commit/f91cf7d83bbccde937de00206092e0474a885fac)) 9 | 10 | 11 | ### Miscellaneous Chores 12 | 13 | * release 1.0.4 ([2b653e3](https://github.com/VVVi/clickhouse-migrations/commit/2b653e3cdcb0b5d7db0bcb47ab99dd6ae4dd389b)) 14 | 15 | ## [1.0.3](https://github.com/VVVi/clickhouse-migrations/compare/v1.0.2...v1.0.3) (2025-01-20) 16 | 17 | 18 | ### Features 19 | 20 | * update packages to latest version ([4125b31](https://github.com/VVVi/clickhouse-migrations/commit/4125b3128d4fbd4ffe31c0170801b24ec2e0bc42)) 21 | 22 | 23 | ### Miscellaneous Chores 24 | 25 | * release 1.0.3 ([3f50c00](https://github.com/VVVi/clickhouse-migrations/commit/3f50c00910d6c76fe2db1e8f3d963ee3451f5864)) 26 | 27 | ## [1.0.2](https://github.com/VVVi/clickhouse-migrations/compare/v1.0.1...v1.0.2) (2024-09-14) 28 | 29 | 30 | ### Miscellaneous Chores 31 | 32 | * release 1.0.2 ([f761a87](https://github.com/VVVi/clickhouse-migrations/commit/f761a875da5a6a8038467a4322ba5d0f3df87e9d)) 33 | * release 1.0.2 ([a194e0c](https://github.com/VVVi/clickhouse-migrations/commit/a194e0ccd316a0f4370995f9da19de02798c5777)) 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 VVVi 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # clickhouse-migrations 3 | 4 | > ClickHouse Migrations CLI 5 | 6 | ## Install 7 | 8 | ```sh 9 | npm install clickhouse-migrations 10 | ``` 11 | 12 | ## Usage 13 | 14 | Create a directory, where migrations will be stored. It will be used as the value for the `--migrations-home` option (or for environment variable `CH_MIGRATIONS_HOME`). 15 | 16 | In the directory, create migration files, which should be named like this: `1_some_text.sql`, `2_other_text.sql`, `10_more_test.sql`. What's important here is that the migration version number should come first, followed by an underscore (`_`), and then any text can follow. The version number should increase for every next migration. Please note that once a migration file has been applied to the database, it cannot be modified or removed. 17 | 18 | For migrations' content should be used correct SQL ClickHouse queries. Multiple queries can be used in a single migration file, and each query should be terminated with a semicolon (;). The queries could be idempotent - for example: `CREATE TABLE IF NOT EXISTS table ...;` Clickhouse settings, that can be included at the query level, can be added like `SET allow_experimental_object_type = 1;`. For adding comments should be used `--`, `# `, `#!`. 19 | 20 | If the database provided in the `--db` option (or in `CH_MIGRATIONS_DB`) doesn't exist, it will be created automatically. 21 | 22 | ``` 23 | Usage 24 | $ clickhouse-migrations migrate 25 | 26 | Required options 27 | --host= Clickhouse hostname 28 | (ex. https://clickhouse:8123) 29 | --user= Username 30 | --password= Password 31 | --db= Database name 32 | --migrations-home= Migrations' directory 33 | 34 | Optional options 35 | --db-engine= ON CLUSTER and/or ENGINE for DB 36 | (default: 'ENGINE=Atomic') 37 | --timeout= Client request timeout 38 | (milliseconds, default: 30000) 39 | 40 | Environment variables 41 | Instead of options can be used environment variables. 42 | CH_MIGRATIONS_HOST Clickhouse hostname (--host) 43 | CH_MIGRATIONS_USER Username (--user) 44 | CH_MIGRATIONS_PASSWORD Password (--password) 45 | CH_MIGRATIONS_DB Database name (--db) 46 | CH_MIGRATIONS_HOME Migrations' directory (--migrations-home) 47 | 48 | CH_MIGRATIONS_DB_ENGINE (optional) DB engine (--db-engine) 49 | CH_MIGRATIONS_TIMEOUT (optional) Client request timeout 50 | (--timeout) 51 | 52 | CLI executions examples 53 | settings are passed as command-line options 54 | clickhouse-migrations migrate --host=http://localhost:8123 55 | --user=default --password='' --db=analytics 56 | --migrations-home=/app/clickhouse/migrations 57 | 58 | settings provided as options, including timeout and db-engine 59 | clickhouse-migrations migrate --host=http://localhost:8123 60 | --user=default --password='' --db=analytics 61 | --migrations-home=/app/clickhouse/migrations --timeout=60000 62 | --db-engine="ON CLUSTER default ENGINE=Replicated('{replica}')" 63 | 64 | settings provided as environment variables 65 | clickhouse-migrations migrate 66 | 67 | settings provided partially through options and environment variables 68 | clickhouse-migrations migrate --timeout=60000 69 | ``` 70 | 71 | Migration file example: 72 | (e.g., located at /app/clickhouse/migrations/1_init.sql) 73 | ``` 74 | -- an example of migration file 1_init.sql 75 | 76 | SET allow_experimental_json_type = 1; 77 | 78 | CREATE TABLE IF NOT EXISTS events ( 79 | timestamp DateTime('UTC'), 80 | session_id UInt64, 81 | event JSON 82 | ) 83 | ENGINE=AggregatingMergeTree 84 | PARTITION BY toYYYYMM(timestamp) 85 | SAMPLE BY session_id 86 | ORDER BY (session_id) 87 | SETTINGS index_granularity = 8192; 88 | ``` -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | #version: '3.8' 2 | 3 | services: 4 | clickhouse: 5 | container_name: 'clickhouse-server' 6 | image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-24.8-alpine}' 7 | ports: 8 | - '8123:8123' 9 | - '9000:9000' 10 | ulimits: 11 | nofile: 12 | soft: 262144 13 | hard: 262144 14 | volumes: 15 | - './.docker/clickhouse/config.xml:/etc/clickhouse-server/config.xml' 16 | - './.docker/clickhouse/users.xml:/etc/clickhouse-server/users.xml' 17 | -------------------------------------------------------------------------------- /eslint.config.mjs: -------------------------------------------------------------------------------- 1 | import eslint from '@eslint/js'; 2 | import tseslint from 'typescript-eslint'; 3 | import eslintConfigPrettier from 'eslint-config-prettier'; 4 | 5 | const eslintConfig = [ 6 | eslint.configs.recommended, 7 | ...tseslint.configs.recommended, 8 | 9 | { 10 | rules: { 11 | '@typescript-eslint/no-unused-vars': 'off', 12 | }, 13 | }, 14 | { 15 | ignores: ['node_modules', 'lib'], 16 | }, 17 | 18 | eslintConfigPrettier, 19 | ]; 20 | 21 | export default eslintConfig; 22 | -------------------------------------------------------------------------------- /jestconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "transform": { 3 | "^.+\\.(t|j)sx?$": "ts-jest" 4 | }, 5 | "testRegex": "(/__tests__/.*|(\\.|/)(test|spec))\\.(jsx?|tsx?)$", 6 | "moduleFileExtensions": ["ts", "tsx", "js", "jsx", "json", "node"] 7 | } -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "clickhouse-migrations", 3 | "version": "1.0.4", 4 | "description": "ClickHouse Migrations", 5 | "bin": { 6 | "clickhouse-migrations": "lib/cli.js" 7 | }, 8 | "main": "lib/migrate.js", 9 | "types": "lib/migrate.d.ts", 10 | "engines": { 11 | "node": ">=20" 12 | }, 13 | "private": false, 14 | "license": "MIT", 15 | "author": "VVVi", 16 | "repository": "VVVi/clickhouse-migrations", 17 | "scripts": { 18 | "build": "rm lib/*; tsc", 19 | "test": "jest --config jestconfig.json", 20 | "format": "prettier --write \"src/**/*.ts\" \"src/**/*.js\"", 21 | "lint": "eslint ./src", 22 | "prepare": "npm run build", 23 | "prepublishOnly": "npm test && npm run lint", 24 | "preversion": "npm run lint", 25 | "version": "npm run format && git add -A src", 26 | "postversion": "git push && git push --tags" 27 | }, 28 | "dependencies": { 29 | "@clickhouse/client": "^1.10.1", 30 | "commander": "^13.1.0" 31 | }, 32 | "devDependencies": { 33 | "@eslint/js": "^9.18.0", 34 | "@types/jest": "^29.5.14", 35 | "@types/node": "^22.10.7", 36 | "@typescript-eslint/eslint-plugin": "^8.21.0", 37 | "@typescript-eslint/parser": "^8.21.0", 38 | "eslint": "^9.18.0", 39 | "eslint-config-prettier": "^10.0.1", 40 | "eslint-plugin-prettier": "^5.2.3", 41 | "jest": "^29.7.0", 42 | "prettier": "^3.4.2", 43 | "ts-jest": "^29.2.5", 44 | "typescript": "^5.7.3", 45 | "typescript-eslint": "^8.21.0" 46 | }, 47 | "files": [ 48 | "lib/**/*" 49 | ], 50 | "keywords": [ 51 | "clickhouse-migrations", 52 | "clickhouse-migration", 53 | "clickhouse-migrate", 54 | "clickhouse", 55 | "migrations", 56 | "migration", 57 | "migrate", 58 | "cli" 59 | ] 60 | } 61 | -------------------------------------------------------------------------------- /src/cli.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | import { migrate } from './migrate'; 4 | 5 | migrate(); 6 | 7 | export {}; 8 | -------------------------------------------------------------------------------- /src/migrate.ts: -------------------------------------------------------------------------------- 1 | import { createClient } from '@clickhouse/client'; 2 | import type { ClickHouseClient, ClickHouseClientConfigOptions } from '@clickhouse/client'; 3 | import { Command } from 'commander'; 4 | import fs from 'fs'; 5 | import crypto from 'crypto'; 6 | 7 | import { sql_queries, sql_sets } from './sql-parse'; 8 | 9 | const log = (type: 'info' | 'error' = 'info', message: string, error?: string) => { 10 | if (type === 'info') { 11 | console.log('\x1b[36m', `clickhouse-migrations :`, '\x1b[0m', message); 12 | } else { 13 | console.error('\x1b[36m', `clickhouse-migrations :`, '\x1b[31m', `Error: ${message}`, error ? `\n\n ${error}` : ''); 14 | } 15 | }; 16 | 17 | const connect = ( 18 | url: string, 19 | username: string, 20 | password: string, 21 | db_name?: string, 22 | timeout?: string, 23 | ): ClickHouseClient => { 24 | const db_params: ClickHouseClientConfigOptions = { 25 | url, 26 | username, 27 | password, 28 | application: 'clickhouse-migrations', 29 | }; 30 | 31 | if (db_name) { 32 | db_params.database = db_name; 33 | } 34 | 35 | if (timeout) { 36 | db_params.request_timeout = Number(timeout); 37 | } 38 | 39 | return createClient(db_params); 40 | }; 41 | 42 | const create_db = async ( 43 | host: string, 44 | username: string, 45 | password: string, 46 | db_name: string, 47 | db_engine: string = 'ENGINE=Atomic', 48 | ): Promise => { 49 | const client = connect(host, username, password); 50 | 51 | const q = `CREATE DATABASE IF NOT EXISTS "${db_name}" ${db_engine}`; 52 | 53 | try { 54 | await client.exec({ 55 | query: q, 56 | clickhouse_settings: { 57 | wait_end_of_query: 1, 58 | }, 59 | }); 60 | } catch (e: unknown) { 61 | log('error', `can't create the database ${db_name}.`, (e as QueryError).message); 62 | process.exit(1); 63 | } 64 | 65 | await client.close(); 66 | }; 67 | 68 | const init_migration_table = async (client: ClickHouseClient): Promise => { 69 | const q = `CREATE TABLE IF NOT EXISTS _migrations ( 70 | uid UUID DEFAULT generateUUIDv4(), 71 | version UInt32, 72 | checksum String, 73 | migration_name String, 74 | applied_at DateTime DEFAULT now() 75 | ) 76 | ENGINE = MergeTree 77 | ORDER BY tuple(applied_at)`; 78 | 79 | try { 80 | await client.exec({ 81 | query: q, 82 | clickhouse_settings: { 83 | wait_end_of_query: 1, 84 | }, 85 | }); 86 | } catch (e: unknown) { 87 | log('error', `can't create the _migrations table.`, (e as QueryError).message); 88 | process.exit(1); 89 | } 90 | }; 91 | 92 | const get_migrations = (migrations_home: string): { version: number; file: string }[] => { 93 | let files; 94 | try { 95 | files = fs.readdirSync(migrations_home); 96 | } catch (e: unknown) { 97 | log('error', `no migration directory ${migrations_home}. Please create it.`); 98 | process.exit(1); 99 | } 100 | 101 | const migrations: MigrationBase[] = []; 102 | files.forEach((file: string) => { 103 | const version = Number(file.split('_')[0]); 104 | 105 | if (!version) { 106 | log( 107 | 'error', 108 | `a migration name should start from number, example: 1_init.sql. Please check, if the migration ${file} is named correctly`, 109 | ); 110 | process.exit(1); 111 | } 112 | 113 | // Manage only .sql files. 114 | if (!file.endsWith('.sql')) return; 115 | 116 | migrations.push({ 117 | version, 118 | file, 119 | }); 120 | }); 121 | 122 | if (!migrations) { 123 | log('error', `no migrations in the ${migrations_home} migrations directory`); 124 | } 125 | 126 | // Order by version. 127 | migrations.sort((m1, m2) => m1.version - m2.version); 128 | 129 | return migrations; 130 | }; 131 | 132 | const apply_migrations = async ( 133 | client: ClickHouseClient, 134 | migrations: MigrationBase[], 135 | migrations_home: string, 136 | ): Promise => { 137 | let migration_query_result: MigrationsRowData[] = []; 138 | try { 139 | const resultSet = await client.query({ 140 | query: `SELECT version, checksum, migration_name FROM _migrations ORDER BY version`, 141 | format: 'JSONEachRow', 142 | }); 143 | migration_query_result = await resultSet.json(); 144 | } catch (e: unknown) { 145 | log('error', `can't select data from the _migrations table.`, (e as QueryError).message); 146 | process.exit(1); 147 | } 148 | 149 | const migrations_applied: MigrationsRowData[] = []; 150 | migration_query_result.forEach((row: MigrationsRowData) => { 151 | migrations_applied[row.version] = { 152 | version: row.version, 153 | checksum: row.checksum, 154 | migration_name: row.migration_name, 155 | }; 156 | 157 | // Check if migration file was not removed after apply. 158 | const migration_exist = migrations.find(({ version }) => version === row.version); 159 | if (!migration_exist) { 160 | log( 161 | 'error', 162 | `a migration file shouldn't be removed after apply. Please, restore the migration ${row.migration_name}.`, 163 | ); 164 | process.exit(1); 165 | } 166 | }); 167 | 168 | let applied_migrations = ''; 169 | 170 | for (const migration of migrations) { 171 | const content = fs.readFileSync(migrations_home + '/' + migration.file).toString(); 172 | const checksum = crypto.createHash('md5').update(content).digest('hex'); 173 | 174 | if (migrations_applied[migration.version]) { 175 | // Check if migration file was not changed after apply. 176 | if (migrations_applied[migration.version].checksum !== checksum) { 177 | log( 178 | 'error', 179 | `a migration file should't be changed after apply. Please, restore content of the ${ 180 | migrations_applied[migration.version].migration_name 181 | } migrations.`, 182 | ); 183 | process.exit(1); 184 | } 185 | 186 | // Skip if a migration is already applied. 187 | continue; 188 | } 189 | 190 | // Extract sql from the migration. 191 | const queries = sql_queries(content); 192 | const sets = sql_sets(content); 193 | 194 | for (const query of queries) { 195 | try { 196 | await client.exec({ 197 | query: query, 198 | clickhouse_settings: sets, 199 | }); 200 | } catch (e: unknown) { 201 | if (applied_migrations) { 202 | log('info', `The migration(s) ${applied_migrations} was successfully applied!`); 203 | } 204 | 205 | log( 206 | 'error', 207 | `the migrations ${migration.file} has an error. Please, fix it (be sure that already executed parts of the migration would not be run second time) and re-run migration script.`, 208 | (e as QueryError).message, 209 | ); 210 | process.exit(1); 211 | } 212 | } 213 | 214 | try { 215 | await client.insert({ 216 | table: '_migrations', 217 | values: [{ version: migration.version, checksum: checksum, migration_name: migration.file }], 218 | format: 'JSONEachRow', 219 | }); 220 | } catch (e: unknown) { 221 | log('error', `can't insert a data into the table _migrations.`, (e as QueryError).message); 222 | process.exit(1); 223 | } 224 | 225 | applied_migrations = applied_migrations ? applied_migrations + ', ' + migration.file : migration.file; 226 | } 227 | 228 | if (applied_migrations) { 229 | log('info', `The migration(s) ${applied_migrations} was successfully applied!`); 230 | } else { 231 | log('info', `No migrations to apply.`); 232 | } 233 | }; 234 | 235 | const migration = async ( 236 | migrations_home: string, 237 | host: string, 238 | username: string, 239 | password: string, 240 | db_name: string, 241 | db_engine?: string, 242 | timeout?: string, 243 | ): Promise => { 244 | const migrations = get_migrations(migrations_home); 245 | 246 | await create_db(host, username, password, db_name, db_engine); 247 | 248 | const client = connect(host, username, password, db_name, timeout); 249 | 250 | await init_migration_table(client); 251 | 252 | await apply_migrations(client, migrations, migrations_home); 253 | 254 | await client.close(); 255 | }; 256 | 257 | const migrate = () => { 258 | const program = new Command(); 259 | 260 | program.name('clickhouse-migrations').description('ClickHouse migrations.').version('1.0.4'); 261 | 262 | program 263 | .command('migrate') 264 | .description('Apply migrations.') 265 | .requiredOption('--host ', 'Clickhouse hostname (ex: http://clickhouse:8123)', process.env.CH_MIGRATIONS_HOST) 266 | .requiredOption('--user ', 'Username', process.env.CH_MIGRATIONS_USER) 267 | .requiredOption('--password ', 'Password', process.env.CH_MIGRATIONS_PASSWORD) 268 | .requiredOption('--db ', 'Database name', process.env.CH_MIGRATIONS_DB) 269 | .requiredOption('--migrations-home ', "Migrations' directory", process.env.CH_MIGRATIONS_HOME) 270 | .option( 271 | '--db-engine ', 272 | 'ON CLUSTER and/or ENGINE clauses for database (default: "ENGINE=Atomic")', 273 | process.env.CH_MIGRATIONS_DB_ENGINE, 274 | ) 275 | .option( 276 | '--timeout ', 277 | 'Client request timeout (milliseconds, default value 30000)', 278 | process.env.CH_MIGRATIONS_TIMEOUT, 279 | ) 280 | .action(async (options: CliParameters) => { 281 | await migration( 282 | options.migrationsHome, 283 | options.host, 284 | options.user, 285 | options.password, 286 | options.db, 287 | options.dbEngine, 288 | options.timeout, 289 | ); 290 | }); 291 | 292 | program.parse(); 293 | }; 294 | 295 | export { migrate, migration }; 296 | -------------------------------------------------------------------------------- /src/sql-parse.ts: -------------------------------------------------------------------------------- 1 | // Extract sql queries from migrations. 2 | const sql_queries = (content: string): string[] => { 3 | const queries = content 4 | .replace(/(--|#!|#\s).*(\n|\r\n|\r|$)/gm, '\n') 5 | .replace(/^\s*(SET\s).*(\n|\r\n|\r|$)/gm, '') 6 | .replace(/(\n|\r\n|\r)/gm, ' ') 7 | .replace(/\s+/g, ' ') 8 | .split(';') 9 | .map((el: string) => el.trim()) 10 | .filter((el: string) => el.length != 0); 11 | 12 | return queries; 13 | }; 14 | 15 | // Extract query settings from migrations. 16 | const sql_sets = (content: string) => { 17 | const sets: { [key: string]: string } = {}; 18 | 19 | const sets_arr = content 20 | .replace(/(--|#!|#\s).*(\n|\r\n|\r|$)/gm, '\n') 21 | .replace(/^\s*(?!SET\s).*(\n|\r\n|\r|$)/gm, '') 22 | .replace(/^\s*(SET\s)/gm, '') 23 | .replace(/(\n|\r\n|\r)/gm, ' ') 24 | .replace(/\s+/g, '') 25 | .split(';'); 26 | 27 | sets_arr.forEach((set_full) => { 28 | const set = set_full.split('='); 29 | if (set[0]) { 30 | sets[set[0]] = set[1]; 31 | } 32 | }); 33 | 34 | return sets; 35 | }; 36 | 37 | export { sql_queries, sql_sets }; 38 | -------------------------------------------------------------------------------- /src/types/cli.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | 3 | type MigrationBase = { 4 | version: number; 5 | file: string; 6 | }; 7 | 8 | type MigrationsRowData = { 9 | version: number; 10 | checksum: string; 11 | migration_name: string; 12 | }; 13 | 14 | type CliParameters = { 15 | migrationsHome: string; 16 | host: string; 17 | user: string; 18 | password: string; 19 | db: string; 20 | dbEngine?: string; 21 | timeout?: string; 22 | }; 23 | 24 | type QueryError = { 25 | message: string; 26 | }; 27 | -------------------------------------------------------------------------------- /tests/cli.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect } from '@jest/globals'; 2 | import exec from 'child_process'; 3 | 4 | const execute = async (script: string, execOptions: any) => { 5 | const result: { error: exec.ExecException | null; stdout: string; stderr: string } = await new Promise((resolve) => { 6 | exec.exec(script, execOptions, (error: exec.ExecException | null, stdout: string, stderr: string) => { 7 | resolve({ 8 | error, 9 | stdout, 10 | stderr, 11 | }); 12 | }); 13 | }); 14 | 15 | return result; 16 | }; 17 | 18 | const envVars = { 19 | CH_MIGRATIONS_HOST: 'http://sometesthost:8123', 20 | CH_MIGRATIONS_USER: 'default', 21 | CH_MIGRATIONS_PASSWORD: '', 22 | CH_MIGRATIONS_DB: 'analytics', 23 | CH_MIGRATIONS_HOME: '/app/clickhouse/migrations', 24 | }; 25 | 26 | describe('Execution tests', () => { 27 | beforeEach(() => { 28 | jest.resetModules(); 29 | }); 30 | 31 | it('No parameters provided', async () => { 32 | const result = await execute('node lib/cli.js migrate', '.'); 33 | 34 | expect(result.stderr).toBe("error: required option '--host ' not specified\n"); 35 | }); 36 | 37 | it('No migration directory', async () => { 38 | const command = 39 | "node ./lib/cli.js migrate --host=http://sometesthost:8123 --user=default --password='' --db=analytics --migrations-home=/app/clickhouse/migrations"; 40 | 41 | const result = await execute(command, '.'); 42 | 43 | expect(result.stderr).toBe( 44 | '\x1B[36m clickhouse-migrations : \x1B[31m Error: no migration directory /app/clickhouse/migrations. Please create it. \n', 45 | ); 46 | }); 47 | 48 | it('Environment variables are provided, but no migration directory', async () => { 49 | const result = await execute('node lib/cli.js migrate', { env: envVars }); 50 | 51 | expect(result.stderr).toBe( 52 | '\x1B[36m clickhouse-migrations : \x1B[31m Error: no migration directory /app/clickhouse/migrations. Please create it. \n', 53 | ); 54 | }); 55 | 56 | it('Incorrectly named migration', async () => { 57 | const command = 58 | "node ./lib/cli.js migrate --host=http://sometesthost:8123 --user=default --password='' --db=analytics --migrations-home=tests/migrations/bad"; 59 | 60 | const result = await execute(command, '.'); 61 | 62 | expect(result.stderr).toBe( 63 | '\x1B[36m clickhouse-migrations : \x1B[31m Error: a migration name should start from number, example: 1_init.sql. Please check, if the migration bad_1.sql is named correctly \n', 64 | ); 65 | }); 66 | }); 67 | -------------------------------------------------------------------------------- /tests/db.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect, jest } from '@jest/globals'; 2 | 3 | import { migration } from '../src/migrate'; 4 | 5 | jest.mock('@clickhouse/client', () => ({ createClient: () => createClient1 })); 6 | 7 | const createClient1 = { 8 | query: jest.fn().mockImplementationOnce(() => Promise.resolve({ json: () => [] })), // return 0 rows 9 | exec: jest.fn().mockImplementationOnce(() => Promise.resolve({})), 10 | insert: jest.fn(), 11 | close: jest.fn(), 12 | }; 13 | 14 | describe('Migration tests', () => { 15 | // beforeEach(() => { 16 | // jest.clearAllMocks(); 17 | // jest.resetAllMocks(); 18 | // jest.resetModules(); 19 | // }); 20 | 21 | // todo: remove only 22 | it.only('First migration', async () => { 23 | const querySpy = jest.spyOn(createClient1, 'query'); 24 | const execSpy = jest.spyOn(createClient1, 'exec'); 25 | const insertSpy = jest.spyOn(createClient1, 'insert'); 26 | 27 | await migration('tests/migrations/one', 'http://sometesthost:8123', 'default', '', 'analytics'); 28 | 29 | expect(execSpy).toHaveBeenCalledTimes(3); 30 | expect(querySpy).toHaveBeenCalledTimes(1); 31 | expect(insertSpy).toHaveBeenCalledTimes(1); 32 | 33 | expect(execSpy).toHaveBeenNthCalledWith(1, { 34 | query: 'CREATE DATABASE IF NOT EXISTS "analytics" ENGINE=Atomic', 35 | clickhouse_settings: { 36 | wait_end_of_query: 1, 37 | }, 38 | }); 39 | expect(execSpy).toHaveBeenNthCalledWith(2, { 40 | query: `CREATE TABLE IF NOT EXISTS _migrations ( 41 | uid UUID DEFAULT generateUUIDv4(), 42 | version UInt32, 43 | checksum String, 44 | migration_name String, 45 | applied_at DateTime DEFAULT now() 46 | ) 47 | ENGINE = MergeTree 48 | ORDER BY tuple(applied_at)`, 49 | clickhouse_settings: { 50 | wait_end_of_query: 1, 51 | }, 52 | }); 53 | expect(execSpy).toHaveBeenNthCalledWith(3, { 54 | clickhouse_settings: { allow_experimental_json_type: '1' }, 55 | query: 56 | 'CREATE TABLE IF NOT EXISTS `events` ( `event_id` UInt64, `event_data` JSON ) ENGINE=MergeTree() ORDER BY (`event_id`) SETTINGS index_granularity = 8192', 57 | }); 58 | 59 | expect(querySpy).toHaveBeenNthCalledWith(1, { 60 | format: 'JSONEachRow', 61 | query: 'SELECT version, checksum, migration_name FROM _migrations ORDER BY version', 62 | }); 63 | 64 | expect(insertSpy).toHaveBeenNthCalledWith(1, { 65 | format: 'JSONEachRow', 66 | table: '_migrations', 67 | values: [{ checksum: '2f66edf1a8c3fa2e29835ad9ac8140a7', migration_name: '1_init.sql', version: 1 }], 68 | }); 69 | }); 70 | }); 71 | -------------------------------------------------------------------------------- /tests/migrations/bad/bad_1.sql: -------------------------------------------------------------------------------- 1 | -- bad query without ; and bad migration file name 2 | 3 | SET allow_experimental_json_type = 1; 4 | 5 | CREATE TABLE IF NOT EXISTS `events` ( 6 | `event_id` UInt64, 7 | `event_data` JSON 8 | ) 9 | ENGINE=MergeTree() 10 | ORDER BY (`event_id`) 11 | -------------------------------------------------------------------------------- /tests/migrations/few/1_init.sql: -------------------------------------------------------------------------------- 1 | -- create table events 2 | 3 | SET allow_experimental_json_type = 1; 4 | 5 | CREATE TABLE IF NOT EXISTS `events` ( 6 | `event_id` UInt64, 7 | `event_data` JSON 8 | ) 9 | ENGINE=MergeTree() 10 | ORDER BY (`event_id`); 11 | -------------------------------------------------------------------------------- /tests/migrations/few/2_more.sql: -------------------------------------------------------------------------------- 1 | -- create table sessions 2 | 3 | CREATE TABLE IF NOT EXISTS `sessions` ( 4 | `session_id` UInt64 5 | ) 6 | ENGINE=MergeTree() 7 | ORDER BY (`session_id`); 8 | -------------------------------------------------------------------------------- /tests/migrations/one/1_init.sql: -------------------------------------------------------------------------------- 1 | -- create table events 2 | 3 | SET allow_experimental_json_type = 1; 4 | 5 | CREATE TABLE IF NOT EXISTS `events` ( 6 | `event_id` UInt64, 7 | `event_data` JSON 8 | ) 9 | ENGINE=MergeTree() 10 | ORDER BY (`event_id`) 11 | SETTINGS index_granularity = 8192; 12 | -------------------------------------------------------------------------------- /tests/unit.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect } from '@jest/globals'; 2 | 3 | import { sql_queries, sql_sets } from '../src/sql-parse'; 4 | 5 | describe('Sql query parse', () => { 6 | beforeEach(() => { 7 | jest.resetModules(); 8 | }); 9 | 10 | it('1 query test', async () => { 11 | const input = '-- any\n\n# other comment\n\n#! also comment\n SELECT * \nFROM events;\n'; 12 | 13 | const output = ['SELECT * FROM events']; 14 | 15 | expect(sql_queries(input)).toEqual(output); 16 | }); 17 | }); 18 | 19 | describe('Sql settings parse', () => { 20 | beforeEach(() => { 21 | jest.resetModules(); 22 | }); 23 | 24 | it('one set and comments with no end of lines', async () => { 25 | const input = '-- any\nSET allow_experimental_json_type = 1;\n\n --set option\nSELECT * FROM events'; 26 | 27 | const output = { allow_experimental_json_type: '1' }; 28 | 29 | expect(sql_sets(input)).toEqual(output); 30 | }); 31 | 32 | it('two sets and comments', async () => { 33 | const input = 34 | '-- any\nSET allow_experimental_json_type = 1; --set option\nSET allow_experimental_object_new = 1;\nSELECT * \n --comment\n FROM events\n'; 35 | 36 | const output = { allow_experimental_json_type: '1', allow_experimental_object_new: '1' }; 37 | 38 | expect(sql_sets(input)).toEqual(output); 39 | }); 40 | }); 41 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es5", 4 | "module": "commonjs", 5 | 6 | "strict": true, 7 | "esModuleInterop": true, 8 | "skipLibCheck": true, 9 | "forceConsistentCasingInFileNames": true, 10 | 11 | "declaration": true, 12 | "outDir": "./lib" 13 | }, 14 | "include": ["src", "types"], 15 | "exclude": ["node_modules", "tests"] 16 | } --------------------------------------------------------------------------------