├── .github ├── dependabot.yml └── workflows │ └── tests.yml ├── LICENCE ├── License ├── README.md ├── clickhouse.go ├── create.go ├── create_test.go ├── delete_test.go ├── docker-compose.yml ├── go.mod ├── go.sum ├── migrator.go ├── migrator_test.go ├── tests_test.go ├── update.go └── update_test.go /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "github-actions" 9 | directory: "/" 10 | schedule: 11 | interval: "daily" 12 | - package-ecosystem: "gomod" 13 | directory: "/" 14 | schedule: 15 | interval: "daily" 16 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: tests 2 | 3 | on: 4 | push: 5 | branches-ignore: 6 | - 'gh-pages' 7 | pull_request: 8 | branches-ignore: 9 | - 'gh-pages' 10 | 11 | jobs: 12 | # Label of the container job 13 | clickhouse: 14 | strategy: 15 | matrix: 16 | dbversion: ['clickhouse/clickhouse-server'] 17 | go: ['1.21', '1.22'] 18 | platform: [ubuntu-latest] 19 | runs-on: ${{ matrix.platform }} 20 | 21 | services: 22 | clickhouse: 23 | image: ${{ matrix.dbversion }} 24 | env: 25 | CLICKHOUSE_DB: gorm 26 | CLICKHOUSE_USER: gorm 27 | CLICKHOUSE_PASSWORD: gorm 28 | ports: 29 | - 9941:8123 30 | - 9942:9000 31 | - 9943:9009 32 | 33 | steps: 34 | - name: Set up Go 1.x 35 | uses: actions/setup-go@v4 36 | with: 37 | go-version: ${{ matrix.go }} 38 | 39 | - name: Check out code into the Go module directory 40 | uses: actions/checkout@v3 41 | 42 | - name: Tests 43 | run: go test 44 | 45 | -------------------------------------------------------------------------------- /LICENCE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2020-NOW Daniel 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /License: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2013-NOW Jinzhu 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GORM ClickHouse Driver 2 | 3 | Clickhouse support for GORM 4 | 5 | [![test status](https://github.com/go-gorm/clickhouse/workflows/tests/badge.svg?branch=master "test status")](https://github.com/go-gorm/clickhouse/actions) 6 | 7 | ## Quick Start 8 | 9 | You can simply test your connection to your database with the following: 10 | 11 | ```go 12 | package main 13 | 14 | import ( 15 | "gorm.io/driver/clickhouse" 16 | "gorm.io/gorm" 17 | ) 18 | 19 | type User struct { 20 | Name string 21 | Age int 22 | } 23 | 24 | func main() { 25 | dsn := "clickhouse://gorm:gorm@localhost:9942/gorm?dial_timeout=10s&read_timeout=20s" 26 | db, err := gorm.Open(clickhouse.Open(dsn), &gorm.Config{}) 27 | if err != nil { 28 | panic("failed to connect database") 29 | } 30 | 31 | // Auto Migrate 32 | db.AutoMigrate(&User{}) 33 | // Set table options 34 | db.Set("gorm:table_options", "ENGINE=Distributed(cluster, default, hits)").AutoMigrate(&User{}) 35 | 36 | // Set table cluster options 37 | db.Set("gorm:table_cluster_options", "on cluster default").AutoMigrate(&User{}) 38 | 39 | // Insert 40 | db.Create(&User{Name: "Angeliz", Age: 18}) 41 | 42 | // Select 43 | db.Find(&User{}, "name = ?", "Angeliz") 44 | 45 | // Batch Insert 46 | user1 := User{Age: 12, Name: "Bruce Lee"} 47 | user2 := User{Age: 13, Name: "Feynman"} 48 | user3 := User{Age: 14, Name: "Angeliz"} 49 | var users = []User{user1, user2, user3} 50 | db.Create(&users) 51 | // ... 52 | } 53 | 54 | ``` 55 | 56 | ## Advanced Configuration 57 | 58 | ```go 59 | package main 60 | 61 | import ( 62 | std_ck "github.com/ClickHouse/clickhouse-go/v2" 63 | "gorm.io/driver/clickhouse" 64 | "gorm.io/gorm" 65 | ) 66 | 67 | sqlDB, err := std_ck.OpenDB(&std_ck.Options{ 68 | Addr: []string{"127.0.0.1:9999"}, 69 | Auth: std_ck.Auth{ 70 | Database: "default", 71 | Username: "default", 72 | Password: "", 73 | }, 74 | TLS: &tls.Config{ 75 | InsecureSkipVerify: true, 76 | }, 77 | Settings: std_ck.Settings{ 78 | "max_execution_time": 60, 79 | }, 80 | DialTimeout: 5 * time.Second, 81 | Compression: &std_ck.Compression{ 82 | std_ck.CompressionLZ4, 83 | }, 84 | Debug: true, 85 | }) 86 | 87 | func main() { 88 | db, err := gorm.Open(clickhouse.New(click.Config{ 89 | Conn: sqlDB, // initialize with existing database conn 90 | }) 91 | } 92 | ``` 93 | 94 | ```go 95 | package main 96 | 97 | import ( 98 | "gorm.io/driver/clickhouse" 99 | "gorm.io/gorm" 100 | ) 101 | 102 | // refer to https://github.com/ClickHouse/clickhouse-go 103 | var dsn = "clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms&max_execution_time=60" 104 | 105 | func main() { 106 | db, err := gorm.Open(clickhouse.New(click.Config{ 107 | DSN: dsn, 108 | Conn: conn, // initialize with existing database conn 109 | DisableDatetimePrecision: true, // disable datetime64 precision, not supported before clickhouse 20.4 110 | DontSupportRenameColumn: true, // rename column not supported before clickhouse 20.4 111 | DontSupportEmptyDefaultValue: false, // do not consider empty strings as valid default values 112 | SkipInitializeWithVersion: false, // smart configure based on used version 113 | DefaultGranularity: 3, // 1 granule = 8192 rows 114 | DefaultCompression: "LZ4", // default compression algorithm. LZ4 is lossless 115 | DefaultIndexType: "minmax", // index stores extremes of the expression 116 | DefaultTableEngineOpts: "ENGINE=MergeTree() ORDER BY tuple()", 117 | }), &gorm.Config{}) 118 | } 119 | ``` 120 | 121 | Checkout [https://gorm.io](https://gorm.io) for details. 122 | -------------------------------------------------------------------------------- /clickhouse.go: -------------------------------------------------------------------------------- 1 | package clickhouse 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "fmt" 7 | "reflect" 8 | "strings" 9 | 10 | "github.com/ClickHouse/clickhouse-go/v2" 11 | "github.com/hashicorp/go-version" 12 | "gorm.io/gorm" 13 | "gorm.io/gorm/callbacks" 14 | "gorm.io/gorm/clause" 15 | "gorm.io/gorm/logger" 16 | "gorm.io/gorm/migrator" 17 | "gorm.io/gorm/schema" 18 | ) 19 | 20 | type Config struct { 21 | DriverName string 22 | DSN string 23 | Conn gorm.ConnPool 24 | DisableDatetimePrecision bool 25 | DontSupportRenameColumn bool 26 | DontSupportColumnPrecision bool 27 | DontSupportEmptyDefaultValue bool 28 | SkipInitializeWithVersion bool 29 | DefaultGranularity int // 1 granule = 8192 rows 30 | DefaultCompression string // default compression algorithm. LZ4 is lossless 31 | DefaultIndexType string // index stores extremes of the expression 32 | DefaultTableEngineOpts string 33 | } 34 | 35 | type Dialector struct { 36 | *Config 37 | options clickhouse.Options 38 | Version string 39 | } 40 | 41 | func Open(dsn string) gorm.Dialector { 42 | return &Dialector{Config: &Config{DSN: dsn}} 43 | } 44 | 45 | func New(config Config) gorm.Dialector { 46 | return &Dialector{Config: &config} 47 | } 48 | 49 | func (dialector Dialector) Name() string { 50 | return "clickhouse" 51 | } 52 | 53 | func (dialector *Dialector) Initialize(db *gorm.DB) (err error) { 54 | // register callbacks 55 | ctx := context.Background() 56 | callbacks.RegisterDefaultCallbacks(db, &callbacks.Config{ 57 | DeleteClauses: []string{"DELETE", "WHERE"}, 58 | }) 59 | db.Callback().Create().Replace("gorm:create", dialector.Create) 60 | db.Callback().Update().Replace("gorm:update", dialector.Update) 61 | 62 | // assign option fields to default values 63 | if dialector.DriverName == "" { 64 | dialector.DriverName = "clickhouse" 65 | } 66 | 67 | // default settings 68 | if dialector.Config.DefaultGranularity == 0 { 69 | dialector.Config.DefaultGranularity = 3 70 | } 71 | 72 | if dialector.Config.DefaultCompression == "" { 73 | dialector.Config.DefaultCompression = "LZ4" 74 | } 75 | 76 | if dialector.DefaultIndexType == "" { 77 | dialector.DefaultIndexType = "minmax" 78 | } 79 | 80 | if dialector.DefaultTableEngineOpts == "" { 81 | dialector.DefaultTableEngineOpts = "ENGINE=MergeTree() ORDER BY tuple()" 82 | } 83 | 84 | if dialector.Conn != nil { 85 | db.ConnPool = dialector.Conn 86 | } else { 87 | db.ConnPool, err = sql.Open(dialector.DriverName, dialector.DSN) 88 | if err != nil { 89 | return err 90 | } 91 | } 92 | 93 | if dialector.DSN != "" { 94 | if opts, err := clickhouse.ParseDSN(dialector.DSN); err == nil { 95 | dialector.options = *opts 96 | } 97 | } 98 | 99 | if !dialector.SkipInitializeWithVersion { 100 | err = db.ConnPool.QueryRowContext(ctx, "SELECT version()").Scan(&dialector.Version) 101 | if err != nil { 102 | return err 103 | } 104 | if dbversion, err := version.NewVersion(dialector.Version); err == nil { 105 | versionNoRenameColumn, _ := version.NewConstraint("< 20.4") 106 | 107 | if versionNoRenameColumn.Check(dbversion) { 108 | dialector.Config.DontSupportRenameColumn = true 109 | } 110 | 111 | versionNoPrecisionColumn, _ := version.NewConstraint("< 21.11") 112 | if versionNoPrecisionColumn.Check(dbversion) { 113 | dialector.DontSupportColumnPrecision = true 114 | } 115 | } 116 | } 117 | 118 | for k, v := range dialector.ClauseBuilders() { 119 | db.ClauseBuilders[k] = v 120 | } 121 | return 122 | } 123 | 124 | func modifyStatementWhereConds(stmt *gorm.Statement) { 125 | if c, ok := stmt.Clauses["WHERE"]; ok { 126 | if where, ok := c.Expression.(clause.Where); ok { 127 | modifyExprs(where.Exprs) 128 | } 129 | } 130 | } 131 | 132 | func modifyExprs(exprs []clause.Expression) { 133 | for idx, expr := range exprs { 134 | switch v := expr.(type) { 135 | case clause.AndConditions: 136 | modifyExprs(v.Exprs) 137 | case clause.NotConditions: 138 | modifyExprs(v.Exprs) 139 | case clause.OrConditions: 140 | modifyExprs(v.Exprs) 141 | default: 142 | reflectValue := reflect.ValueOf(expr) 143 | if reflectValue.Kind() == reflect.Struct { 144 | if field := reflectValue.FieldByName("Column"); field.IsValid() && !field.IsZero() { 145 | if column, ok := field.Interface().(clause.Column); ok { 146 | column.Table = "" 147 | result := reflect.New(reflectValue.Type()).Elem() 148 | result.Set(reflectValue) 149 | result.FieldByName("Column").Set(reflect.ValueOf(column)) 150 | exprs[idx] = result.Interface().(clause.Expression) 151 | } 152 | } 153 | } 154 | } 155 | } 156 | } 157 | 158 | func (dialector Dialector) ClauseBuilders() map[string]clause.ClauseBuilder { 159 | clauseBuilders := map[string]clause.ClauseBuilder{ 160 | "DELETE": func(c clause.Clause, builder clause.Builder) { 161 | builder.WriteString("ALTER TABLE ") 162 | 163 | var addedTable bool 164 | if stmt, ok := builder.(*gorm.Statement); ok { 165 | if c, ok := stmt.Clauses["FROM"]; ok { 166 | addedTable = true 167 | c.Name = "" 168 | c.Build(builder) 169 | } 170 | modifyStatementWhereConds(stmt) 171 | } 172 | 173 | if !addedTable { 174 | builder.WriteQuoted(clause.Table{Name: clause.CurrentTable}) 175 | } 176 | builder.WriteString(" DELETE") 177 | }, 178 | "UPDATE": func(c clause.Clause, builder clause.Builder) { 179 | builder.WriteString("ALTER TABLE ") 180 | 181 | var addedTable bool 182 | if stmt, ok := builder.(*gorm.Statement); ok { 183 | if c, ok := stmt.Clauses["FROM"]; ok { 184 | addedTable = true 185 | c.Name = "" 186 | c.Build(builder) 187 | } 188 | modifyStatementWhereConds(stmt) 189 | } 190 | 191 | if !addedTable { 192 | builder.WriteQuoted(clause.Table{Name: clause.CurrentTable}) 193 | } 194 | builder.WriteString(" UPDATE") 195 | }, 196 | "SET": func(c clause.Clause, builder clause.Builder) { 197 | c.Name = "" 198 | c.Build(builder) 199 | }, 200 | } 201 | 202 | return clauseBuilders 203 | } 204 | 205 | func (dialector Dialector) Migrator(db *gorm.DB) gorm.Migrator { 206 | return Migrator{ 207 | Migrator: migrator.Migrator{ 208 | Config: migrator.Config{ 209 | DB: db, 210 | Dialector: &dialector, 211 | }, 212 | }, 213 | Dialector: dialector, 214 | } 215 | } 216 | 217 | func (dialector Dialector) DataTypeOf(field *schema.Field) string { 218 | switch field.DataType { 219 | case schema.Bool: 220 | return "UInt8" 221 | case schema.Int, schema.Uint: 222 | sqlType := "Int64" 223 | switch { 224 | case field.Size <= 8: 225 | sqlType = "Int8" 226 | case field.Size <= 16: 227 | sqlType = "Int16" 228 | case field.Size <= 32: 229 | sqlType = "Int32" 230 | } 231 | if field.DataType == schema.Uint { 232 | sqlType = "U" + sqlType 233 | } 234 | return sqlType 235 | case schema.Float: 236 | if field.Precision > 0 { 237 | return fmt.Sprintf("decimal(%d, %d)", field.Precision, field.Scale) 238 | } 239 | if field.Size <= 32 { 240 | return "Float32" 241 | } 242 | return "Float64" 243 | case schema.String: 244 | if field.Size == 0 { 245 | return "String" 246 | } 247 | return fmt.Sprintf("FixedString(%d)", field.Size) 248 | case schema.Bytes: 249 | return "String" 250 | case schema.Time: 251 | // TODO: support TimeZone 252 | precision := "" 253 | if !dialector.DisableDatetimePrecision { 254 | if field.Precision == 0 { 255 | field.Precision = 3 256 | } 257 | if field.Precision > 0 { 258 | precision = fmt.Sprintf("(%d)", field.Precision) 259 | } 260 | } 261 | return "DateTime64" + precision 262 | } 263 | 264 | return string(field.DataType) 265 | } 266 | 267 | func (dialector Dialector) DefaultValueOf(field *schema.Field) clause.Expression { 268 | return clause.Expr{SQL: "DEFAULT"} 269 | } 270 | 271 | func (dialector Dialector) BindVarTo(writer clause.Writer, stmt *gorm.Statement, v interface{}) { 272 | writer.WriteByte('?') 273 | } 274 | 275 | func (dialector Dialector) QuoteTo(writer clause.Writer, str string) { 276 | writer.WriteByte('`') 277 | if strings.Contains(str, ".") { 278 | for idx, str := range strings.Split(str, ".") { 279 | if idx > 0 { 280 | writer.WriteString(".`") 281 | } 282 | writer.WriteString(str) 283 | writer.WriteByte('`') 284 | } 285 | } else { 286 | writer.WriteString(str) 287 | writer.WriteByte('`') 288 | } 289 | } 290 | 291 | func (dialector Dialector) Explain(sql string, vars ...interface{}) string { 292 | return logger.ExplainSQL(sql, nil, `'`, vars...) 293 | } 294 | 295 | func (dialectopr Dialector) SavePoint(tx *gorm.DB, name string) error { 296 | return gorm.ErrUnsupportedDriver 297 | } 298 | 299 | func (dialectopr Dialector) RollbackTo(tx *gorm.DB, name string) error { 300 | return gorm.ErrUnsupportedDriver 301 | } 302 | -------------------------------------------------------------------------------- /create.go: -------------------------------------------------------------------------------- 1 | package clickhouse 2 | 3 | import ( 4 | "gorm.io/gorm" 5 | "gorm.io/gorm/callbacks" 6 | "gorm.io/gorm/clause" 7 | ) 8 | 9 | func (dialector *Dialector) Create(db *gorm.DB) { 10 | if db.Error == nil { 11 | if db.Statement.Schema != nil && !db.Statement.Unscoped { 12 | for _, c := range db.Statement.Schema.CreateClauses { 13 | db.Statement.AddClause(c) 14 | } 15 | } 16 | 17 | if db.Statement.SQL.String() == "" { 18 | db.Statement.SQL.Grow(180) 19 | db.Statement.AddClauseIfNotExists(clause.Insert{}) 20 | 21 | if values := callbacks.ConvertToCreateValues(db.Statement); len(values.Values) >= 1 { 22 | prepareValues := clause.Values{ 23 | Columns: values.Columns, 24 | Values: [][]interface{}{values.Values[0]}, 25 | } 26 | db.Statement.AddClause(prepareValues) 27 | db.Statement.Build("INSERT", "VALUES", "ON CONFLICT") 28 | 29 | stmt, err := db.Statement.ConnPool.PrepareContext(db.Statement.Context, db.Statement.SQL.String()) 30 | if db.AddError(err) != nil { 31 | return 32 | } 33 | defer stmt.Close() 34 | 35 | for _, value := range values.Values { 36 | if _, err := stmt.Exec(value...); db.AddError(err) != nil { 37 | return 38 | } 39 | } 40 | return 41 | } 42 | } 43 | 44 | if !db.DryRun && db.Error == nil { 45 | result, err := db.Statement.ConnPool.ExecContext(db.Statement.Context, db.Statement.SQL.String(), db.Statement.Vars...) 46 | 47 | if db.Statement.Result != nil { 48 | db.Statement.Result.Result = result 49 | } 50 | db.AddError(err) 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /create_test.go: -------------------------------------------------------------------------------- 1 | package clickhouse_test 2 | 3 | import ( 4 | "slices" 5 | "testing" 6 | 7 | "gorm.io/gorm/utils/tests" 8 | ) 9 | 10 | func TestCreate(t *testing.T) { 11 | user := User{ID: 1, Name: "create", FirstName: "zhang", LastName: "jinzhu", Age: 18, Active: true, Salary: 8.8888, Attrs: map[string]string{ 12 | "a": "a", 13 | "b": "b", 14 | }} 15 | 16 | if err := DB.Create(&user).Error; err != nil { 17 | t.Fatalf("failed to create user, got error %v", err) 18 | } 19 | 20 | var result User 21 | if err := DB.Find(&result, user.ID).Error; err != nil { 22 | t.Fatalf("failed to query user, got error %v", err) 23 | } 24 | 25 | tests.AssertEqual(t, result, user) 26 | 27 | type partialUser struct { 28 | Name string 29 | } 30 | var partialResult partialUser 31 | if err := DB.Raw("select * from users where id = ?", user.ID).Scan(&partialResult).Error; err != nil { 32 | t.Fatalf("failed to query partial, got error %v", err) 33 | } 34 | 35 | var names []string 36 | if err := DB.Select("name").Model(&User{}).Find(&names).Error; err != nil { 37 | t.Fatalf("failed to query user, got error %v", err) 38 | } 39 | 40 | if !slices.Contains(names, user.Name) { 41 | t.Errorf("name should be included in the result") 42 | } 43 | } 44 | 45 | func TestBatchCreate(t *testing.T) { 46 | users := []User{ 47 | {ID: 11, Name: "batch_create_1", FirstName: "zhang", LastName: "jinzhu", Age: 18, Active: true, Salary: 6}, 48 | {ID: 12, Name: "batch_create_2", FirstName: "zhang", LastName: "jinzhu", Age: 18, Active: false, Salary: 6.12}, 49 | {ID: 13, Name: "batch_create_3", FirstName: "zhang", LastName: "jinzhu", Age: 18, Active: true, Salary: 6.1234}, 50 | {ID: 14, Name: "batch_create_4", FirstName: "zhang", LastName: "jinzhu", Age: 18, Active: false, Salary: 6.123456}, 51 | } 52 | 53 | if err := DB.Create(&users).Error; err != nil { 54 | t.Fatalf("failed to create users, got error %v", err) 55 | } 56 | 57 | var results []User 58 | DB.Find(&results) 59 | 60 | for _, u := range users { 61 | var result User 62 | if err := DB.Find(&result, u.ID).Error; err != nil { 63 | t.Fatalf("failed to query user, got error %v", err) 64 | } 65 | 66 | tests.AssertEqual(t, result, u) 67 | } 68 | } 69 | 70 | func TestCreateWithMap(t *testing.T) { 71 | user := User{ID: 122, Name: "create2", FirstName: "zhang", LastName: "jinzhu", Age: 18, Active: true, Salary: 6.6666} 72 | 73 | if err := DB.Table("users").Create(&map[string]interface{}{ 74 | "id": user.ID, "name": user.Name, "first_name": user.FirstName, "last_name": user.LastName, "age": user.Age, "active": user.Active, "salary": user.Salary, 75 | }).Error; err != nil { 76 | t.Fatalf("failed to create user, got error %v", err) 77 | } 78 | 79 | var result User 80 | if err := DB.Find(&result, user.ID).Error; err != nil { 81 | t.Fatalf("failed to query user, got error %v", err) 82 | } 83 | 84 | user.CreatedAt = result.CreatedAt 85 | user.UpdatedAt = result.UpdatedAt 86 | tests.AssertEqual(t, result, user) 87 | } 88 | -------------------------------------------------------------------------------- /delete_test.go: -------------------------------------------------------------------------------- 1 | package clickhouse_test 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "gorm.io/gorm/utils/tests" 8 | ) 9 | 10 | func TestDelete(t *testing.T) { 11 | user := User{ID: 2, Name: "delete", FirstName: "zhang", LastName: "jinzhu", Age: 18, Active: true, Salary: 8.8888} 12 | 13 | if err := DB.Create(&user).Error; err != nil { 14 | t.Fatalf("failed to create user, got error %v", err) 15 | } 16 | 17 | var result User 18 | if err := DB.Find(&result, user.ID).Error; err != nil { 19 | t.Fatalf("failed to query user, got error %v", err) 20 | } 21 | 22 | tests.AssertEqual(t, result, user) 23 | 24 | if err := DB.Delete(&result).Error; err != nil { 25 | t.Fatalf("failed to delete user, got error %v", err) 26 | } 27 | 28 | time.Sleep(500 * time.Millisecond) 29 | if err := DB.First(&result, user.ID).Error; err == nil { 30 | t.Fatalf("should raise ErrRecordNotFound, got error %v", err) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | server: 4 | image: yandex/clickhouse-server:latest 5 | environment: 6 | - CLICKHOUSE_DB=gorm 7 | - CLICKHOUSE_USER=gorm 8 | - CLICKHOUSE_PASSWORD=gorm 9 | ports: 10 | - 9941:8123 11 | - 9942:9000 12 | - 9943:9009 13 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module gorm.io/driver/clickhouse 2 | 3 | go 1.21 4 | 5 | require ( 6 | github.com/ClickHouse/clickhouse-go/v2 v2.30.0 7 | github.com/hashicorp/go-version v1.6.0 8 | gorm.io/gorm v1.30.0 9 | ) 10 | 11 | require golang.org/x/text v0.20.0 // indirect 12 | 13 | require ( 14 | github.com/ClickHouse/ch-go v0.61.5 // indirect 15 | github.com/andybalholm/brotli v1.1.1 // indirect 16 | github.com/go-faster/city v1.0.1 // indirect 17 | github.com/go-faster/errors v0.7.1 // indirect 18 | github.com/google/uuid v1.6.0 // indirect 19 | github.com/jinzhu/inflection v1.0.0 // indirect 20 | github.com/jinzhu/now v1.1.5 // indirect 21 | github.com/klauspost/compress v1.17.8 // indirect 22 | github.com/kr/text v0.2.0 // indirect 23 | github.com/paulmach/orb v0.11.1 // indirect 24 | github.com/pierrec/lz4/v4 v4.1.21 // indirect 25 | github.com/pkg/errors v0.9.1 // indirect 26 | github.com/segmentio/asm v1.2.0 // indirect 27 | github.com/shopspring/decimal v1.4.0 // indirect 28 | go.opentelemetry.io/otel v1.26.0 // indirect 29 | go.opentelemetry.io/otel/trace v1.26.0 // indirect 30 | golang.org/x/sys v0.26.0 // indirect 31 | gopkg.in/yaml.v3 v3.0.1 // indirect 32 | ) 33 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/ClickHouse/ch-go v0.61.5 h1:zwR8QbYI0tsMiEcze/uIMK+Tz1D3XZXLdNrlaOpeEI4= 2 | github.com/ClickHouse/ch-go v0.61.5/go.mod h1:s1LJW/F/LcFs5HJnuogFMta50kKDO0lf9zzfrbl0RQg= 3 | github.com/ClickHouse/clickhouse-go/v2 v2.30.0 h1:AG4D/hW39qa58+JHQIFOSnxyL46H6h2lrmGGk17dhFo= 4 | github.com/ClickHouse/clickhouse-go/v2 v2.30.0/go.mod h1:i9ZQAojcayW3RsdCb3YR+n+wC2h65eJsZCscZ1Z1wyo= 5 | github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= 6 | github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= 7 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 8 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 9 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 10 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 11 | github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= 12 | github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw= 13 | github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg= 14 | github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo= 15 | github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= 16 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 17 | github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 18 | github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 19 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 20 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 21 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 22 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 23 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 24 | github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= 25 | github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= 26 | github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= 27 | github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= 28 | github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= 29 | github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= 30 | github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= 31 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 32 | github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= 33 | github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= 34 | github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= 35 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 36 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 37 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 38 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 39 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 40 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 41 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 42 | github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= 43 | github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU= 44 | github.com/paulmach/orb v0.11.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= 45 | github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= 46 | github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= 47 | github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= 48 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 49 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 50 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 51 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 52 | github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= 53 | github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= 54 | github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= 55 | github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= 56 | github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= 57 | github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= 58 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 59 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 60 | github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= 61 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 62 | github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= 63 | github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= 64 | github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= 65 | github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= 66 | github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= 67 | github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= 68 | github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= 69 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 70 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 71 | go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= 72 | go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs= 73 | go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4= 74 | go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA= 75 | go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0= 76 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 77 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 78 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 79 | golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= 80 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 81 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 82 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 83 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 84 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 85 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 86 | golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 87 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 88 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 89 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 90 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 91 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 92 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 93 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 94 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 95 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 96 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 97 | golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= 98 | golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 99 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 100 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 101 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 102 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 103 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 104 | golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= 105 | golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= 106 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 107 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 108 | golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 109 | golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 110 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 111 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 112 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 113 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 114 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 115 | google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 116 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 117 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 118 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 119 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 120 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 121 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 122 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 123 | gorm.io/gorm v1.30.0 h1:qbT5aPv1UH8gI99OsRlvDToLxW5zR7FzS9acZDOZcgs= 124 | gorm.io/gorm v1.30.0/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE= 125 | -------------------------------------------------------------------------------- /migrator.go: -------------------------------------------------------------------------------- 1 | package clickhouse 2 | 3 | import ( 4 | "bufio" 5 | "database/sql" 6 | "errors" 7 | "fmt" 8 | "strconv" 9 | "strings" 10 | 11 | "gorm.io/gorm" 12 | "gorm.io/gorm/clause" 13 | "gorm.io/gorm/migrator" 14 | "gorm.io/gorm/schema" 15 | ) 16 | 17 | // Errors enumeration 18 | var ( 19 | ErrRenameColumnUnsupported = errors.New("renaming column is not supported in your clickhouse version < 20.4") 20 | ErrRenameIndexUnsupported = errors.New("renaming index is not supported") 21 | ErrCreateIndexFailed = errors.New("failed to create index with name") 22 | ) 23 | 24 | type Migrator struct { 25 | migrator.Migrator 26 | Dialector 27 | } 28 | 29 | // Database 30 | 31 | func (m Migrator) CurrentDatabase() (name string) { 32 | m.DB.Raw("SELECT currentDatabase()").Row().Scan(&name) 33 | return 34 | } 35 | 36 | func (m Migrator) FullDataTypeOf(field *schema.Field) (expr clause.Expr) { 37 | // Infer the ClickHouse datatype from schema.Field information 38 | expr.SQL = m.Migrator.DataTypeOf(field) 39 | 40 | // NOTE: 41 | // NULL and UNIQUE keyword is not supported in clickhouse. 42 | // Hence, skipping checks for field.Unique and field.NotNull 43 | 44 | // Build DEFAULT clause after DataTypeOf() expression optionally 45 | if field.HasDefaultValue && (field.DefaultValueInterface != nil || field.DefaultValue != "") { 46 | if field.DefaultValueInterface != nil { 47 | defaultStmt := &gorm.Statement{Vars: []interface{}{field.DefaultValueInterface}} 48 | m.Dialector.BindVarTo(defaultStmt, defaultStmt, field.DefaultValueInterface) 49 | expr.SQL += " DEFAULT " + m.Dialector.Explain(defaultStmt.SQL.String(), field.DefaultValueInterface) 50 | } else if field.DefaultValue != "(-)" { 51 | expr.SQL += " DEFAULT " + field.DefaultValue 52 | } 53 | } 54 | 55 | // Build COMMENT clause optionally after DEFAULT 56 | if comment, ok := field.TagSettings["COMMENT"]; ok { 57 | expr.SQL += " COMMENT " + m.Dialector.Explain("?", comment) 58 | } 59 | 60 | // Build TTl clause optionally after COMMENT 61 | if ttl, ok := field.TagSettings["TTL"]; ok && ttl != "" { 62 | expr.SQL += " TTL " + ttl 63 | } 64 | 65 | // Build CODEC compression algorithm optionally 66 | // NOTE: the codec algo name is case sensitive! 67 | if codecstr, ok := field.TagSettings["CODEC"]; ok && codecstr != "" { 68 | // parse codec one by one in the codec option 69 | codecSlice := strings.Split(codecstr, ",") 70 | codecArgsSQL := m.Dialector.DefaultCompression 71 | if len(codecSlice) > 0 { 72 | codecArgsSQL = strings.Join(codecSlice, ",") 73 | } 74 | codecSQL := fmt.Sprintf(" CODEC(%s) ", codecArgsSQL) 75 | expr.SQL += codecSQL 76 | } 77 | 78 | return expr 79 | } 80 | 81 | // Tables 82 | 83 | func (m Migrator) CreateTable(models ...interface{}) error { 84 | for _, model := range m.ReorderModels(models, false) { 85 | tx := m.DB.Session(new(gorm.Session)) 86 | if err := m.RunWithValue(model, func(stmt *gorm.Statement) (err error) { 87 | var ( 88 | createTableSQL = "CREATE TABLE ?%s(%s %s %s) %s" 89 | args = []interface{}{clause.Table{Name: stmt.Table}} 90 | ) 91 | 92 | // Step 1. Build column datatype SQL string 93 | columnSlice := make([]string, 0, len(stmt.Schema.DBNames)) 94 | for _, dbName := range stmt.Schema.DBNames { 95 | field := stmt.Schema.FieldsByDBName[dbName] 96 | columnSlice = append(columnSlice, "? ?") 97 | args = append(args, 98 | clause.Column{Name: dbName}, 99 | m.FullDataTypeOf(field), 100 | ) 101 | } 102 | columnStr := strings.Join(columnSlice, ",") 103 | 104 | // Step 2. Build constraint check SQL string if any constraint 105 | constrSlice := make([]string, 0, len(columnSlice)) 106 | for _, check := range stmt.Schema.ParseCheckConstraints() { 107 | constrSlice = append(constrSlice, "CONSTRAINT ? CHECK ?") 108 | args = append(args, 109 | clause.Column{Name: check.Name}, 110 | clause.Expr{SQL: check.Constraint}, 111 | ) 112 | } 113 | constrStr := strings.Join(constrSlice, ",") 114 | if len(constrSlice) > 0 { 115 | constrStr = ", " + constrStr 116 | } 117 | 118 | // Step 3. Build index SQL string 119 | // NOTE: clickhouse does not support for index class. 120 | indexSlice := make([]string, 0, 10) 121 | for _, index := range stmt.Schema.ParseIndexes() { 122 | if m.CreateIndexAfterCreateTable { 123 | defer func(model interface{}, indexName string) { 124 | // TODO (iqdf): what if there are multiple errors 125 | // when creating indices after create table? 126 | err = tx.Migrator().CreateIndex(model, indexName) 127 | }(model, index.Name) 128 | continue 129 | } 130 | // TODO(iqdf): support primary key by put it as pass the fieldname 131 | // as MergeTree(...) parameters. But somehow it complained. 132 | // Note that primary key doesn't ensure uniqueness 133 | 134 | // Get indexing type `gorm:"index,type:minmax"` 135 | // Choice: minmax | set(n) | ngrambf_v1(n, size, hash, seed) | bloomfilter() 136 | indexType := m.Dialector.DefaultIndexType 137 | if index.Type != "" { 138 | indexType = index.Type 139 | } 140 | 141 | // Get expression for index options 142 | // Syntax: (`colname1`, ...) 143 | buildIndexOptions := tx.Migrator().(migrator.BuildIndexOptionsInterface) 144 | indexOptions := buildIndexOptions.BuildIndexOptions(index.Fields, stmt) 145 | 146 | // Stringify index builder 147 | // TODO (iqdf): support granularity 148 | str := fmt.Sprintf("INDEX ? ? TYPE %s GRANULARITY %d", indexType, m.getIndexGranularityOption(index.Fields)) 149 | indexSlice = append(indexSlice, str) 150 | args = append(args, clause.Expr{SQL: index.Name}, indexOptions) 151 | } 152 | indexStr := strings.Join(indexSlice, ", ") 153 | if len(indexSlice) > 0 { 154 | indexStr = ", " + indexStr 155 | } 156 | 157 | // Step 4. Finally assemble CREATE TABLE ... SQL string 158 | engineOpts := m.Dialector.DefaultTableEngineOpts 159 | if tableOption, ok := m.DB.Get("gorm:table_options"); ok { 160 | engineOpts = fmt.Sprint(tableOption) 161 | } 162 | 163 | clusterOpts := "" 164 | if clusterOption, ok := m.DB.Get("gorm:table_cluster_options"); ok { 165 | clusterOpts = " " + fmt.Sprint(clusterOption) + " " 166 | } 167 | 168 | createTableSQL = fmt.Sprintf(createTableSQL, clusterOpts, columnStr, constrStr, indexStr, engineOpts) 169 | 170 | err = tx.Exec(createTableSQL, args...).Error 171 | 172 | return 173 | }); err != nil { 174 | return err 175 | } 176 | } 177 | return nil 178 | } 179 | 180 | func (m Migrator) HasTable(value interface{}) bool { 181 | var count int64 182 | m.RunWithValue(value, func(stmt *gorm.Statement) error { 183 | currentDatabase := m.DB.Migrator().CurrentDatabase() 184 | return m.DB.Raw( 185 | "SELECT count(*) FROM system.tables WHERE database = ? AND name = ? AND is_temporary = ?", 186 | currentDatabase, 187 | stmt.Table, 188 | uint8(0)).Row().Scan(&count) 189 | }) 190 | return count > 0 191 | } 192 | 193 | func (m Migrator) GetTables() (tableList []string, err error) { 194 | // table_type Enum8('BASE TABLE' = 1, 'VIEW' = 2, 'FOREIGN TABLE' = 3, 'LOCAL TEMPORARY' = 4, 'SYSTEM VIEW' = 5) 195 | err = m.DB.Raw("SELECT TABLE_NAME FROM information_schema.tables where table_schema=? and table_type =1", m.CurrentDatabase()).Scan(&tableList).Error 196 | return 197 | } 198 | 199 | // Columns 200 | 201 | func (m Migrator) AddColumn(value interface{}, field string) error { 202 | return m.RunWithValue(value, func(stmt *gorm.Statement) error { 203 | if field := stmt.Schema.LookUpField(field); field != nil { 204 | clusterOpts := "" 205 | if clusterOption, ok := m.DB.Get("gorm:table_cluster_options"); ok { 206 | clusterOpts = " " + fmt.Sprint(clusterOption) + " " 207 | } 208 | sQL := fmt.Sprintf("ALTER TABLE ? %s ADD COLUMN ? ?", clusterOpts) 209 | return m.DB.Exec( 210 | sQL, 211 | clause.Table{Name: stmt.Table}, clause.Column{Name: field.DBName}, 212 | m.FullDataTypeOf(field), 213 | ).Error 214 | } 215 | return fmt.Errorf("failed to look up field with name: %s", field) 216 | }) 217 | } 218 | 219 | func (m Migrator) DropColumn(value interface{}, name string) error { 220 | return m.RunWithValue(value, func(stmt *gorm.Statement) error { 221 | if field := stmt.Schema.LookUpField(name); field != nil { 222 | name = field.DBName 223 | } 224 | clusterOpts := "" 225 | if clusterOption, ok := m.DB.Get("gorm:table_cluster_options"); ok { 226 | clusterOpts = " " + fmt.Sprint(clusterOption) + " " 227 | } 228 | sQL := fmt.Sprintf("ALTER TABLE ? %s DROP COLUMN ?", clusterOpts) 229 | return m.DB.Exec( 230 | sQL, 231 | clause.Table{Name: stmt.Table}, clause.Column{Name: name}, 232 | ).Error 233 | }) 234 | } 235 | 236 | func (m Migrator) AlterColumn(value interface{}, field string) error { 237 | return m.RunWithValue(value, func(stmt *gorm.Statement) error { 238 | if field := stmt.Schema.LookUpField(field); field != nil { 239 | clusterOpts := "" 240 | if clusterOption, ok := m.DB.Get("gorm:table_cluster_options"); ok { 241 | clusterOpts = " " + fmt.Sprint(clusterOption) + " " 242 | } 243 | sQL := fmt.Sprintf("ALTER TABLE ? %s MODIFY COLUMN ? ?", clusterOpts) 244 | return m.DB.Exec( 245 | sQL, 246 | clause.Table{Name: stmt.Table}, 247 | clause.Column{Name: field.DBName}, 248 | m.FullDataTypeOf(field), 249 | ).Error 250 | } 251 | return fmt.Errorf("altercolumn() failed to look up column with name: %s", field) 252 | }) 253 | } 254 | 255 | // NOTE: Only supported after ClickHouse 20.4 and above. 256 | // See: https://github.com/ClickHouse/ClickHouse/issues/146 257 | func (m Migrator) RenameColumn(value interface{}, oldName, newName string) error { 258 | return m.RunWithValue(value, func(stmt *gorm.Statement) error { 259 | if !m.Dialector.DontSupportRenameColumn { 260 | var field *schema.Field 261 | if f := stmt.Schema.LookUpField(oldName); f != nil { 262 | oldName = f.DBName 263 | field = f 264 | } 265 | if f := stmt.Schema.LookUpField(newName); f != nil { 266 | newName = f.DBName 267 | field = f 268 | } 269 | if field != nil { 270 | clusterOpts := "" 271 | if clusterOption, ok := m.DB.Get("gorm:table_cluster_options"); ok { 272 | clusterOpts = " " + fmt.Sprint(clusterOption) + " " 273 | } 274 | sQL := fmt.Sprintf("ALTER TABLE ? %s RENAME COLUMN ? TO ?", clusterOpts) 275 | return m.DB.Exec( 276 | sQL, 277 | clause.Table{Name: stmt.Table}, 278 | clause.Column{Name: oldName}, 279 | clause.Column{Name: newName}, 280 | ).Error 281 | } 282 | return fmt.Errorf("renamecolumn() failed to look up column with name: %s", oldName) 283 | } 284 | return ErrRenameIndexUnsupported 285 | }) 286 | } 287 | 288 | func (m Migrator) HasColumn(value interface{}, field string) bool { 289 | var count int64 290 | m.RunWithValue(value, func(stmt *gorm.Statement) error { 291 | currentDatabase := m.DB.Migrator().CurrentDatabase() 292 | name := field 293 | 294 | if stmt.Schema != nil { 295 | if field := stmt.Schema.LookUpField(field); field != nil { 296 | name = field.DBName 297 | } 298 | } 299 | 300 | return m.DB.Raw( 301 | "SELECT count(*) FROM system.columns WHERE database = ? AND table = ? AND name = ?", 302 | currentDatabase, stmt.Table, name, 303 | ).Row().Scan(&count) 304 | }) 305 | 306 | return count > 0 307 | } 308 | 309 | // ColumnTypes return columnTypes []gorm.ColumnType and execErr error 310 | func (m Migrator) ColumnTypes(value interface{}) ([]gorm.ColumnType, error) { 311 | columnTypes := make([]gorm.ColumnType, 0) 312 | execErr := m.RunWithValue(value, func(stmt *gorm.Statement) (err error) { 313 | rows, err := m.DB.Session(&gorm.Session{}).Table(stmt.Table).Limit(1).Rows() 314 | if err != nil { 315 | return err 316 | } 317 | 318 | defer func() { 319 | if err == nil { 320 | err = rows.Close() 321 | } 322 | }() 323 | 324 | var rawColumnTypes []*sql.ColumnType 325 | rawColumnTypes, err = rows.ColumnTypes() 326 | 327 | columnTypeSQL := "SELECT name, type, default_expression, comment, is_in_primary_key, character_octet_length, numeric_precision, numeric_precision_radix, numeric_scale, datetime_precision FROM system.columns WHERE database = ? AND table = ?" 328 | if m.Dialector.DontSupportColumnPrecision { 329 | columnTypeSQL = "SELECT name, type, default_expression, comment, is_in_primary_key FROM system.columns WHERE database = ? AND table = ?" 330 | } 331 | columns, rowErr := m.DB.Raw(columnTypeSQL, m.CurrentDatabase(), stmt.Table).Rows() 332 | if rowErr != nil { 333 | return rowErr 334 | } 335 | 336 | defer columns.Close() 337 | 338 | for columns.Next() { 339 | var ( 340 | column migrator.ColumnType 341 | decimalSizeValue *uint64 342 | datetimePrecision *uint64 343 | radixValue *uint64 344 | scaleValue *uint64 345 | lengthValue *uint64 346 | values = []interface{}{ 347 | &column.NameValue, &column.DataTypeValue, &column.DefaultValueValue, &column.CommentValue, &column.PrimaryKeyValue, &lengthValue, &decimalSizeValue, &radixValue, &scaleValue, &datetimePrecision, 348 | } 349 | ) 350 | 351 | if m.Dialector.DontSupportColumnPrecision { 352 | values = []interface{}{&column.NameValue, &column.DataTypeValue, &column.DefaultValueValue, &column.CommentValue, &column.PrimaryKeyValue} 353 | } 354 | 355 | if scanErr := columns.Scan(values...); scanErr != nil { 356 | return scanErr 357 | } 358 | 359 | column.ColumnTypeValue = column.DataTypeValue 360 | 361 | if decimalSizeValue != nil { 362 | column.DecimalSizeValue.Int64 = int64(*decimalSizeValue) 363 | column.DecimalSizeValue.Valid = true 364 | } else if datetimePrecision != nil { 365 | column.DecimalSizeValue.Int64 = int64(*datetimePrecision) 366 | column.DecimalSizeValue.Valid = true 367 | } 368 | 369 | if scaleValue != nil { 370 | column.ScaleValue.Int64 = int64(*scaleValue) 371 | column.ScaleValue.Valid = true 372 | } 373 | 374 | if lengthValue != nil { 375 | column.LengthValue.Int64 = int64(*lengthValue) 376 | column.LengthValue.Valid = true 377 | } 378 | 379 | if column.DefaultValueValue.Valid { 380 | column.DefaultValueValue.String = strings.Trim(column.DefaultValueValue.String, "'") 381 | } 382 | 383 | if m.Dialector.DontSupportEmptyDefaultValue && column.DefaultValueValue.String == "" { 384 | column.DefaultValueValue.Valid = false 385 | } 386 | 387 | for _, c := range rawColumnTypes { 388 | if c.Name() == column.NameValue.String { 389 | column.SQLColumnType = c 390 | break 391 | } 392 | } 393 | 394 | columnTypes = append(columnTypes, column) 395 | } 396 | 397 | return 398 | }) 399 | 400 | return columnTypes, execErr 401 | } 402 | 403 | // Indexes 404 | 405 | func (m Migrator) BuildIndexOptions(opts []schema.IndexOption, stmt *gorm.Statement) (results []interface{}) { 406 | for _, indexOpt := range opts { 407 | str := stmt.Quote(indexOpt.DBName) 408 | if indexOpt.Expression != "" { 409 | str = indexOpt.Expression 410 | } 411 | results = append(results, clause.Expr{SQL: str}) 412 | } 413 | return 414 | } 415 | 416 | func (m Migrator) CreateIndex(value interface{}, name string) error { 417 | return m.RunWithValue(value, func(stmt *gorm.Statement) error { 418 | if index := stmt.Schema.LookIndex(name); index != nil { 419 | opts := m.BuildIndexOptions(index.Fields, stmt) 420 | values := []interface{}{ 421 | clause.Table{Name: stmt.Table}, 422 | clause.Column{Name: index.Name}, 423 | opts, 424 | } 425 | 426 | // Get indexing type `gorm:"index,type:minmax"` 427 | // Choice: minmax | set(n) | ngrambf_v1(n, size, hash, seed) | bloomfilter() 428 | indexType := m.Dialector.DefaultIndexType 429 | if index.Type != "" { 430 | indexType = index.Type 431 | } 432 | 433 | // NOTE: concept of UNIQUE | FULLTEXT | SPATIAL index 434 | // is NOT supported in clickhouse 435 | createIndexSQL := "ALTER TABLE ? ADD INDEX ? ? TYPE %s GRANULARITY %d" // TODO(iqdf): how to inject Granularity 436 | createIndexSQL = fmt.Sprintf(createIndexSQL, indexType, m.getIndexGranularityOption(index.Fields)) // Granularity: 1 (default) 437 | return m.DB.Exec(createIndexSQL, values...).Error 438 | } 439 | return ErrCreateIndexFailed 440 | }) 441 | } 442 | 443 | func (m Migrator) RenameIndex(value interface{}, oldName, newName string) error { 444 | // TODO(iqdf): drop index and add the index again with different name 445 | // DROP INDEX ? 446 | // ADD INDEX ? TYPE ? GRANULARITY ? 447 | return ErrRenameIndexUnsupported 448 | } 449 | 450 | func (m Migrator) DropIndex(value interface{}, name string) error { 451 | return m.RunWithValue(value, func(stmt *gorm.Statement) error { 452 | if stmt.Schema != nil { 453 | if idx := stmt.Schema.LookIndex(name); idx != nil { 454 | name = idx.Name 455 | } 456 | } 457 | dropIndexSQL := "ALTER TABLE ? DROP INDEX ?" 458 | return m.DB.Exec(dropIndexSQL, 459 | clause.Table{Name: stmt.Table}, 460 | clause.Column{Name: name}).Error 461 | }) 462 | } 463 | 464 | func (m Migrator) HasIndex(value interface{}, name string) bool { 465 | var count int 466 | m.RunWithValue(value, func(stmt *gorm.Statement) error { 467 | currentDatabase := m.DB.Migrator().CurrentDatabase() 468 | 469 | if idx := stmt.Schema.LookIndex(name); idx != nil { 470 | name = idx.Name 471 | } 472 | 473 | showCreateTableSQL := fmt.Sprintf("SHOW CREATE TABLE %s.%s", currentDatabase, stmt.Table) 474 | var createStmt string 475 | if err := m.DB.Raw(showCreateTableSQL).Row().Scan(&createStmt); err != nil { 476 | return err 477 | } 478 | 479 | indexNames := m.extractIndexNamesFromCreateStmt(createStmt) 480 | 481 | // fmt.Printf("==== DEBUG ==== m.Mirror.HasIndex(%v, %v) count = %v, stmt: [\n%v\n]\nnames: %v\n", 482 | // stmt.Table, name, count, createStmt, indexNames) 483 | 484 | for _, indexName := range indexNames { 485 | if indexName == name { 486 | count = 1 487 | break 488 | } 489 | } 490 | return nil 491 | }) 492 | 493 | return count > 0 494 | } 495 | 496 | // Helper 497 | 498 | // Index 499 | 500 | func (m Migrator) getIndexGranularityOption(opts []schema.IndexOption) int { 501 | for _, indexOpt := range opts { 502 | if settingStr, ok := indexOpt.Field.TagSettings["INDEX"]; ok { 503 | // e.g. settingStr: "a,expression:u64*i32,type:minmax,granularity:3" 504 | for _, str := range strings.Split(settingStr, ",") { 505 | // e.g. str: "granularity:3" 506 | keyVal := strings.Split(str, ":") 507 | if len(keyVal) > 1 && strings.ToLower(keyVal[0]) == "granularity" { 508 | if len(keyVal) < 2 { 509 | // continue search for other setting which 510 | // may contain granularity: 511 | continue 512 | } 513 | // try to convert into an integer > 0 514 | // if check fails, continue search for other 515 | // settings which may contain granularity: 516 | num, err := strconv.Atoi(keyVal[1]) 517 | if err != nil || num < 0 { 518 | continue 519 | } 520 | return num 521 | } 522 | } 523 | } 524 | } 525 | return m.Dialector.DefaultGranularity 526 | } 527 | 528 | /* 529 | sample input: 530 | 531 | CREATE TABLE my_database.my_foo_bar 532 | ( 533 | 534 | `id` UInt64, 535 | `created_at` DateTime64(3), 536 | `updated_at` DateTime64(3), 537 | `deleted_at` DateTime64(3), 538 | `foo` String, 539 | `bar` String, 540 | INDEX idx_my_foo_bar_deleted_at deleted_at TYPE minmax GRANULARITY 3, 541 | INDEX my_fb_foo_bar (foo, bar) TYPE minmax GRANULARITY 3 542 | 543 | ) 544 | ENGINE = MergeTree 545 | PARTITION BY toYYYYMM(created_at) 546 | ORDER BY (foo, bar) 547 | SETTINGS index_granularity = 8192 548 | */ 549 | func (m Migrator) extractIndexNamesFromCreateStmt(createStmt string) []string { 550 | var names []string 551 | scanner := bufio.NewScanner(strings.NewReader(createStmt)) 552 | state := 0 // 0: before create body, 1: in create body, 2: after create body 553 | for scanner.Scan() && state < 2 { 554 | line := scanner.Text() 555 | switch state { 556 | case 0: 557 | if strings.HasPrefix(line, "(") { 558 | state = 1 559 | } 560 | case 1: 561 | if strings.HasPrefix(line, ")") { 562 | state = 2 563 | continue 564 | } 565 | line = strings.TrimSpace(line) 566 | if strings.HasPrefix(line, "INDEX ") { 567 | line = strings.TrimPrefix(line, "INDEX ") 568 | elems := strings.Split(line, " ") 569 | if len(elems) > 0 { 570 | names = append(names, elems[0]) 571 | } 572 | } 573 | } 574 | } 575 | return names 576 | } 577 | -------------------------------------------------------------------------------- /migrator_test.go: -------------------------------------------------------------------------------- 1 | package clickhouse_test 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | clickhousego "github.com/ClickHouse/clickhouse-go/v2" 8 | "gorm.io/driver/clickhouse" 9 | "gorm.io/gorm" 10 | ) 11 | 12 | type User struct { 13 | ID uint64 `gorm:"primaryKey"` 14 | Name string 15 | FirstName string 16 | LastName string 17 | Age int64 `gorm:"type:Nullable(Int64)"` 18 | Active bool 19 | Salary float32 20 | Attrs map[string]string `gorm:"type:Map(String,String);"` 21 | CreatedAt time.Time 22 | UpdatedAt time.Time 23 | } 24 | 25 | func TestAutoMigrate(t *testing.T) { 26 | type UserMigrateColumn struct { 27 | ID uint64 28 | Name string 29 | IsAdmin bool 30 | Birthday time.Time `gorm:"precision:4"` 31 | Debit float64 `gorm:"precision:4"` 32 | Note string `gorm:"size:10;comment:my note"` 33 | DefaultValue string `gorm:"default:hello world"` 34 | } 35 | 36 | if DB.Migrator().HasColumn("users", "is_admin") { 37 | t.Fatalf("users's is_admin column should not exists") 38 | } 39 | 40 | if err := DB.Table("users").AutoMigrate(&UserMigrateColumn{}); err != nil { 41 | t.Fatalf("no error should happen when auto migrate, but got %v", err) 42 | } 43 | 44 | if !DB.Migrator().HasTable("users") { 45 | t.Fatalf("users should exists") 46 | } 47 | 48 | if !DB.Migrator().HasColumn("users", "is_admin") { 49 | t.Fatalf("users's is_admin column should exists after auto migrate") 50 | } 51 | 52 | columnTypes, err := DB.Migrator().ColumnTypes("users") 53 | if err != nil { 54 | t.Fatalf("failed to get column types, got error %v", err) 55 | } 56 | 57 | for _, columnType := range columnTypes { 58 | switch columnType.Name() { 59 | case "id": 60 | if columnType.DatabaseTypeName() != "UInt64" { 61 | t.Fatalf("column id primary key should be correct, name: %v, column: %#v", columnType.Name(), columnType) 62 | } 63 | case "note": 64 | if length, ok := columnType.Length(); !ok || length != 10 { 65 | t.Fatalf("column name length should be correct, name: %v, column: %#v", columnType.Name(), columnType) 66 | } 67 | 68 | if comment, ok := columnType.Comment(); !ok || comment != "my note" { 69 | t.Fatalf("column name length should be correct, name: %v, column: %#v", columnType.Name(), columnType) 70 | } 71 | case "default_value": 72 | if defaultValue, ok := columnType.DefaultValue(); !ok || defaultValue != "hello world" { 73 | t.Fatalf("column name default_value should be correct, name: %v, column: %#v", columnType.Name(), columnType) 74 | } 75 | case "debit": 76 | if decimal, scale, ok := columnType.DecimalSize(); !ok || (scale != 0 || decimal != 4) { 77 | t.Fatalf("column name debit should be correct, name: %v, column: %#v", columnType.Name(), columnType) 78 | } 79 | case "birthday": 80 | if decimal, scale, ok := columnType.DecimalSize(); !ok || (scale != 0 || decimal != 4) { 81 | t.Fatalf("column name birthday should be correct, name: %v, column: %#v", columnType.Name(), columnType) 82 | } 83 | } 84 | } 85 | } 86 | 87 | func TestMigrator_HasIndex(t *testing.T) { 88 | type UserWithIndex struct { 89 | FirstName string `gorm:"index:full_name"` 90 | LastName string `gorm:"index:full_name"` 91 | CreatedAt time.Time `gorm:"index"` 92 | } 93 | if DB.Migrator().HasIndex("users", "full_name") { 94 | t.Fatalf("users's full_name index should not exists") 95 | } 96 | 97 | if err := DB.Table("users").AutoMigrate(&UserWithIndex{}); err != nil { 98 | t.Fatalf("no error should happen when auto migrate, but got %v", err) 99 | } 100 | 101 | if !DB.Migrator().HasIndex("users", "full_name") { 102 | t.Fatalf("users's full_name index should exists after auto migrate") 103 | } 104 | 105 | if err := DB.Table("users").AutoMigrate(&UserWithIndex{}); err != nil { 106 | t.Fatalf("no error should happen when auto migrate again") 107 | } 108 | } 109 | 110 | func TestMigrator_DontSupportEmptyDefaultValue(t *testing.T) { 111 | options, err := clickhousego.ParseDSN(dbDSN) 112 | if err != nil { 113 | t.Fatalf("Can not parse dsn, got error %v", err) 114 | } 115 | 116 | DB, err := gorm.Open(clickhouse.New(clickhouse.Config{ 117 | Conn: clickhousego.OpenDB(options), 118 | DontSupportEmptyDefaultValue: true, 119 | })) 120 | if err != nil { 121 | t.Fatalf("failed to connect database, got error %v", err) 122 | } 123 | 124 | type MyTable struct { 125 | MyField string 126 | } 127 | 128 | // Create the table with AutoMigrate 129 | if err := DB.Table("mytable").AutoMigrate(&MyTable{}); err != nil { 130 | t.Fatalf("no error should happen when auto migrate, but got %v", err) 131 | } 132 | 133 | // Replace every gorm raw SQL command with a function that appends the SQL string to a slice 134 | sqlStrings := make([]string, 0) 135 | if err := DB.Callback().Raw().Replace("gorm:raw", func(db *gorm.DB) { 136 | sqlToExecute := db.Statement.SQL.String() 137 | sqlStrings = append(sqlStrings, sqlToExecute) 138 | }); err != nil { 139 | t.Fatalf("no error should happen when registering a callback, but got %v", err) 140 | } 141 | 142 | if err := DB.Table("mytable").AutoMigrate(&MyTable{}); err != nil { 143 | t.Fatalf("no error should happen when auto migrate, but got %v", err) 144 | } 145 | if len(sqlStrings) > 0 { 146 | t.Fatalf("should not auto-migrate table if there have not been any changes to the schema") 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /tests_test.go: -------------------------------------------------------------------------------- 1 | package clickhouse_test 2 | 3 | import ( 4 | "log" 5 | "math/rand" 6 | "os" 7 | "time" 8 | 9 | "gorm.io/driver/clickhouse" 10 | "gorm.io/gorm" 11 | ) 12 | 13 | var DB *gorm.DB 14 | 15 | const dbDSN = "clickhouse://gorm:gorm@127.0.0.1:9942/gorm?dial_timeout=10s&read_timeout=20s" 16 | 17 | func init() { 18 | var ( 19 | err error 20 | ) 21 | 22 | if DB, err = gorm.Open(clickhouse.Open(dbDSN), &gorm.Config{}); err != nil { 23 | log.Printf("failed to connect database, got error %v", err) 24 | os.Exit(1) 25 | } 26 | 27 | RunMigrations() 28 | 29 | if os.Getenv("DEBUG") == "true" { 30 | DB = DB.Debug() 31 | } 32 | } 33 | 34 | func RunMigrations() { 35 | allModels := []interface{}{&User{}} 36 | rand.Seed(time.Now().UnixNano()) 37 | rand.Shuffle(len(allModels), func(i, j int) { allModels[i], allModels[j] = allModels[j], allModels[i] }) 38 | 39 | if err := DB.Migrator().DropTable(allModels...); err != nil { 40 | log.Printf("Failed to drop table, got error %v\n", err) 41 | os.Exit(1) 42 | } 43 | 44 | if err := DB.AutoMigrate(allModels...); err != nil { 45 | log.Printf("Failed to auto migrate, but got error %v\n", err) 46 | os.Exit(1) 47 | } 48 | 49 | for _, m := range allModels { 50 | if !DB.Migrator().HasTable(m) { 51 | log.Printf("Failed to create table for %#v\n", m) 52 | os.Exit(1) 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /update.go: -------------------------------------------------------------------------------- 1 | package clickhouse 2 | 3 | import ( 4 | "regexp" 5 | 6 | "github.com/ClickHouse/clickhouse-go/v2" 7 | "gorm.io/gorm" 8 | "gorm.io/gorm/callbacks" 9 | "gorm.io/gorm/clause" 10 | ) 11 | 12 | const updateLocalTableName = "gorm:clickhouse:update_local_table_name" 13 | 14 | var tableRegexp = regexp.MustCompile("(?i)(alter\\s+table\\s+(?:`?[\\d\\w_]+`?\\.)?`?)([\\d\\w_]+)(`?)") 15 | 16 | type UpdateLocalTable struct { 17 | Table string 18 | Prefix string 19 | Suffix string 20 | } 21 | 22 | // ModifyStatement modify operation mode 23 | func (t UpdateLocalTable) ModifyStatement(stmt *gorm.Statement) { 24 | stmt.Settings.Store(updateLocalTableName, t) 25 | } 26 | 27 | // Build implements clause.Expression interface 28 | func (t UpdateLocalTable) Build(clause.Builder) { 29 | } 30 | 31 | func (t UpdateLocalTable) ModifySQL(sql string) string { 32 | switch { 33 | case t.Suffix != "": 34 | return tableRegexp.ReplaceAllString(sql, "${1}${2}"+t.Suffix+"${3}") 35 | case t.Prefix != "": 36 | return tableRegexp.ReplaceAllString(sql, "${1}"+t.Prefix+"${2}${3}") 37 | case t.Table != "": 38 | return tableRegexp.ReplaceAllString(sql, "${1}"+t.Table+"${3}") 39 | } 40 | return sql 41 | } 42 | 43 | func (dialector *Dialector) Update(db *gorm.DB) { 44 | if db.Error != nil { 45 | return 46 | } 47 | 48 | if db.Statement.Schema != nil { 49 | for _, c := range db.Statement.Schema.UpdateClauses { 50 | db.Statement.AddClause(c) 51 | } 52 | } 53 | 54 | if db.Statement.SQL.Len() == 0 { 55 | db.Statement.SQL.Grow(180) 56 | db.Statement.AddClauseIfNotExists(clause.Update{}) 57 | if _, ok := db.Statement.Clauses["SET"]; !ok { 58 | if set := callbacks.ConvertToAssignments(db.Statement); len(set) != 0 { 59 | defer delete(db.Statement.Clauses, "SET") 60 | db.Statement.AddClause(set) 61 | } else { 62 | return 63 | } 64 | } 65 | 66 | db.Statement.Build(db.Statement.BuildClauses...) 67 | } 68 | 69 | if db.Error != nil { 70 | return 71 | } 72 | 73 | updateSQL := db.Statement.SQL.String() 74 | if updateLocalTableClause, ok := db.Statement.Settings.Load(updateLocalTableName); ok && len(dialector.options.Addr) >= 1 { 75 | if updateLocalTable, ok := updateLocalTableClause.(UpdateLocalTable); ok { 76 | var ( 77 | err error 78 | opts = dialector.options 79 | addrs = opts.Addr 80 | updateSQL = updateLocalTable.ModifySQL(updateSQL) 81 | ) 82 | 83 | db.Statement.SQL.Reset() 84 | db.Statement.SQL.WriteString(updateSQL) 85 | 86 | if !db.DryRun { 87 | for i := 0; i < len(addrs); i++ { 88 | opts := opts 89 | opts.Addr = []string{addrs[i]} 90 | 91 | for j := 0; j < 3; j++ { 92 | if conn, e := clickhouse.Open(&opts); e == nil { 93 | defer conn.Close() 94 | if e = conn.Exec(db.Statement.Context, updateSQL, db.Statement.Vars...); e == nil { 95 | break 96 | } 97 | err = e 98 | } 99 | } 100 | 101 | if err != nil { 102 | break 103 | } 104 | } 105 | } 106 | db.AddError(err) 107 | return 108 | } 109 | } 110 | 111 | if !db.DryRun { 112 | result, err := db.Statement.ConnPool.ExecContext(db.Statement.Context, updateSQL, db.Statement.Vars...) 113 | 114 | if db.AddError(err) == nil { 115 | db.RowsAffected, _ = result.RowsAffected() 116 | 117 | if db.Statement.Result != nil { 118 | db.Statement.Result.Result = result 119 | db.Statement.Result.RowsAffected = db.RowsAffected 120 | } 121 | } 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /update_test.go: -------------------------------------------------------------------------------- 1 | package clickhouse_test 2 | 3 | import ( 4 | "regexp" 5 | "testing" 6 | "time" 7 | 8 | "gorm.io/driver/clickhouse" 9 | "gorm.io/gorm" 10 | "gorm.io/gorm/utils/tests" 11 | ) 12 | 13 | func TestUpdateLocalTable(t *testing.T) { 14 | updateLocalTable := clickhouse.UpdateLocalTable{Suffix: "_local"} 15 | for k, v := range map[string]string{ 16 | "alter table hello_world.hello_world2 update id=1": "alter table hello_world.hello_world2_local update id=1", 17 | "Alter table `hello_world`.hello_world2 update id=1": "Alter table `hello_world`.hello_world2_local update id=1", 18 | "ALTER TABLE hello_world.`hello_world2` update id=1": "ALTER TABLE hello_world.`hello_world2_local` update id=1", 19 | "alter TABLE `hello_world`.`hello_world2` update id=1": "alter TABLE `hello_world`.`hello_world2_local` update id=1", 20 | "ALTER TABLE `users` UPDATE `name`=?,`updated_at`=? WHERE `id` = ?": "ALTER TABLE `users_local` UPDATE `name`=?,`updated_at`=? WHERE `id` = ?", 21 | } { 22 | if updateLocalTable.ModifySQL(k) != v { 23 | t.Errorf("failed to update sql, expect: %v, got %v", v, updateLocalTable.ModifySQL(k)) 24 | } 25 | } 26 | 27 | updateLocalTable = clickhouse.UpdateLocalTable{Prefix: "local_"} 28 | for k, v := range map[string]string{ 29 | "alter table hello_world.hello_world2 update id=1": "alter table hello_world.local_hello_world2 update id=1", 30 | "alter table `hello_world`.hello_world2 update id=1": "alter table `hello_world`.local_hello_world2 update id=1", 31 | "alter table hello_world.`hello_world2` update id=1": "alter table hello_world.`local_hello_world2` update id=1", 32 | "alter table `hello_world`.`hello_world2` update id=1": "alter table `hello_world`.`local_hello_world2` update id=1", 33 | } { 34 | if updateLocalTable.ModifySQL(k) != v { 35 | t.Errorf("failed to update sql, expect: %v, got %v", v, updateLocalTable.ModifySQL(k)) 36 | } 37 | } 38 | 39 | updateLocalTable = clickhouse.UpdateLocalTable{Table: "local_table"} 40 | for k, v := range map[string]string{ 41 | "alter table hello_world.hello_world2 update id=1": "alter table hello_world.local_table update id=1", 42 | "ALTER table `hello_world`.hello_world2 update id=1": "ALTER table `hello_world`.local_table update id=1", 43 | "alter table hello_world.`hello_world2` update id=1": "alter table hello_world.`local_table` update id=1", 44 | "ALTER TABLE `hello_world`.`hello_world2` update id=1": "ALTER TABLE `hello_world`.`local_table` update id=1", 45 | } { 46 | if updateLocalTable.ModifySQL(k) != v { 47 | t.Errorf("failed to update sql, expect: %v, got %v", v, updateLocalTable.ModifySQL(k)) 48 | } 49 | } 50 | } 51 | 52 | func TestUpdate(t *testing.T) { 53 | user := User{ID: 3, Name: "update", FirstName: "zhang", LastName: "jinzhu", Age: 18, Active: true, Salary: 8.8888} 54 | 55 | if err := DB.Create(&user).Error; err != nil { 56 | t.Fatalf("failed to create user, got error %v", err) 57 | } 58 | 59 | var result User 60 | if err := DB.Find(&result, user.ID).Error; err != nil { 61 | t.Fatalf("failed to query user, got error %v", err) 62 | } 63 | 64 | tests.AssertEqual(t, result, user) 65 | 66 | if err := DB.Model(&result).Update("name", "update-1").Error; err != nil { 67 | t.Fatalf("failed to update user, got error %v", err) 68 | } 69 | 70 | time.Sleep(200 * time.Millisecond) 71 | 72 | var result2 User 73 | if err := DB.First(&result2, user.ID).Error; err != nil { 74 | t.Fatalf("failed to query user, got error %v", err) 75 | } 76 | 77 | user.Name = "update-1" 78 | tests.AssertEqual(t, result2, user) 79 | 80 | sql := DB.ToSQL(func(tx *gorm.DB) *gorm.DB { 81 | return tx.Clauses(clickhouse.UpdateLocalTable{Suffix: "_local"}).Model(&result).Update("name", "update-1") 82 | }) 83 | 84 | if !regexp.MustCompile("`users_local`").MatchString(sql) { 85 | t.Errorf("Table with namer, got %v", sql) 86 | } 87 | } 88 | 89 | func TestUpdateWithMap(t *testing.T) { 90 | user := User{ID: 33, Name: "update2", FirstName: "zhang", LastName: "jinzhu", Age: 18, Active: true, Salary: 8.8888} 91 | 92 | if err := DB.Create(&user).Error; err != nil { 93 | t.Fatalf("failed to create user, got error %v", err) 94 | } 95 | 96 | var result User 97 | if err := DB.Find(&result, user.ID).Error; err != nil { 98 | t.Fatalf("failed to query user, got error %v", err) 99 | } 100 | 101 | tests.AssertEqual(t, result, user) 102 | 103 | if err := DB.Table("users").Where("id = ?", user.ID).Update("name", "update-2").Error; err != nil { 104 | t.Fatalf("failed to update user, got error %v", err) 105 | } 106 | 107 | time.Sleep(200 * time.Millisecond) 108 | 109 | var result2 User 110 | if err := DB.First(&result2, user.ID).Error; err != nil { 111 | t.Fatalf("failed to query user, got error %v", err) 112 | } 113 | 114 | user.Name = "update-2" 115 | tests.AssertEqual(t, result2, user) 116 | 117 | if err := DB.Table("users").Where("id = ?", user.ID).Updates(map[string]interface{}{"name": "update-3"}).Error; err != nil { 118 | t.Fatalf("failed to update user, got error %v", err) 119 | } 120 | 121 | time.Sleep(200 * time.Millisecond) 122 | 123 | var result3 User 124 | if err := DB.First(&result3, user.ID).Error; err != nil { 125 | t.Fatalf("failed to query user, got error %v", err) 126 | } 127 | 128 | user.Name = "update-3" 129 | tests.AssertEqual(t, result3, user) 130 | } 131 | --------------------------------------------------------------------------------