├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── cluster.go ├── cluster_test.go ├── conn.go ├── conn_test.go ├── errors.go ├── errors_test.go ├── helpers.go ├── helpers_test.go ├── marshal.go ├── marshal_test.go ├── query.go ├── query_test.go ├── transport.go └── transport_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | vendor/ 2 | glide.lock 3 | 4 | .idea 5 | 6 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 7 | *.o 8 | *.a 9 | *.so 10 | 11 | *.gor 12 | 13 | # Folders 14 | _obj 15 | _test 16 | 17 | # Architecture specific extensions/prefixes 18 | *.[568vq] 19 | [568vq].out 20 | 21 | *.cgo1.go 22 | *.cgo2.c 23 | _cgo_defun.c 24 | _cgo_gotypes.go 25 | _cgo_export.* 26 | 27 | _testmain.go 28 | 29 | *.exe 30 | *.test 31 | 32 | 33 | *.sublime-workspace 34 | *.sw* 35 | *.un* 36 | 37 | app.conf.json 38 | docker.conf.json 39 | 40 | target/ 41 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - 1.3 5 | - 1.4 6 | - 1.5 7 | - 1.6 8 | - 1.7 9 | - 1.8 10 | - 1.9 11 | 12 | before_install: 13 | - go get github.com/mattn/goveralls 14 | - go get golang.org/x/tools/cmd/cover 15 | script: 16 | - $HOME/gopath/bin/goveralls -service=travis-ci 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Roistat 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # go-clickhouse 2 | # [![Travis status](https://img.shields.io/travis/roistat/go-clickhouse.svg)](https://travis-ci.org/roistat/go-clickhouse) [![Coverage Status](https://img.shields.io/coveralls/roistat/go-clickhouse.svg)](https://coveralls.io/github/roistat/go-clickhouse) [![Go Report](https://goreportcard.com/badge/github.com/roistat/go-clickhouse)](https://goreportcard.com/report/github.com/roistat/go-clickhouse) ![](https://img.shields.io/github/license/roistat/go-clickhouse.svg) 3 | 4 | Golang [Yandex ClickHouse](https://clickhouse.yandex/) connector 5 | 6 | ClickHouse manages extremely large volumes of data in a stable and sustainable manner. 7 | It currently powers Yandex.Metrica, world’s second largest web analytics platform, 8 | with over 13 trillion database records and over 20 billion events a day, generating 9 | customized reports on-the-fly, directly from non-aggregated data. This system was 10 | successfully implemented at CERN’s LHCb experiment to store and process metadata on 11 | 10bn events with over 1000 attributes per event registered in 2011. 12 | 13 | ## Examples 14 | 15 | #### Query rows 16 | 17 | ```go 18 | conn := clickhouse.NewConn("localhost:8123", clickhouse.NewHttpTransport()) 19 | query := clickhouse.NewQuery("SELECT name, date FROM clicks") 20 | iter := query.Iter(conn) 21 | var ( 22 | name string 23 | date string 24 | ) 25 | for iter.Scan(&name, &date) { 26 | // 27 | } 28 | if iter.Error() != nil { 29 | log.Panicln(iter.Error()) 30 | } 31 | ``` 32 | 33 | #### Single insert 34 | ```go 35 | conn := clickhouse.NewConn("localhost:8123", clickhouse.NewHttpTransport()) 36 | query, err := clickhouse.BuildInsert("clicks", 37 | clickhouse.Columns{"name", "date", "sourceip"}, 38 | clickhouse.Row{"Test name", "2016-01-01 21:01:01", clickhouse.Func{"IPv4StringToNum", "192.0.2.192"}}, 39 | ) 40 | if err == nil { 41 | err = query.Exec(conn) 42 | if err == nil { 43 | // 44 | } 45 | } 46 | ``` 47 | 48 | #### External data for query processing 49 | 50 | [See documentation for details](https://clickhouse.yandex/reference_en.html#External%20data%20for%20query%20processing) 51 | 52 | ```go 53 | conn := clickhouse.NewConn("localhost:8123", clickhouse.NewHttpTransport()) 54 | query := clickhouse.NewQuery("SELECT Num, Name FROM extdata") 55 | query.AddExternal("extdata", "Num UInt32, Name String", []byte("1 first\n2 second")) // tab separated 56 | 57 | 58 | iter := query.Iter(conn) 59 | var ( 60 | num int 61 | name string 62 | ) 63 | for iter.Scan(&num, &name) { 64 | // 65 | } 66 | if iter.Error() != nil { 67 | log.Panicln(iter.Error()) 68 | } 69 | ``` 70 | 71 | ## Cluster 72 | 73 | Cluster is useful if you have several servers with same `Distributed` table (master). In this case you can send 74 | requests to random master to balance load. 75 | 76 | * `cluster.Check()` pings all connections and filters active ones 77 | * `cluster.ActiveConn()` returns random active connection 78 | * `cluster.OnCheckError()` is called when any connection fails 79 | 80 | **Important**: You should call method `Check()` at least once after initialization, but we recommend 81 | to call it continuously, so `ActiveConn()` will always return filtered active connection. 82 | 83 | ```go 84 | http := clickhouse.NewHttpTransport() 85 | conn1 := clickhouse.NewConn("host1", http) 86 | conn2 := clickhouse.NewConn("host2", http) 87 | 88 | cluster := clickhouse.NewCluster(conn1, conn2) 89 | cluster.OnCheckError(func (c *clickhouse.Conn) { 90 | log.Fatalf("Clickhouse connection failed %s", c.Host) 91 | }) 92 | // Ping connections every second 93 | go func() { 94 | for { 95 | cluster.Check() 96 | time.Sleep(time.Second) 97 | } 98 | }() 99 | ``` 100 | 101 | ## Transport options 102 | 103 | ### Timeout 104 | 105 | ```go 106 | t := clickhouse.NewHttpTransport() 107 | t.Timeout = time.Second * 5 108 | 109 | conn := clickhouse.NewConn("host", t) 110 | ``` 111 | -------------------------------------------------------------------------------- /cluster.go: -------------------------------------------------------------------------------- 1 | package clickhouse 2 | 3 | import ( 4 | "math/rand" 5 | "sync" 6 | ) 7 | 8 | type PingErrorFunc func(*Conn) 9 | 10 | type Cluster struct { 11 | conn []*Conn 12 | active []*Conn 13 | fail PingErrorFunc 14 | mx sync.Mutex 15 | } 16 | 17 | func NewCluster(conn ...*Conn) *Cluster { 18 | return &Cluster{ 19 | conn: conn, 20 | } 21 | } 22 | 23 | func (c *Cluster) IsDown() bool { 24 | c.mx.Lock() 25 | defer c.mx.Unlock() 26 | return len(c.active) < 1 27 | } 28 | 29 | func (c *Cluster) OnCheckError(f PingErrorFunc) { 30 | c.fail = f 31 | } 32 | 33 | func (c *Cluster) ActiveConn() *Conn { 34 | c.mx.Lock() 35 | defer c.mx.Unlock() 36 | l := len(c.active) 37 | if l < 1 { 38 | return nil 39 | } 40 | return c.active[rand.Intn(l)] 41 | } 42 | 43 | func (c *Cluster) Check() { 44 | var ( 45 | err error 46 | res []*Conn 47 | ) 48 | 49 | for _, conn := range c.conn { 50 | err = conn.Ping() 51 | if err == nil { 52 | res = append(res, conn) 53 | } else { 54 | if c.fail != nil { 55 | c.fail(conn) 56 | } 57 | } 58 | } 59 | 60 | c.mx.Lock() 61 | defer c.mx.Unlock() 62 | 63 | c.active = res 64 | } 65 | -------------------------------------------------------------------------------- /cluster_test.go: -------------------------------------------------------------------------------- 1 | package clickhouse 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestPing(t *testing.T) { 9 | goodTr := getMockTransport("Ok.") 10 | badTr := getMockTransport("Code: 9999, Error: ...") 11 | 12 | conn1 := NewConn("host1", badTr) 13 | conn2 := NewConn("host2", goodTr) 14 | 15 | cl := NewCluster(conn1, conn2) 16 | assert.Equal(t, conn1, cl.conn[0]) 17 | assert.Equal(t, conn2, cl.conn[1]) 18 | 19 | assert.True(t, cl.IsDown()) 20 | 21 | cl.OnCheckError(func(c *Conn) { 22 | assert.Equal(t, conn1, c) 23 | }) 24 | 25 | cl.Check() 26 | 27 | assert.Equal(t, conn2.Host, cl.ActiveConn().Host) 28 | 29 | assert.False(t, cl.IsDown()) 30 | 31 | cl.conn[0] = NewConn("host1", goodTr) 32 | cl.conn[1] = NewConn("host2", badTr) 33 | 34 | cl.OnCheckError(func(c *Conn) { 35 | assert.Equal(t, conn2.Host, c.Host) 36 | }) 37 | 38 | cl.Check() 39 | 40 | assert.Equal(t, conn1.Host, cl.ActiveConn().Host) 41 | 42 | cl.conn[0] = NewConn("host1", badTr) 43 | cl.conn[1] = NewConn("host2", badTr) 44 | 45 | cl.OnCheckError(func(c *Conn) {}) 46 | cl.Check() 47 | 48 | assert.Nil(t, cl.ActiveConn()) 49 | 50 | assert.True(t, cl.IsDown()) 51 | } 52 | -------------------------------------------------------------------------------- /conn.go: -------------------------------------------------------------------------------- 1 | package clickhouse 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | ) 7 | 8 | const ( 9 | successTestResponse = "Ok." 10 | ) 11 | 12 | type Conn struct { 13 | Host string 14 | transport Transport 15 | User string 16 | Password string 17 | } 18 | 19 | func (c *Conn) Ping() (err error) { 20 | var res string 21 | res, err = c.transport.Exec(c, Query{Stmt: ""}, true) 22 | if err == nil { 23 | if !strings.Contains(res, successTestResponse) { 24 | err = fmt.Errorf("Clickhouse host response was '%s', expected '%s'.", res, successTestResponse) 25 | } 26 | } 27 | 28 | return err 29 | } 30 | -------------------------------------------------------------------------------- /conn_test.go: -------------------------------------------------------------------------------- 1 | package clickhouse 2 | 3 | import ( 4 | "errors" 5 | "github.com/stretchr/testify/assert" 6 | "testing" 7 | ) 8 | 9 | func TestConnect(t *testing.T) { 10 | var conn *Conn 11 | tr := getMockTransport("Ok.") 12 | 13 | conn = NewConn("host.local", tr) 14 | assert.Equal(t, "http://host.local/", conn.Host) 15 | 16 | conn = NewConn("http://host.local/", tr) 17 | assert.Equal(t, "http://host.local/", conn.Host) 18 | 19 | conn = NewConn("https://host.local/", tr) 20 | assert.Equal(t, "https://host.local/", conn.Host) 21 | 22 | conn = NewConn("http:/host.local", tr) 23 | assert.Equal(t, "http://http:/host.local/", conn.Host) 24 | } 25 | 26 | func TestConn_Ping(t *testing.T) { 27 | tr := getMockTransport("Ok.") 28 | conn := NewConn("host.local", tr) 29 | assert.NoError(t, conn.Ping()) 30 | } 31 | 32 | func TestConn_Ping2(t *testing.T) { 33 | tr := getMockTransport("") 34 | conn := NewConn("host.local", tr) 35 | assert.Error(t, conn.Ping()) 36 | } 37 | 38 | func TestConn_Ping3(t *testing.T) { 39 | tr := badTransport{err: errors.New("Connection timeout")} 40 | conn := NewConn("host.local", tr) 41 | assert.Error(t, conn.Ping()) 42 | assert.Equal(t, "Connection timeout", conn.Ping().Error()) 43 | } 44 | -------------------------------------------------------------------------------- /errors.go: -------------------------------------------------------------------------------- 1 | package clickhouse 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | "strconv" 7 | "strings" 8 | ) 9 | 10 | type DbError struct { 11 | code int 12 | msg string 13 | resp string 14 | } 15 | 16 | func (e *DbError) Code() int { 17 | return e.code 18 | } 19 | 20 | func (e *DbError) Message() string { 21 | return e.msg 22 | } 23 | 24 | func (e *DbError) Response() string { 25 | return e.resp 26 | } 27 | 28 | func (e *DbError) Error() string { 29 | return fmt.Sprintf("clickhouse error: [%d] %s", e.code, e.msg) 30 | } 31 | 32 | func (e *DbError) String() string { 33 | return fmt.Sprintf("[error code=%d message=%q]", e.code, e.msg) 34 | } 35 | 36 | func errorFromResponse(resp string) error { 37 | if resp == "" { 38 | return nil 39 | } 40 | errorPattern, err := regexp.Compile(`Code:\s(\d+)[.,]?(.*)`) 41 | if err != nil { 42 | return err 43 | } 44 | if !errorPattern.MatchString(resp) { 45 | return nil 46 | } 47 | 48 | matches := errorPattern.FindStringSubmatch(resp) 49 | code, err := strconv.Atoi(matches[1]) 50 | if err != nil { 51 | return err 52 | } 53 | msg := matches[2] 54 | msg = strings.ReplaceAll(msg, "e.displayText() = ", "") 55 | msg = strings.ReplaceAll(msg, ", e.what()", "") 56 | msg = strings.TrimSpace(msg) 57 | return &DbError{code, msg, resp} 58 | } 59 | -------------------------------------------------------------------------------- /errors_test.go: -------------------------------------------------------------------------------- 1 | package clickhouse 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestErrorFromResponse(t *testing.T) { 9 | var err *DbError 10 | 11 | assert.NoError(t, errorFromResponse("")) 12 | assert.NoError(t, errorFromResponse("Ok.")) 13 | 14 | err = errorFromResponse("Code: 140, 000000").(*DbError) 15 | 16 | assert.Error(t, err) 17 | assert.Equal(t, 140, err.Code()) 18 | assert.Equal(t, "", err.Message()) 19 | 20 | err = errorFromResponse("Code: 62, e.displayText() = DB::Exception: Syntax error: failed at end of query.\n" + 21 | "Expected identifier, e.what() = DB::Exception").(*DbError) 22 | 23 | assert.Error(t, err) 24 | assert.Equal(t, 62, err.Code()) 25 | assert.Equal(t, "clickhouse error: [62] DB::Exception: Syntax error: failed at end of query.\nExpected identifier", 26 | err.Error()) 27 | assert.Equal(t, "[error code=62 message=\"DB::Exception: Syntax error: failed at end of query.\\nExpected identifier\"]", 28 | err.String()) 29 | assert.Equal(t, "DB::Exception: Syntax error: failed at end of query.\nExpected identifier", err.Message()) 30 | 31 | resp := "Code: 3, e.displayText() = DB::Exception: Syntax error: failed at end of query.\nExpected identifier," 32 | err = errorFromResponse(resp).(*DbError) 33 | 34 | assert.Error(t, err) 35 | assert.Equal(t, 3, err.Code()) 36 | assert.Equal(t, resp, err.Response()) 37 | assert.Equal(t, "DB::Exception: Syntax error: failed at end of query.\nExpected identifier,", err.Message()) 38 | } 39 | -------------------------------------------------------------------------------- /helpers.go: -------------------------------------------------------------------------------- 1 | package clickhouse 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "strings" 7 | ) 8 | 9 | type ( 10 | Column string 11 | Columns []string 12 | Row []interface{} 13 | Rows []Row 14 | Array []interface{} 15 | ) 16 | 17 | func NewHttpTransport() HttpTransport { 18 | return HttpTransport{} 19 | } 20 | 21 | func NewConn(host string, t Transport) *Conn { 22 | return NewConnWithAuth(host, t, "", "") 23 | } 24 | 25 | func NewConnWithAuth(host string, t Transport, user string, password string) *Conn { 26 | if strings.Index(host, "http://") < 0 && strings.Index(host, "https://") < 0 { 27 | host = "http://" + host 28 | } 29 | host = strings.TrimRight(host, "/") + "/" 30 | 31 | return &Conn{ 32 | Host: host, 33 | transport: t, 34 | User: user, 35 | Password: password, 36 | } 37 | } 38 | 39 | func NewQuery(stmt string, args ...interface{}) Query { 40 | return Query{ 41 | Stmt: stmt, 42 | args: args, 43 | } 44 | } 45 | 46 | func BuildInsert(tbl string, cols Columns, row Row) (Query, error) { 47 | return BuildMultiInsert(tbl, cols, Rows{row}) 48 | } 49 | 50 | func BuildMultiInsert(tbl string, cols Columns, rows Rows) (Query, error) { 51 | var ( 52 | stmt string 53 | args []interface{} 54 | ) 55 | 56 | if len(cols) == 0 || len(rows) == 0 { 57 | return Query{}, errors.New("rows and cols cannot be empty") 58 | } 59 | 60 | colCount := len(cols) 61 | rowCount := len(rows) 62 | args = make([]interface{}, colCount*rowCount) 63 | argi := 0 64 | 65 | for _, row := range rows { 66 | if len(row) != colCount { 67 | return Query{}, errors.New("Amount of row items does not match column count") 68 | } 69 | for _, val := range row { 70 | args[argi] = val 71 | argi++ 72 | } 73 | } 74 | 75 | binds := strings.Repeat("?,", colCount) 76 | binds = "(" + binds[:len(binds)-1] + ")," 77 | batch := strings.Repeat(binds, rowCount) 78 | batch = batch[:len(batch)-1] 79 | 80 | stmt = fmt.Sprintf("INSERT INTO %s (%s) VALUES %s", tbl, strings.Join(cols, ","), batch) 81 | 82 | return NewQuery(stmt, args...), nil 83 | } 84 | -------------------------------------------------------------------------------- /helpers_test.go: -------------------------------------------------------------------------------- 1 | package clickhouse 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestNewHttpTransport(t *testing.T) { 9 | tr := NewHttpTransport() 10 | assert.IsType(t, HttpTransport{}, tr) 11 | } 12 | 13 | func TestNewQuery(t *testing.T) { 14 | stmt := "SELECT * FROM table WHERE ?" 15 | q := NewQuery(stmt, 1) 16 | assert.Equal(t, stmt, q.Stmt) 17 | assert.Equal(t, []interface{}{1}, q.args) 18 | } 19 | 20 | func TestBuildInsert(t *testing.T) { 21 | var ( 22 | q Query 23 | err error 24 | ) 25 | 26 | q, err = BuildInsert("test", Columns{"col1", "col2"}, Row{"val1", "val2"}) 27 | assert.Equal(t, "INSERT INTO test (col1,col2) VALUES (?,?)", q.Stmt) 28 | assert.Equal(t, []interface{}{"val1", "val2"}, q.args) 29 | assert.NoError(t, err) 30 | 31 | q, err = BuildInsert("test", Columns{"col1", "col2"}, Row{"val1"}) 32 | assert.Equal(t, "", q.Stmt) 33 | assert.Error(t, err) 34 | } 35 | 36 | func TestBuildInsertArray(t *testing.T) { 37 | var ( 38 | q Query 39 | err error 40 | ) 41 | 42 | q, err = BuildInsert("test", Columns{"col1", "col2"}, Row{"val1", Array{"val2", "val3"}}) 43 | assert.Equal(t, "INSERT INTO test (col1,col2) VALUES (?,?)", q.Stmt) 44 | assert.Equal(t, []interface{}{"val1", Array{"val2", "val3"}}, q.args) 45 | assert.NoError(t, err) 46 | } 47 | 48 | func TestNewMultiInsert(t *testing.T) { 49 | var ( 50 | q Query 51 | err error 52 | ) 53 | 54 | q, err = BuildMultiInsert("test", Columns{"col1", "col2"}, Rows{ 55 | Row{"val1", "val2"}, 56 | Row{"val3", "val4"}, 57 | }) 58 | assert.Equal(t, "INSERT INTO test (col1,col2) VALUES (?,?),(?,?)", q.Stmt) 59 | assert.Equal(t, []interface{}{"val1", "val2", "val3", "val4"}, q.args) 60 | assert.NoError(t, err) 61 | 62 | q, err = BuildMultiInsert("test", Columns{"col1", "col2"}, Rows{ 63 | Row{"val1", "val2"}, 64 | Row{"val3"}, 65 | }) 66 | assert.Equal(t, "", q.Stmt) 67 | assert.Error(t, err) 68 | 69 | //Test empty insert 70 | q, err = BuildMultiInsert("test", Columns{}, Rows{}) 71 | assert.Equal(t, "", q.Stmt) 72 | assert.Error(t, err) 73 | } 74 | 75 | func BenchmarkNewInsert(b *testing.B) { 76 | for i := 0; i < b.N; i++ { 77 | BuildInsert("test", Columns{"col1", "col2"}, Row{"val1", "val2"}) 78 | } 79 | } 80 | 81 | func getRows(n int, r Row) Rows { 82 | res := make(Rows, n) 83 | for i := 0; i < n; i++ { 84 | res[i] = r 85 | } 86 | return res 87 | } 88 | 89 | func BenchmarkNewMultiInsert100(b *testing.B) { 90 | columns := Columns{"col1", "col2", "col3", "col4"} 91 | rows := getRows(100, Row{"val1", "val2", "val3", "val4"}) 92 | b.ResetTimer() 93 | 94 | for i := 0; i < b.N; i++ { 95 | BuildMultiInsert("test", columns, rows) 96 | } 97 | } 98 | 99 | func BenchmarkNewMultiInsert1000(b *testing.B) { 100 | columns := Columns{"col1", "col2", "col3", "col4"} 101 | rows := getRows(1000, Row{"val1", "val2", "val3", "val4"}) 102 | b.ResetTimer() 103 | 104 | for i := 0; i < b.N; i++ { 105 | BuildMultiInsert("test", columns, rows) 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /marshal.go: -------------------------------------------------------------------------------- 1 | package clickhouse 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "strconv" 7 | "strings" 8 | "time" 9 | ) 10 | 11 | func escape(s string) string { 12 | s = strings.Replace(s, `\`, `\\`, -1) 13 | s = strings.Replace(s, `'`, `\'`, -1) 14 | return s 15 | } 16 | 17 | func unescape(s string) string { 18 | s = strings.Replace(s, `\\`, `\`, -1) 19 | s = strings.Replace(s, `\'`, `'`, -1) 20 | return s 21 | } 22 | 23 | func isArray(s string) bool { 24 | return strings.HasPrefix(s, "[") && strings.HasSuffix(s, "]") 25 | } 26 | 27 | func isEmptyArray(s string) bool { 28 | return s == "[]" 29 | } 30 | 31 | func splitStringToItems(s string) []string { 32 | return strings.Split(string(s[1:len(s)-1]), ",") 33 | } 34 | 35 | func unmarshal(value interface{}, data string) (err error) { 36 | var m interface{} 37 | switch v := value.(type) { 38 | case *int: 39 | *v, err = strconv.Atoi(data) 40 | return err 41 | case *int8: 42 | m, err = strconv.ParseInt(data, 10, 8) 43 | *v = int8(m.(int64)) 44 | case *int16: 45 | m, err = strconv.ParseInt(data, 10, 16) 46 | *v = int16(m.(int64)) 47 | case *int32: 48 | m, err = strconv.ParseInt(data, 10, 32) 49 | *v = int32(m.(int64)) 50 | case *int64: 51 | *v, err = strconv.ParseInt(data, 10, 64) 52 | case *float32: 53 | m, err = strconv.ParseFloat(data, 32) 54 | *v = float32(m.(float64)) 55 | case *float64: 56 | m, err = strconv.ParseFloat(data, 64) 57 | *v = m.(float64) 58 | case *string: 59 | *v = unescape(data) 60 | case *time.Time: 61 | *v, err = time.ParseInLocation("2006-01-02 15:04:05", data, time.UTC) 62 | case *[]int: 63 | if !isArray(data) { 64 | //noinspection GoPlaceholderCount 65 | return fmt.Errorf("Column data is not of type []int") 66 | } 67 | if isEmptyArray(data) { 68 | *v = []int{} 69 | return 70 | } 71 | 72 | items := splitStringToItems(data) 73 | res := make([]int, len(items)) 74 | for i := 0; i < len(items); i++ { 75 | unmarshal(&res[i], items[i]) 76 | } 77 | 78 | *v = res 79 | case *[]string: 80 | if !isArray(data) { 81 | //noinspection GoPlaceholderCount 82 | return fmt.Errorf("Column data is not of type []string") 83 | } 84 | if isEmptyArray(data) { 85 | *v = []string{} 86 | return 87 | } 88 | 89 | items := splitStringToItems(data) 90 | res := make([]string, len(items)) 91 | for i := 0; i < len(items); i++ { 92 | var s string 93 | unmarshal(&s, items[i]) 94 | res[i] = string(s[1 : len(s)-1]) 95 | } 96 | 97 | *v = res 98 | case *Array: 99 | if !isArray(data) { 100 | //noinspection GoPlaceholderCount 101 | return fmt.Errorf("Column data is not of type Array") 102 | } 103 | if isEmptyArray(data) { 104 | *v = Array{} 105 | return 106 | } 107 | 108 | items := splitStringToItems(data) 109 | res := make(Array, len(items)) 110 | 111 | var intval int 112 | err = unmarshal(&intval, items[0]) 113 | if err == nil { 114 | for i := 0; i < len(items); i++ { 115 | unmarshal(&intval, items[i]) 116 | res[i] = intval 117 | } 118 | 119 | *v = res 120 | return 121 | } 122 | 123 | var floatval float64 124 | err = unmarshal(&floatval, items[0]) 125 | if err == nil { 126 | for i := 0; i < len(items); i++ { 127 | unmarshal(&floatval, items[i]) 128 | res[i] = floatval 129 | } 130 | 131 | *v = res 132 | return 133 | } 134 | 135 | var stringval string 136 | err = unmarshal(&stringval, items[0]) 137 | if err == nil { 138 | for i := 0; i < len(items); i++ { 139 | unmarshal(&stringval, items[i]) 140 | res[i] = string(stringval[1 : len(stringval)-1]) 141 | } 142 | 143 | *v = res 144 | return 145 | } 146 | default: 147 | return fmt.Errorf("Type %T is not supported for unmarshaling", v) 148 | } 149 | 150 | return err 151 | } 152 | 153 | func marshal(value interface{}) string { 154 | if reflect.TypeOf(value).Kind() == reflect.Slice { 155 | var res []string 156 | v := reflect.ValueOf(value) 157 | for i := 0; i < v.Len(); i++ { 158 | res = append(res, marshal(v.Index(i).Interface())) 159 | } 160 | return "[" + strings.Join(res, ",") + "]" 161 | } 162 | if t := reflect.TypeOf(value); t.Kind() == reflect.Struct && strings.HasSuffix(t.String(), "Func") { 163 | return fmt.Sprintf("%s(%v)", value.(Func).Name, marshal(value.(Func).Args)) 164 | } 165 | switch v := value.(type) { 166 | case string: 167 | return fmt.Sprintf("'%s'", escape(v)) 168 | case int, int8, int16, int32, int64, 169 | uint, uint8, uint16, uint32, uint64, 170 | float32, float64: 171 | return fmt.Sprintf("%v", v) 172 | //https://clickhouse.yandex/reference_en.html#Boolean values 173 | case bool: 174 | if value.(bool) { 175 | return "1" 176 | } 177 | return "0" 178 | //Convert time to Date type https://clickhouse.yandex/reference_en.html#Date 179 | case time.Time: 180 | return value.(time.Time).Format("2006-01-02") 181 | } 182 | 183 | return "''" 184 | } 185 | -------------------------------------------------------------------------------- /marshal_test.go: -------------------------------------------------------------------------------- 1 | package clickhouse 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func BenchmarkMarshalString(b *testing.B) { 10 | for i := 0; i < b.N; i++ { 11 | marshal("test") 12 | } 13 | } 14 | 15 | func TestUnmarshal(t *testing.T) { 16 | var ( 17 | err error 18 | valInt int 19 | valInt8 int8 20 | valInt16 int16 21 | valInt32 int32 22 | valInt64 int64 23 | valString string 24 | valTime time.Time 25 | valUnsupported testing.T 26 | valFloat32 float32 27 | valFloat64 float64 28 | valArrayString []string 29 | valArrayInt []int 30 | valArray Array 31 | ) 32 | 33 | err = unmarshal(&valInt, "10") 34 | assert.Equal(t, int(10), valInt) 35 | assert.NoError(t, err) 36 | 37 | err = unmarshal(&valInt8, "10") 38 | assert.Equal(t, int8(10), valInt8) 39 | assert.NoError(t, err) 40 | 41 | err = unmarshal(&valInt16, "10") 42 | assert.Equal(t, int16(10), valInt16) 43 | assert.NoError(t, err) 44 | 45 | err = unmarshal(&valInt32, "10") 46 | assert.Equal(t, int32(10), valInt32) 47 | assert.NoError(t, err) 48 | 49 | err = unmarshal(&valInt64, "10") 50 | assert.Equal(t, int64(10), valInt64) 51 | assert.NoError(t, err) 52 | 53 | err = unmarshal(&valString, "10") 54 | assert.Equal(t, "10", valString) 55 | assert.NoError(t, err) 56 | 57 | err = unmarshal(&valString, "String1\\'") 58 | assert.Equal(t, "String1'", valString) 59 | assert.NoError(t, err) 60 | 61 | err = unmarshal(&valTime, "2016-10-07 19:21:17") 62 | assert.Equal(t, time.Date(2016, 10, 7, 19, 21, 17, 0, time.UTC), valTime) 63 | assert.NoError(t, err) 64 | 65 | err = unmarshal(&valUnsupported, "10") 66 | assert.Error(t, err) 67 | 68 | err = unmarshal(&valFloat32, "3.141592") 69 | assert.Equal(t, float32(3.141592), valFloat32) 70 | assert.NoError(t, err) 71 | 72 | err = unmarshal(&valFloat64, "3.1415926535") 73 | assert.Equal(t, float64(3.1415926535), valFloat64) 74 | assert.NoError(t, err) 75 | 76 | err = unmarshal(&valArrayString, "['k10','20']") 77 | assert.Equal(t, []string{"k10", "20"}, valArrayString) 78 | assert.NoError(t, err) 79 | 80 | err = unmarshal(&valArrayString, "") 81 | assert.Error(t, err, "Column data is not of type []string") 82 | 83 | err = unmarshal(&valArrayString, "[]") 84 | assert.Equal(t, []string{}, valArrayString) 85 | assert.NoError(t, err) 86 | 87 | err = unmarshal(&valArrayInt, "[10,20]") 88 | assert.Equal(t, []int{10, 20}, valArrayInt) 89 | assert.NoError(t, err) 90 | 91 | err = unmarshal(&valArrayInt, "") 92 | assert.Error(t, err, "Column data is not of type []int") 93 | 94 | err = unmarshal(&valArrayInt, "[]") 95 | assert.Equal(t, []int{}, valArrayInt) 96 | assert.NoError(t, err) 97 | 98 | err = unmarshal(&valArray, "['k10','20']") 99 | assert.Equal(t, Array{"k10", "20"}, valArray) 100 | assert.NoError(t, err) 101 | 102 | err = unmarshal(&valArray, "[10,20]") 103 | assert.Equal(t, Array{10, 20}, valArray) 104 | assert.NoError(t, err) 105 | 106 | err = unmarshal(&valArray, "[3.14,5.25]") 107 | assert.Equal(t, Array{3.14, 5.25}, valArray) 108 | assert.NoError(t, err) 109 | 110 | err = unmarshal(&valArray, "") 111 | assert.Error(t, err, "Column data is not of type Array") 112 | 113 | err = unmarshal(&valArray, "[]") 114 | assert.Equal(t, Array{}, valArray) 115 | assert.NoError(t, err) 116 | } 117 | 118 | func TestMarshal(t *testing.T) { 119 | assert.Equal(t, "10", marshal(10)) 120 | assert.Equal(t, "10", marshal(int8(10))) 121 | assert.Equal(t, "10", marshal(int16(10))) 122 | assert.Equal(t, "10", marshal(int32(10))) 123 | assert.Equal(t, "10", marshal(int64(10))) 124 | 125 | assert.Equal(t, "1", marshal(true)) 126 | assert.Equal(t, "0", marshal(false)) 127 | 128 | assert.Equal(t, "3.141592", marshal(float32(3.141592))) 129 | assert.Equal(t, "3.1415926535", marshal(float64(3.1415926535))) 130 | 131 | assert.Equal(t, "'10'", marshal("10")) 132 | assert.Equal(t, "'String1\\''", marshal("String1'")) 133 | assert.Equal(t, "'String\r'", marshal("String\r")) 134 | assert.Equal(t, "'String\r'", marshal("String\r")) 135 | assert.Equal(t, `'String\\'`, marshal(`String\`)) 136 | assert.Equal(t, "[10,20,30]", marshal(Array{10, 20, 30})) 137 | assert.Equal(t, "['k10','20','30val']", marshal(Array{"k10", "20", "30val"})) 138 | assert.Equal(t, "['k10','20','30val']", marshal([]string{"k10", "20", "30val"})) 139 | assert.Equal(t, "['k10','20','30val\\\\']", marshal([]string{"k10", "20", "30val\\"})) 140 | assert.Equal(t, "[10,20,30]", marshal([]int{10, 20, 30})) 141 | assert.Equal(t, "IPv4StringToNum('192.0.2.128')", marshal(Func{"IPv4StringToNum", "192.0.2.128"})) 142 | assert.Equal(t, "IPv4NumToString(3221225985)", marshal(Func{"IPv4NumToString", 3221225985})) 143 | assert.Equal(t, "''", marshal(t)) 144 | assert.Equal(t, "2017-04-10", marshal(time.Date(2017, 04, 10, 0, 0, 0, 0, time.UTC))) 145 | } -------------------------------------------------------------------------------- /query.go: -------------------------------------------------------------------------------- 1 | package clickhouse 2 | 3 | import ( 4 | "errors" 5 | "strings" 6 | ) 7 | 8 | type External struct { 9 | Name string 10 | Structure string 11 | Data []byte 12 | } 13 | 14 | type Func struct { 15 | Name string 16 | Args interface{} 17 | } 18 | 19 | type Query struct { 20 | Stmt string 21 | args []interface{} 22 | externals []External 23 | } 24 | 25 | func (q *Query) AddExternal(name string, structure string, data []byte) { 26 | q.externals = append(q.externals, External{Name: name, Structure: structure, Data: data}) 27 | } 28 | 29 | func (q Query) Iter(conn *Conn) *Iter { 30 | if conn == nil { 31 | return &Iter{err: errors.New("Connection pointer is nil")} 32 | } 33 | resp, err := conn.transport.Exec(conn, q, false) 34 | if err != nil { 35 | return &Iter{err: err} 36 | } 37 | 38 | err = errorFromResponse(resp) 39 | if err != nil { 40 | return &Iter{err: err} 41 | } 42 | 43 | return &Iter{text: resp} 44 | } 45 | 46 | func (q Query) Exec(conn *Conn) (err error) { 47 | if conn == nil { 48 | return errors.New("Connection pointer is nil") 49 | } 50 | resp, err := conn.transport.Exec(conn, q, false) 51 | if err == nil { 52 | err = errorFromResponse(resp) 53 | } 54 | 55 | return err 56 | } 57 | 58 | type Iter struct { 59 | err error 60 | text string 61 | } 62 | 63 | func (r *Iter) Error() error { 64 | return r.err 65 | } 66 | 67 | func (r *Iter) Scan(vars ...interface{}) bool { 68 | row := r.fetchNext() 69 | if len(row) == 0 { 70 | return false 71 | } 72 | a := strings.Split(row, "\t") 73 | if len(a) < len(vars) { 74 | return false 75 | } 76 | for i, v := range vars { 77 | err := unmarshal(v, a[i]) 78 | if err != nil { 79 | r.err = err 80 | return false 81 | } 82 | } 83 | return true 84 | } 85 | 86 | func (r *Iter) fetchNext() string { 87 | var res string 88 | pos := strings.Index(r.text, "\n") 89 | if pos == -1 { 90 | res = r.text 91 | r.text = "" 92 | } else { 93 | res = r.text[:pos] 94 | r.text = r.text[pos+1:] 95 | } 96 | return res 97 | } 98 | -------------------------------------------------------------------------------- /query_test.go: -------------------------------------------------------------------------------- 1 | package clickhouse 2 | 3 | import ( 4 | "errors" 5 | "github.com/stretchr/testify/assert" 6 | "testing" 7 | ) 8 | 9 | type mockTransport struct { 10 | response string 11 | } 12 | 13 | type badTransport struct { 14 | response string 15 | err error 16 | } 17 | 18 | func (m mockTransport) Exec(conn *Conn, q Query, readOnly bool) (r string, err error) { 19 | return m.response, nil 20 | } 21 | 22 | func (m badTransport) Exec(conn *Conn, q Query, readOnly bool) (r string, err error) { 23 | return "", m.err 24 | } 25 | 26 | func TestQuery_Iter(t *testing.T) { 27 | tr := getMockTransport("Code: 62, ") 28 | conn := NewConn(getHost(), tr) 29 | iter := NewQuery("SELECT 1").Iter(conn) 30 | assert.Error(t, iter.Error()) 31 | assert.Equal(t, 62, iter.Error().(*DbError).Code()) 32 | } 33 | 34 | func TestQuery_Iter2(t *testing.T) { 35 | tr := badTransport{err: errors.New("No connection")} 36 | conn := NewConn(getHost(), tr) 37 | iter := NewQuery("SELECT 1").Iter(conn) 38 | assert.Error(t, iter.Error()) 39 | assert.Equal(t, "No connection", iter.Error().Error()) 40 | } 41 | 42 | func TestIter_ScanInt(t *testing.T) { 43 | tr := getMockTransport("1\t2") 44 | conn := NewConn(getHost(), tr) 45 | 46 | iter := NewQuery("SELECT 1, 2").Iter(conn) 47 | var v1, v2 int 48 | scan := iter.Scan(&v1, &v2) 49 | assert.True(t, scan) 50 | if scan { 51 | assert.Equal(t, 1, v1) 52 | assert.Equal(t, 2, v2) 53 | } 54 | } 55 | 56 | func TestIter_ScanInt64(t *testing.T) { 57 | tr := getMockTransport("1\t2") 58 | conn := NewConn(getHost(), tr) 59 | 60 | iter := NewQuery("SELECT 1, 2").Iter(conn) 61 | var v1, v2 int64 62 | scan := iter.Scan(&v1, &v2) 63 | assert.True(t, scan) 64 | if scan { 65 | assert.Equal(t, int64(1), v1) 66 | assert.Equal(t, int64(2), v2) 67 | } 68 | } 69 | 70 | func TestIter_ScanString(t *testing.T) { 71 | tr := getMockTransport("test1\ttest2") 72 | conn := NewConn(getHost(), tr) 73 | 74 | iter := NewQuery("SELECT 'test1', 'test2'").Iter(conn) 75 | var v1, v2 string 76 | scan := iter.Scan(&v1, &v2) 77 | assert.True(t, scan) 78 | if scan { 79 | assert.Equal(t, "test1", v1) 80 | assert.Equal(t, "test2", v2) 81 | } 82 | } 83 | 84 | func TestIter_ScanStringMultiple(t *testing.T) { 85 | tr := getMockTransport("test1\ttest2\ntest3\ttest4") 86 | conn := NewConn(getHost(), tr) 87 | 88 | iter := NewQuery("SELECT 'test1', 'test2'").Iter(conn) 89 | var v1, v2 string 90 | scan := iter.Scan(&v1, &v2) 91 | assert.True(t, scan) 92 | if scan { 93 | assert.Equal(t, "test1", v1) 94 | assert.Equal(t, "test2", v2) 95 | } 96 | 97 | scan = iter.Scan(&v1, &v2) 98 | assert.True(t, scan) 99 | if scan { 100 | assert.Equal(t, "test3", v1) 101 | assert.Equal(t, "test4", v2) 102 | } 103 | } 104 | 105 | func TestIter_ScanErrors(t *testing.T) { 106 | tr := getMockTransport("test1\ttest2\ntest3\ttest4") 107 | conn := NewConn(getHost(), tr) 108 | 109 | iter := NewQuery("SELECT 'test1', 'test2'").Iter(conn) 110 | var v1, v2, v3 string 111 | scan := iter.Scan(&v1, &v2, &v3) 112 | assert.False(t, scan) 113 | assert.NoError(t, iter.Error()) 114 | 115 | var u1 Conn 116 | scan = iter.Scan(&u1) 117 | assert.False(t, scan) 118 | assert.Error(t, iter.Error()) 119 | 120 | tr = getMockTransport("") 121 | conn = NewConn(getHost(), tr) 122 | 123 | iter = NewQuery("SELECT 'test1', 'test2'").Iter(conn) 124 | scan = iter.Scan(&u1) 125 | assert.False(t, scan) 126 | assert.NoError(t, iter.Error()) 127 | } 128 | 129 | func TestQuery_Exec(t *testing.T) { 130 | tr := getMockTransport("") 131 | conn := NewConn(getHost(), tr) 132 | 133 | err := NewQuery("INSERT INTO table VALUES 1").Exec(conn) 134 | assert.NoError(t, err) 135 | 136 | tr = getMockTransport("Code: 69, ") 137 | conn = NewConn(getHost(), tr) 138 | 139 | err = NewQuery("INSERT INTO table VALUES 1").Exec(conn) 140 | assert.Error(t, err) 141 | assert.Equal(t, 69, err.(*DbError).Code()) 142 | } 143 | 144 | func TestQuery_Exec2(t *testing.T) { 145 | err := NewQuery("SELECT 1").Exec(nil) 146 | assert.Error(t, err) 147 | } 148 | 149 | func TestQuery_Iter3(t *testing.T) { 150 | iter := NewQuery("INSERT 1").Iter(nil) 151 | assert.Error(t, iter.err) 152 | } 153 | 154 | func getMockTransport(resp string) mockTransport { 155 | tr := mockTransport{} 156 | tr.response = resp 157 | return tr 158 | } 159 | 160 | func getHost() string { 161 | return "host.local" 162 | } 163 | -------------------------------------------------------------------------------- /transport.go: -------------------------------------------------------------------------------- 1 | package clickhouse 2 | 3 | import ( 4 | "bytes" 5 | "mime/multipart" 6 | "net/http" 7 | "net/url" 8 | "strings" 9 | "time" 10 | ) 11 | 12 | const ( 13 | httpTransportBodyType = "text/plain" 14 | ) 15 | 16 | type Transport interface { 17 | Exec(conn *Conn, q Query, readOnly bool) (res string, err error) 18 | } 19 | 20 | type HttpTransport struct { 21 | Timeout time.Duration 22 | } 23 | 24 | func (t HttpTransport) Exec(conn *Conn, q Query, readOnly bool) (res string, err error) { 25 | var req *http.Request 26 | var resp *http.Response 27 | query := prepareHttp(q.Stmt, q.args) 28 | client := &http.Client{Timeout: t.Timeout} 29 | if readOnly { 30 | if len(query) > 0 { 31 | query = "?query=" + query 32 | } 33 | req, err = http.NewRequest("GET", conn.Host+query, nil) 34 | if err != nil { 35 | return "", err 36 | } 37 | } else { 38 | req, err = prepareExecPostRequest(conn.Host, q) 39 | if err != nil { 40 | return "", err 41 | } 42 | } 43 | 44 | if len(conn.User) > 0 { 45 | req.Header.Set("X-ClickHouse-User", conn.User) 46 | } 47 | if len(conn.Password) > 0 { 48 | req.Header.Set("X-ClickHouse-Key", conn.Password) 49 | } 50 | 51 | resp, err = client.Do(req) 52 | if err != nil { 53 | return "", err 54 | } 55 | defer resp.Body.Close() 56 | buf := new(bytes.Buffer) 57 | _, err = buf.ReadFrom(resp.Body) 58 | 59 | return buf.String(), err 60 | } 61 | 62 | func prepareExecPostRequest(host string, q Query) (*http.Request, error) { 63 | query := prepareHttp(q.Stmt, q.args) 64 | var req *http.Request 65 | var err error = nil 66 | if len(q.externals) > 0 { 67 | if len(query) > 0 { 68 | query = "?query=" + url.QueryEscape(query) 69 | } 70 | 71 | body := &bytes.Buffer{} 72 | writer := multipart.NewWriter(body) 73 | 74 | for _, ext := range q.externals { 75 | query = query + "&" + ext.Name + "_structure=" + url.QueryEscape(ext.Structure) 76 | part, err := writer.CreateFormFile(ext.Name, ext.Name) 77 | if err != nil { 78 | return nil, err 79 | } 80 | _, err = part.Write(ext.Data) 81 | if err != nil { 82 | return nil, err 83 | } 84 | } 85 | 86 | err = writer.Close() 87 | if err != nil { 88 | return nil, err 89 | } 90 | 91 | req, err = http.NewRequest("POST", host+query, body) 92 | if err != nil { 93 | return nil, err 94 | } 95 | req.Header.Set("Content-Type", writer.FormDataContentType()) 96 | } else { 97 | req, err = http.NewRequest("POST", host, strings.NewReader(query)) 98 | if err != nil { 99 | return nil, err 100 | } 101 | req.Header.Set("Content-Type", httpTransportBodyType) 102 | } 103 | return req, err 104 | } 105 | 106 | func prepareHttp(stmt string, args []interface{}) string { 107 | var res []byte 108 | buf := []byte(stmt) 109 | res = make([]byte, 0) 110 | k := 0 111 | for _, ch := range buf { 112 | if ch == '?' { 113 | res = append(res, []byte(marshal(args[k]))...) 114 | k++ 115 | } else { 116 | res = append(res, ch) 117 | } 118 | } 119 | 120 | return string(res) 121 | } 122 | -------------------------------------------------------------------------------- /transport_test.go: -------------------------------------------------------------------------------- 1 | package clickhouse 2 | 3 | import ( 4 | "fmt" 5 | "github.com/stretchr/testify/assert" 6 | "io/ioutil" 7 | 8 | "mime" 9 | "mime/multipart" 10 | "net/http" 11 | "net/http/httptest" 12 | "net/url" 13 | "strings" 14 | "testing" 15 | ) 16 | 17 | type TestHandler struct { 18 | Result string 19 | } 20 | 21 | func (h *TestHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 22 | w.Header().Add("Content-Type", "text/tab-separated-values; charset=UTF-8") 23 | fmt.Fprint(w, h.Result) 24 | } 25 | 26 | func TestExec(t *testing.T) { 27 | handler := &TestHandler{Result: "1 2.5 clickid68235\n2 -0.14 clickidsdkjhj44"} 28 | server := httptest.NewServer(handler) 29 | defer server.Close() 30 | 31 | transport := HttpTransport{} 32 | conn := Conn{Host: server.URL, transport: transport} 33 | q := NewQuery("SELECT * FROM testdata") 34 | resp, err := transport.Exec(&conn, q, false) 35 | assert.Equal(t, nil, err) 36 | assert.Equal(t, handler.Result, resp) 37 | 38 | } 39 | 40 | func TestExecReadOnly(t *testing.T) { 41 | handler := &TestHandler{Result: "1 2.5 clickid68235\n2 -0.14 clickidsdkjhj44"} 42 | server := httptest.NewServer(handler) 43 | defer server.Close() 44 | 45 | transport := HttpTransport{} 46 | conn := Conn{Host: server.URL, transport: transport} 47 | q := NewQuery(url.QueryEscape("SELECT * FROM testdata")) 48 | query := prepareHttp(q.Stmt, q.args) 49 | query = "?query=" + url.QueryEscape(query) 50 | resp, err := transport.Exec(&conn, q, true) 51 | assert.Equal(t, nil, err) 52 | assert.Equal(t, handler.Result, resp) 53 | 54 | } 55 | 56 | func TestPrepareHttp(t *testing.T) { 57 | p := prepareHttp("SELECT * FROM table WHERE key = ?", []interface{}{"test"}) 58 | assert.Equal(t, "SELECT * FROM table WHERE key = 'test'", p) 59 | } 60 | 61 | func TestPrepareHttpArray(t *testing.T) { 62 | p := prepareHttp("INSERT INTO table (arr) VALUES (?)", Row{Array{"val1", "val2"}}) 63 | assert.Equal(t, "INSERT INTO table (arr) VALUES (['val1','val2'])", p) 64 | } 65 | 66 | func TestPrepareExecPostRequest(t *testing.T) { 67 | q := NewQuery("SELECT * FROM testdata") 68 | req, err := prepareExecPostRequest("http://127.0.0.0:8123/", q) 69 | assert.Equal(t, nil, err) 70 | data, err := ioutil.ReadAll(req.Body) 71 | assert.Equal(t, nil, err) 72 | assert.Equal(t, "SELECT * FROM testdata", string(data)) 73 | } 74 | 75 | func TestPrepareExecPostRequestWithExternalData(t *testing.T) { 76 | q := NewQuery("SELECT * FROM testdata") 77 | q.AddExternal("data1", "ID String, Num UInt32", []byte("Hello\t22\nHi\t44")) 78 | q.AddExternal("extdata", "Num UInt32, Name String", []byte("1 first\n2 second")) 79 | 80 | req, err := prepareExecPostRequest("http://127.0.0.0:8123/", q) 81 | assert.Equal(t, nil, err) 82 | assert.Equal(t, "SELECT * FROM testdata", req.URL.Query().Get("query")) 83 | assert.Equal(t, "ID String, Num UInt32", req.URL.Query().Get("data1_structure")) 84 | assert.Equal(t, "Num UInt32, Name String", req.URL.Query().Get("extdata_structure")) 85 | 86 | mediaType, params, err := mime.ParseMediaType(req.Header.Get("Content-Type")) 87 | assert.Equal(t, nil, err) 88 | assert.Equal(t, true, strings.HasPrefix(mediaType, "multipart/")) 89 | 90 | reader := multipart.NewReader(req.Body, params["boundary"]) 91 | 92 | p, err := reader.NextPart() 93 | assert.Equal(t, nil, err) 94 | 95 | data, err := ioutil.ReadAll(p) 96 | assert.Equal(t, nil, err) 97 | assert.Equal(t, "Hello\t22\nHi\t44", string(data)) 98 | 99 | p, err = reader.NextPart() 100 | assert.Equal(t, nil, err) 101 | 102 | data, err = ioutil.ReadAll(p) 103 | assert.Equal(t, nil, err) 104 | assert.Equal(t, "1\tfirst\n2\tsecond", string(data)) 105 | } 106 | 107 | func BenchmarkPrepareHttp(b *testing.B) { 108 | params := strings.Repeat("(?,?,?,?,?,?,?,?)", 1000) 109 | args := make([]interface{}, 8000) 110 | for i := 0; i < 8000; i++ { 111 | args[i] = "test" 112 | } 113 | b.ResetTimer() 114 | 115 | for i := 0; i < b.N; i++ { 116 | prepareHttp("INSERT INTO t VALUES "+params, args) 117 | } 118 | } 119 | --------------------------------------------------------------------------------