├── column ├── tuples_template │ ├── tuple2.json │ ├── tuple3.json │ ├── tuple4.json │ ├── tuple5.json │ └── tuple.go.tmpl ├── point.go ├── nested.go ├── errors.go ├── string.go ├── helper_test.go ├── lc_indices.go ├── base_little_cpu.go ├── tuple1.go ├── base_big_cpu.go ├── size.go ├── array2.go ├── array3.go ├── bench_test.go ├── array.go ├── tuple2_gen.go ├── map_nullable.go ├── array3_nullable.go ├── column_helper.go ├── map.go ├── array2_nullable.go ├── tuple3_gen.go ├── array_nullable.go ├── tuple4_gen.go ├── lc_nullable.go ├── lc_test.go ├── tuple5_gen.go ├── base.go ├── nested_test.go ├── tuple.go ├── date.go ├── array_base.go ├── base_validate.go └── nullable_test.go ├── .gitignore ├── .github ├── dependabot.yml └── workflows │ ├── lint.yaml │ └── ci.yaml ├── types ├── ipv6.go ├── uuid_test.go ├── ipv4.go ├── decimal_test.go ├── tuple.go ├── ip_test.go ├── uuid.go ├── int256_test.go ├── int128_test.go ├── uint256_test.go ├── uint128_test.go ├── decimal.go ├── date_type.go ├── uint128.go ├── uint256.go ├── int128.go └── Int256.go ├── .codecov.yml ├── doc.go ├── chpool ├── insert_stmt.go ├── select_stmt.go ├── stat.go ├── conn.go └── common_test.go ├── go.mod ├── profile_event.go ├── helper_test.go ├── internal ├── helper │ ├── features.go │ ├── strs.go │ └── validator.go ├── readerwriter │ ├── consts.go │ ├── writer.go │ ├── reader.go │ ├── compress_writer.go │ └── compress_reader.go └── ctxwatch │ ├── context_watcher.go │ └── context_watcher_test.go ├── ping.go ├── LICENSE ├── profile.go ├── sqlbuilder ├── injection.go └── select_test.go ├── progress.go ├── server_info_test.go ├── server_info.go ├── block_test.go ├── client_info.go ├── go.sum ├── ping_test.go ├── profile_test.go ├── doc_test.go ├── .golangci.yml ├── errors_test.go └── settings.go /column/tuples_template/tuple2.json: -------------------------------------------------------------------------------- 1 | { 2 | "Numbrer": "2" 3 | } -------------------------------------------------------------------------------- /column/tuples_template/tuple3.json: -------------------------------------------------------------------------------- 1 | { 2 | "Numbrer": "3" 3 | } -------------------------------------------------------------------------------- /column/tuples_template/tuple4.json: -------------------------------------------------------------------------------- 1 | { 2 | "Numbrer": "4" 3 | } -------------------------------------------------------------------------------- /column/tuples_template/tuple5.json: -------------------------------------------------------------------------------- 1 | { 2 | "Numbrer": "5" 3 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .envrc 2 | bin/ 3 | vendor/ 4 | build/ 5 | coverage.out 6 | -------------------------------------------------------------------------------- /column/point.go: -------------------------------------------------------------------------------- 1 | package column 2 | 3 | import "github.com/vahid-sohrabloo/chconn/v2/types" 4 | 5 | func NewPoint() *Tuple2[types.Point, float64, float64] { 6 | return NewTuple2[types.Point, float64, float64](New[float64](), New[float64]()) 7 | } 8 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: gomod 4 | directory: "/" 5 | schedule: 6 | interval: daily 7 | - package-ecosystem: github-actions 8 | directory: "/" 9 | schedule: 10 | interval: daily -------------------------------------------------------------------------------- /types/ipv6.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import "net/netip" 4 | 5 | type IPv6 [16]byte 6 | 7 | func (ip IPv6) NetIP() netip.Addr { 8 | return netip.AddrFrom16(ip) 9 | } 10 | 11 | func IPv6FromAddr(ipAddr netip.Addr) IPv6 { 12 | return IPv6(ipAddr.As16()) 13 | } 14 | -------------------------------------------------------------------------------- /column/nested.go: -------------------------------------------------------------------------------- 1 | package column 2 | 3 | // NewNested create a new nested of Nested(T1,T2,.....,Tn) ClickHouse data type 4 | // 5 | // this is actually an alias for NewTuple(T1,T2,.....,Tn).Array() 6 | func NewNested(columns ...ColumnBasic) *ArrayBase { 7 | return NewTuple(columns...).Array() 8 | } 9 | -------------------------------------------------------------------------------- /types/uuid_test.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/google/uuid" 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestUUID(t *testing.T) { 11 | u := uuid.New() 12 | uuidData := UUIDFromBigEndian(u) 13 | assert.Equal(t, uuidData.BigEndian(), [16]byte(u)) 14 | } 15 | -------------------------------------------------------------------------------- /.codecov.yml: -------------------------------------------------------------------------------- 1 | ignore: 2 | - "**/main.go" 3 | - "./internal/readerwriter/*" 4 | coverage: 5 | status: 6 | project: 7 | default: 8 | target: 50% 9 | threshold: null 10 | patch: false 11 | changes: false 12 | range: 70..95 13 | round: up 14 | precision: 1 15 | 16 | 17 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | // Package chconn is a low-level Clickhouse database driver. 2 | /* 3 | chconn is a pure Go driver for [ClickHouse] that use Native protocol 4 | chconn aims to be low-level, fast, and performant. 5 | 6 | If you have any suggestion or comment, please feel free to open an issue on this tutorial's GitHub page! 7 | */ 8 | package chconn 9 | -------------------------------------------------------------------------------- /column/errors.go: -------------------------------------------------------------------------------- 1 | package column 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | type ErrInvalidType struct { 8 | column ColumnBasic 9 | ColumnType string 10 | } 11 | 12 | func (e ErrInvalidType) Error() string { 13 | return fmt.Sprintf("mismatch column type: ClickHouse Type: %s, column types: %s", 14 | string(e.column.Type()), 15 | e.column.ColumnType(), 16 | ) 17 | } 18 | -------------------------------------------------------------------------------- /types/ipv4.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import "net/netip" 4 | 5 | // IPv4 is a compatible type for IPv4 address in clickhouse. 6 | // 7 | // clickhouse use Little endian for IPv4. but golang use big endian 8 | type IPv4 [4]byte 9 | 10 | func (ip IPv4) NetIP() netip.Addr { 11 | return netip.AddrFrom4([4]byte{ip[3], ip[2], ip[1], ip[0]}) 12 | } 13 | 14 | func IPv4FromAddr(ipAddr netip.Addr) IPv4 { 15 | ip := ipAddr.As4() 16 | return IPv4{ip[3], ip[2], ip[1], ip[0]} 17 | } 18 | -------------------------------------------------------------------------------- /types/decimal_test.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestDecimal(t *testing.T) { 10 | d32 := Decimal32(12_234) 11 | assert.Equal(t, d32.Float64(3), float64(12.234)) 12 | d64 := Decimal64(12_234) 13 | assert.Equal(t, d64.Float64(3), float64(12.234)) 14 | assert.Equal(t, Decimal32FromFloat64(12.2334, 3), Decimal32(12233)) 15 | assert.Equal(t, Decimal64FromFloat64(12.2334, 3), Decimal64(12233)) 16 | } 17 | -------------------------------------------------------------------------------- /types/tuple.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | type Point Tuple2[float64, float64] 4 | 5 | type Tuple2[T1, T2 any] struct { 6 | Col1 T1 7 | Col2 T2 8 | } 9 | 10 | type Tuple3[T1, T2, T3 any] struct { 11 | Col1 T1 12 | Col2 T2 13 | Col3 T3 14 | } 15 | 16 | type Tuple4[T1, T2, T3, T4 any] struct { 17 | Col1 T1 18 | Col2 T2 19 | Col3 T3 20 | Col4 T4 21 | } 22 | 23 | type Tuple5[T1, T2, T3, T4, T5 any] struct { 24 | Col1 T1 25 | Col2 T2 26 | Col3 T3 27 | Col4 T4 28 | Col5 T5 29 | } 30 | -------------------------------------------------------------------------------- /types/ip_test.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "net/netip" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestIP(t *testing.T) { 11 | ipv4 := IPv4FromAddr(netip.AddrFrom4([4]byte{1, 2, 3, 4})) 12 | assert.Equal(t, ipv4.NetIP().As4(), [4]byte{1, 2, 3, 4}) 13 | ipv6 := IPv6FromAddr(netip.AddrFrom16([16]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16})) 14 | assert.Equal(t, ipv6.NetIP().As16(), [16]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}) 15 | } 16 | -------------------------------------------------------------------------------- /types/uuid.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | type UUID [16]byte 4 | 5 | func UUIDFromBigEndian(b [16]byte) UUID { 6 | var val [16]byte 7 | val[0], val[7] = b[7], b[0] 8 | val[1], val[6] = b[6], b[1] 9 | val[2], val[5] = b[5], b[2] 10 | val[3], val[4] = b[4], b[3] 11 | val[8], val[15] = b[15], b[8] 12 | val[9], val[14] = b[14], b[9] 13 | val[10], val[13] = b[13], b[10] 14 | val[11], val[12] = b[12], b[11] 15 | return val 16 | } 17 | 18 | func (u UUID) BigEndian() [16]byte { 19 | return UUIDFromBigEndian(u) 20 | } 21 | -------------------------------------------------------------------------------- /chpool/insert_stmt.go: -------------------------------------------------------------------------------- 1 | package chpool 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/vahid-sohrabloo/chconn/v2" 7 | ) 8 | 9 | type insertStmt struct { 10 | chconn.InsertStmt 11 | conn Conn 12 | } 13 | 14 | func (s *insertStmt) Flush(ctx context.Context) error { 15 | if s.conn == nil { 16 | return nil 17 | } 18 | defer s.conn.Release() 19 | return s.InsertStmt.Flush(ctx) 20 | } 21 | 22 | func (s *insertStmt) Close() { 23 | if s.conn == nil { 24 | return 25 | } 26 | s.InsertStmt.Close() 27 | s.conn.Release() 28 | } 29 | -------------------------------------------------------------------------------- /.github/workflows/lint.yaml: -------------------------------------------------------------------------------- 1 | name: golangci-lint 2 | on: 3 | push: 4 | branches: 5 | - main 6 | pull_request: 7 | 8 | jobs: 9 | lint: 10 | name: lint 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: Set up Go 14 | uses: actions/setup-go@v3 15 | with: 16 | go-version: 1.19 17 | - name: Checkout code 18 | uses: actions/checkout@v3.3.0 19 | - name: golangci-lint 20 | uses: golangci/golangci-lint-action@v3 21 | with: 22 | version: v1.50 23 | args: --timeout=10m -------------------------------------------------------------------------------- /column/string.go: -------------------------------------------------------------------------------- 1 | package column 2 | 3 | // String is a column of String ClickHouse data type 4 | type String struct { 5 | StringBase[string] 6 | } 7 | 8 | // NewString is a column of String ClickHouse data type 9 | func NewString() *String { 10 | return &String{} 11 | } 12 | 13 | func (c *String) Elem(arrayLevel int, nullable, lc bool) ColumnBasic { 14 | if nullable { 15 | return c.Nullable().elem(arrayLevel, lc) 16 | } 17 | if lc { 18 | return c.LowCardinality().elem(arrayLevel) 19 | } 20 | if arrayLevel > 0 { 21 | return c.Array().elem(arrayLevel - 1) 22 | } 23 | return c 24 | } 25 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/vahid-sohrabloo/chconn/v2 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/go-faster/city v1.0.1 7 | github.com/google/uuid v1.3.0 8 | github.com/jackc/puddle/v2 v2.1.2 9 | github.com/klauspost/compress v1.15.15 10 | github.com/pierrec/lz4/v4 v4.1.17 11 | github.com/stretchr/testify v1.8.1 12 | ) 13 | 14 | require ( 15 | github.com/davecgh/go-spew v1.1.1 // indirect 16 | github.com/pmezard/go-difflib v1.0.0 // indirect 17 | go.uber.org/atomic v1.10.0 // indirect 18 | golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7 // indirect 19 | gopkg.in/yaml.v3 v3.0.1 // indirect 20 | ) 21 | -------------------------------------------------------------------------------- /chpool/select_stmt.go: -------------------------------------------------------------------------------- 1 | package chpool 2 | 3 | import ( 4 | "github.com/vahid-sohrabloo/chconn/v2" 5 | ) 6 | 7 | type selectStmt struct { 8 | chconn.SelectStmt 9 | conn Conn 10 | } 11 | 12 | func (s *selectStmt) Next() bool { 13 | if s.conn == nil { 14 | return false 15 | } 16 | next := s.SelectStmt.Next() 17 | if s.SelectStmt.Err() != nil && s.conn != nil { 18 | s.conn.Release() 19 | s.conn = nil 20 | } 21 | if !next && s.conn != nil { 22 | s.conn.Release() 23 | s.conn = nil 24 | } 25 | return next 26 | } 27 | 28 | func (s *selectStmt) Close() { 29 | if s.conn == nil { 30 | return 31 | } 32 | s.SelectStmt.Close() 33 | s.conn.Release() 34 | } 35 | -------------------------------------------------------------------------------- /column/helper_test.go: -------------------------------------------------------------------------------- 1 | package column_test 2 | 3 | import ( 4 | "io" 5 | ) 6 | 7 | type readErrorHelper struct { 8 | numberValid int 9 | err error 10 | r io.Reader 11 | count int 12 | } 13 | 14 | func (r *readErrorHelper) Read(p []byte) (int, error) { 15 | r.count++ 16 | if r.count > r.numberValid { 17 | return 0, r.err 18 | } 19 | return r.r.Read(p) 20 | } 21 | 22 | type writerErrorHelper struct { 23 | numberValid int 24 | err error 25 | w io.Writer 26 | count int 27 | } 28 | 29 | func (w *writerErrorHelper) Write(p []byte) (int, error) { 30 | w.count++ 31 | if w.count > w.numberValid { 32 | return 0, w.err 33 | } 34 | return w.w.Write(p) 35 | } 36 | -------------------------------------------------------------------------------- /profile_event.go: -------------------------------------------------------------------------------- 1 | package chconn 2 | 3 | import ( 4 | "github.com/vahid-sohrabloo/chconn/v2/column" 5 | ) 6 | 7 | // Profile detail of profile select query 8 | type ProfileEvent struct { 9 | Host *column.String 10 | Time *column.Base[uint32] 11 | ThreadID *column.Base[uint64] 12 | Type *column.Base[int8] 13 | Name *column.String 14 | Value *column.Base[int64] 15 | } 16 | 17 | func newProfileEvent() *ProfileEvent { 18 | return &ProfileEvent{ 19 | Host: column.NewString(), 20 | Time: column.New[uint32](), 21 | ThreadID: column.New[uint64](), 22 | Type: column.New[int8](), 23 | Name: column.NewString(), 24 | Value: column.New[int64](), 25 | } 26 | } 27 | 28 | func (p ProfileEvent) read(c *conn) error { 29 | return c.block.readColumnsData(c, true, p.Host, p.Time, p.ThreadID, p.Type, p.Name, p.Value) 30 | } 31 | -------------------------------------------------------------------------------- /types/int256_test.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "math/big" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | // TestUint256 unit tests for various Int256 helpers. 11 | func TestInt256(t *testing.T) { 12 | t.Run("FromBig", func(t *testing.T) { 13 | if got := Int256FromBig(nil); !got.Equals(Int256Zero()) { 14 | t.Fatalf("FromBig(nil) does not equal to 0, got %#x", got) 15 | } 16 | 17 | if got := Int256FromBig(new(big.Int).Lsh(big.NewInt(1), 257)); !got.Equals(Int256Max()) { 18 | t.Fatalf("FromBig(2^129) does not equal to Max(), got %#x", got) 19 | } 20 | }) 21 | t.Run("ToBig", func(t *testing.T) { 22 | i := new(big.Int).SetInt64(124) 23 | assert.Equal(t, Int256FromBig(i).Big().String(), "124") 24 | 25 | int256From64 := Int256From64(124) 26 | assert.Equal(t, int256From64.Big().String(), "124") 27 | }) 28 | } 29 | -------------------------------------------------------------------------------- /types/int128_test.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "math/big" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | // TestUint128 unit tests for various Int128 helpers. 11 | func TestInt128(t *testing.T) { 12 | t.Run("FromBig", func(t *testing.T) { 13 | if got := Int128FromBig(nil); !got.Equals(Int128Zero()) { 14 | t.Fatalf("Int128FromBig(nil) does not equal to 0, got %#x", got) 15 | } 16 | 17 | if got := Int128FromBig(new(big.Int).Lsh(big.NewInt(1), 129)); !got.Equals(Int128Max()) { 18 | t.Fatalf("Int128FromBig(2^129) does not equal to Max(), got %#x", got) 19 | } 20 | }) 21 | t.Run("ToBig", func(t *testing.T) { 22 | i := new(big.Int).SetInt64(-124) 23 | assert.Equal(t, Int128FromBig(i).Big().String(), "-124") 24 | 25 | int128From64 := Int128From64(-124) 26 | assert.Equal(t, int128From64.Big().String(), "-124") 27 | }) 28 | } 29 | -------------------------------------------------------------------------------- /types/uint256_test.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "math/big" 5 | "testing" 6 | ) 7 | 8 | // TestUint256 unit tests for various Uint256 helpers. 9 | func TestUint256(t *testing.T) { 10 | t.Run("FromBig", func(t *testing.T) { 11 | if got := Uint256FromBig(nil); !got.Equals(Uint256Zero()) { 12 | t.Fatalf("Uint256FromBig(nil) does not equal to 0, got %#x", got) 13 | } 14 | 15 | if got := Uint256FromBig(big.NewInt(-1)); !got.Equals(Uint256Zero()) { 16 | t.Fatalf("Uint256FromBig(-1) does not equal to 0, got %#x", got) 17 | } 18 | 19 | if got := Uint256FromBig(big.NewInt(124)).Big().String(); got != "124" { 20 | t.Fatalf("Uint256FromBig(big.NewInt(124)) does not equal to 0, got %#x", got) 21 | } 22 | 23 | if got := Uint256FromBig(new(big.Int).Lsh(big.NewInt(1), 257)); !got.Equals(Uint256Max()) { 24 | t.Fatalf("Uint256FromBig(2^129) does not equal to Max(), got %#x", got) 25 | } 26 | }) 27 | } 28 | -------------------------------------------------------------------------------- /helper_test.go: -------------------------------------------------------------------------------- 1 | package chconn 2 | 3 | import ( 4 | "io" 5 | "time" 6 | ) 7 | 8 | type readErrorHelper struct { 9 | numberValid int 10 | err error 11 | r io.Reader 12 | count int 13 | } 14 | 15 | func (r *readErrorHelper) Read(p []byte) (int, error) { 16 | r.count++ 17 | if r.count > r.numberValid { 18 | return 0, r.err 19 | } 20 | return r.r.Read(p) 21 | } 22 | 23 | type writerErrorHelper struct { 24 | numberValid int 25 | err error 26 | w io.Writer 27 | count int 28 | } 29 | 30 | func (w *writerErrorHelper) Write(p []byte) (int, error) { 31 | w.count++ 32 | if w.count > w.numberValid { 33 | return 0, w.err 34 | } 35 | return w.w.Write(p) 36 | } 37 | 38 | type writerSlowHelper struct { 39 | w io.Writer 40 | sleep time.Duration 41 | } 42 | 43 | func (w *writerSlowHelper) Write(p []byte) (int, error) { 44 | time.Sleep(w.sleep) 45 | return w.w.Write(p) 46 | } 47 | -------------------------------------------------------------------------------- /internal/helper/features.go: -------------------------------------------------------------------------------- 1 | package helper 2 | 3 | const ( 4 | DbmsMinRevisionWithClientInfo = 54032 5 | DbmsMinRevisionWithServerTimezone = 54058 6 | DbmsMinRevisionWithQuotaKeyInClientInfo = 54060 7 | DbmsMinRevisionWithServerDisplayName = 54372 8 | DbmsMinRevisionWithVersionPatch = 54401 9 | DbmsMinRevisionWithClientWriteInfo = 54420 10 | DbmsMinRevisionWithSettingsSerializedAsStrings = 54429 11 | DbmsMinRevisionWithInterServerSecret = 54441 12 | DbmsMinRevisionWithOpenTelemetry = 54442 13 | DbmsMinProtocolVersionWithDistributedDepth = 54448 14 | DbmsMinProtocolVersionWithInitialQueryStartTime = 54449 15 | DbmsMinProtocolVersionWithParallelReplicas = 54453 16 | DbmsMinProtocolWithCustomSerialization = 54454 17 | DbmsMinProtocolWithQuotaKey = 54458 18 | DbmsMinProtocolWithParameters = 54459 19 | DbmsMinProtocolWithServerQueryTimeInProgress = 54460 20 | ) 21 | -------------------------------------------------------------------------------- /column/lc_indices.go: -------------------------------------------------------------------------------- 1 | package column 2 | 3 | import ( 4 | "io" 5 | "unsafe" 6 | 7 | "github.com/vahid-sohrabloo/chconn/v2/internal/readerwriter" 8 | ) 9 | 10 | type indicesColumnI interface { 11 | ReadRaw(num int, r *readerwriter.Reader) error 12 | WriteTo(io.Writer) (int64, error) 13 | appendInts([]int) 14 | readInt(value *[]int) 15 | Reset() 16 | } 17 | 18 | type indicatedTypes interface { 19 | uint8 | uint16 | uint32 | uint64 20 | } 21 | 22 | type indicesColumn[T indicatedTypes] struct { 23 | Base[T] 24 | } 25 | 26 | func newIndicesColumn[T indicatedTypes]() *indicesColumn[T] { 27 | var tmpValue T 28 | size := int(unsafe.Sizeof(tmpValue)) 29 | return &indicesColumn[T]{ 30 | Base: Base[T]{ 31 | size: size, 32 | }, 33 | } 34 | } 35 | 36 | func (c *indicesColumn[T]) readInt(value *[]int) { 37 | for _, v := range c.Data() { 38 | *value = append(*value, 39 | int(v), 40 | ) 41 | } 42 | } 43 | 44 | func (c *indicesColumn[T]) appendInts(values []int) { 45 | for _, v := range values { 46 | c.values = append(c.values, T(v)) 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /ping.go: -------------------------------------------------------------------------------- 1 | package chconn 2 | 3 | import ( 4 | "context" 5 | ) 6 | 7 | type pong struct{} 8 | 9 | // Check that connection to the server is alive. 10 | func (ch *conn) Ping(ctx context.Context) error { 11 | if ctx != context.Background() { 12 | select { 13 | case <-ctx.Done(): 14 | return newContextAlreadyDoneError(ctx) 15 | default: 16 | } 17 | ch.contextWatcher.Watch(ctx) 18 | defer ch.contextWatcher.Unwatch() 19 | } 20 | ch.writer.Uvarint(clientPing) 21 | var hasError bool 22 | defer func() { 23 | if hasError { 24 | ch.Close() 25 | } 26 | }() 27 | if _, err := ch.writer.WriteTo(ch.writerTo); err != nil { 28 | hasError = true 29 | return &writeError{"ping: write packet type", preferContextOverNetTimeoutError(ctx, err)} 30 | } 31 | 32 | res, err := ch.receiveAndProcessData(emptyOnProgress) 33 | if err != nil { 34 | hasError = true 35 | return preferContextOverNetTimeoutError(ctx, err) 36 | } 37 | if _, ok := res.(*pong); !ok { 38 | hasError = true 39 | return &unexpectedPacket{expected: "serverPong", actual: res} 40 | } 41 | 42 | return nil 43 | } 44 | -------------------------------------------------------------------------------- /column/base_little_cpu.go: -------------------------------------------------------------------------------- 1 | //go:build 386 || amd64 || amd64p32 || arm || arm64 || mipsle || mips64le || mips64p32le || ppc64le || riscv || riscv64 2 | // +build 386 amd64 amd64p32 arm arm64 mipsle mips64le mips64p32le ppc64le riscv riscv64 3 | 4 | package column 5 | 6 | import ( 7 | "io" 8 | "unsafe" 9 | ) 10 | 11 | func (c *Base[T]) readyBufferHook() { 12 | } 13 | 14 | // slice is the runtime representation of a slice. 15 | // It cannot be used safely or portably and its representation may 16 | // change in a later release. 17 | // Moreover, the Data field is not sufficient to guarantee the data 18 | // it references will not be garbage collected, so programs must keep 19 | // a separate, correctly typed pointer to the underlying data. 20 | type slice struct { 21 | Data uintptr 22 | Len int 23 | Cap int 24 | } 25 | 26 | func (c *Base[T]) WriteTo(w io.Writer) (int64, error) { 27 | s := *(*slice)(unsafe.Pointer(&c.values)) 28 | s.Len *= c.size 29 | s.Cap *= c.size 30 | var n int64 31 | src := *(*[]byte)(unsafe.Pointer(&s)) 32 | nw, err := w.Write(src) 33 | return int64(nw) + n, err 34 | } 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 vahid-sohrabloo 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /types/uint128_test.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "math/big" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | // TestUint128 unit tests for various Uint128 helpers. 11 | func TestUint128(t *testing.T) { 12 | t.Run("FromBig", func(t *testing.T) { 13 | if got := Uint128FromBig(nil); !got.Equals(Uint128Zero()) { 14 | t.Fatalf("Uint128FromBig(nil) does not equal to 0, got %#x", got) 15 | } 16 | 17 | if got := Uint128FromBig(big.NewInt(-1)); !got.Equals(Uint128Zero()) { 18 | t.Fatalf("Uint128FromBig(-1) does not equal to 0, got %#x", got) 19 | } 20 | 21 | if got := Uint256FromBig(big.NewInt(124)).Big().String(); got != "124" { 22 | t.Fatalf("Uint256FromBig(big.NewInt(124)) does not equal to 0, got %#x", got) 23 | } 24 | 25 | if got := Uint128FromBig(new(big.Int).Lsh(big.NewInt(1), 129)); !got.Equals(Uint128Max()) { 26 | t.Fatalf("Uint128FromBig(2^129) does not equal to Max(), got %#x", got) 27 | } 28 | }) 29 | 30 | t.Run("ToBig", func(t *testing.T) { 31 | i := new(big.Int).SetInt64(124) 32 | assert.Equal(t, Uint256FromBig(i).Big().String(), "124") 33 | 34 | Uint256From64 := Uint256From64(124) 35 | assert.Equal(t, Uint256From64.Big().String(), "124") 36 | }) 37 | } 38 | -------------------------------------------------------------------------------- /profile.go: -------------------------------------------------------------------------------- 1 | package chconn 2 | 3 | // Profile detail of profile select query 4 | type Profile struct { 5 | Rows uint64 6 | Blocks uint64 7 | Bytes uint64 8 | RowsBeforeLimit uint64 9 | AppliedLimit uint8 10 | CalculatedRowsBeforeLimit uint8 11 | } 12 | 13 | func newProfile() *Profile { 14 | return &Profile{} 15 | } 16 | 17 | func (p *Profile) read(ch *conn) (err error) { 18 | if p.Rows, err = ch.reader.Uvarint(); err != nil { 19 | return &readError{"profile: read Rows", err} 20 | } 21 | if p.Blocks, err = ch.reader.Uvarint(); err != nil { 22 | return &readError{"profile: read Blocks", err} 23 | } 24 | if p.Bytes, err = ch.reader.Uvarint(); err != nil { 25 | return &readError{"profile: read Bytes", err} 26 | } 27 | 28 | if p.AppliedLimit, err = ch.reader.ReadByte(); err != nil { 29 | return &readError{"profile: read AppliedLimit", err} 30 | } 31 | if p.RowsBeforeLimit, err = ch.reader.Uvarint(); err != nil { 32 | return &readError{"profile: read RowsBeforeLimit", err} 33 | } 34 | if p.CalculatedRowsBeforeLimit, err = ch.reader.ReadByte(); err != nil { 35 | return &readError{"profile: read CalculatedRowsBeforeLimit", err} 36 | } 37 | return nil 38 | } 39 | -------------------------------------------------------------------------------- /types/decimal.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | // Decimal32 represents a 32-bit decimal number. 4 | type Decimal32 int32 5 | 6 | // Decimal64 represents a 64-bit decimal number. 7 | type Decimal64 int64 8 | 9 | // Decimal128 represents a 128-bit decimal number. 10 | type Decimal128 Int128 11 | 12 | // Decimal256 represents a 256-bit decimal number. 13 | type Decimal256 Int256 14 | 15 | // Table of powers of 10 for fast casting from floating types to decimal type 16 | // representations. 17 | var factors10 = []float64{ 18 | 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, 1e12, 1e13, 19 | 1e14, 1e15, 1e16, 1e17, 1e18, 20 | } 21 | 22 | // Float64 converts decimal number to float64. 23 | func (d Decimal32) Float64(scale int) float64 { 24 | return float64(d) / factors10[scale] 25 | } 26 | 27 | // Float64 converts decimal number to float64. 28 | func (d Decimal64) Float64(scale int) float64 { 29 | return float64(d) / factors10[scale] 30 | } 31 | 32 | // Decimal32FromFloat64 converts float64 to decimal32 number. 33 | func Decimal32FromFloat64(f float64, scale int) Decimal32 { 34 | return Decimal32(f * factors10[scale]) 35 | } 36 | 37 | // Decimal64FromFloat64 converts float64 to decimal64 number. 38 | func Decimal64FromFloat64(f float64, scale int) Decimal64 { 39 | return Decimal64(f * factors10[scale]) 40 | } 41 | -------------------------------------------------------------------------------- /sqlbuilder/injection.go: -------------------------------------------------------------------------------- 1 | // sqlbuilder is a builder for SQL statements for clickhouse. 2 | // copy from https://github.com/huandu/go-sqlbuilder 3 | // change for chconn 4 | package sqlbuilder 5 | 6 | import ( 7 | "bytes" 8 | "strings" 9 | ) 10 | 11 | // injection is a helper type to manage injected SQLs in all builders. 12 | type injection struct { 13 | markerSQLs map[injectionMarker][]string 14 | } 15 | 16 | type injectionMarker int 17 | 18 | // newInjection creates a new injection. 19 | func newInjection() *injection { 20 | return &injection{ 21 | markerSQLs: map[injectionMarker][]string{}, 22 | } 23 | } 24 | 25 | // SQL adds sql to injection's sql list. 26 | // All sqls inside injection is ordered by marker in ascending order. 27 | func (injection *injection) SQL(marker injectionMarker, sql string) { 28 | injection.markerSQLs[marker] = append(injection.markerSQLs[marker], sql) 29 | } 30 | 31 | // WriteTo joins all SQL strings at the same marker value with blank (" ") 32 | // and writes the joined value to buf. 33 | func (injection *injection) WriteTo(buf *bytes.Buffer, marker injectionMarker) { 34 | sqls := injection.markerSQLs[marker] 35 | empty := buf.Len() == 0 36 | 37 | if len(sqls) == 0 { 38 | return 39 | } 40 | 41 | if !empty { 42 | buf.WriteByte(' ') 43 | } 44 | 45 | s := strings.Join(sqls, " ") 46 | buf.WriteString(s) 47 | 48 | if empty { 49 | buf.WriteByte(' ') 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /column/tuple1.go: -------------------------------------------------------------------------------- 1 | package column 2 | 3 | // Tuple1 is a column of Tuple(T1) ClickHouse data type 4 | type Tuple1[T1 any] struct { 5 | Tuple 6 | col1 Column[T1] 7 | } 8 | 9 | // NewTuple1 create a new tuple of Tuple(T1) ClickHouse data type 10 | func NewTuple1[T1 any]( 11 | column1 Column[T1], 12 | ) *Tuple1[T1] { 13 | return &Tuple1[T1]{ 14 | Tuple: Tuple{ 15 | columns: []ColumnBasic{ 16 | column1, 17 | }, 18 | }, 19 | col1: column1, 20 | } 21 | } 22 | 23 | // NewNested1 create a new nested of Nested(T1) ClickHouse data type 24 | // 25 | // this is actually an alias for NewTuple1(T1).Array() 26 | func NewNested1[T any]( 27 | column1 Column[T], 28 | ) *Array[T] { 29 | return NewTuple1( 30 | column1, 31 | ).Array() 32 | } 33 | 34 | // Data get all the data in current block as a slice. 35 | func (c *Tuple1[T]) Data() []T { 36 | return c.col1.Data() 37 | } 38 | 39 | // Read reads all the data in current block and append to the input. 40 | func (c *Tuple1[T]) Read(value []T) []T { 41 | return c.col1.Read(value) 42 | } 43 | 44 | // Row return the value of given row. 45 | // NOTE: Row number start from zero 46 | func (c *Tuple1[T]) Row(row int) T { 47 | return c.col1.Row(row) 48 | } 49 | 50 | // Append value for insert 51 | func (c *Tuple1[T]) Append(v ...T) { 52 | c.col1.Append(v...) 53 | } 54 | 55 | // Array return a Array type for this column 56 | func (c *Tuple1[T]) Array() *Array[T] { 57 | return NewArray[T](c) 58 | } 59 | -------------------------------------------------------------------------------- /progress.go: -------------------------------------------------------------------------------- 1 | package chconn 2 | 3 | import "github.com/vahid-sohrabloo/chconn/v2/internal/helper" 4 | 5 | // Progress details of progress select query 6 | type Progress struct { 7 | ReadRows uint64 8 | ReadBytes uint64 9 | TotalRows uint64 10 | WriterRows uint64 11 | WrittenBytes uint64 12 | ElapsedNS uint64 13 | } 14 | 15 | func newProgress() *Progress { 16 | return &Progress{} 17 | } 18 | 19 | func (p *Progress) read(ch *conn) (err error) { 20 | if p.ReadRows, err = ch.reader.Uvarint(); err != nil { 21 | return &readError{"progress: read ReadRows", err} 22 | } 23 | if p.ReadBytes, err = ch.reader.Uvarint(); err != nil { 24 | return &readError{"progress: read ReadBytes", err} 25 | } 26 | 27 | if p.TotalRows, err = ch.reader.Uvarint(); err != nil { 28 | return &readError{"progress: read TotalRows", err} 29 | } 30 | 31 | if ch.serverInfo.Revision >= helper.DbmsMinRevisionWithClientWriteInfo { 32 | if p.WriterRows, err = ch.reader.Uvarint(); err != nil { 33 | return &readError{"progress: read WriterRows", err} 34 | } 35 | if p.WrittenBytes, err = ch.reader.Uvarint(); err != nil { 36 | return &readError{"progress: read WrittenBytes", err} 37 | } 38 | } 39 | if ch.serverInfo.Revision >= helper.DbmsMinProtocolWithServerQueryTimeInProgress { 40 | if p.ElapsedNS, err = ch.reader.Uvarint(); err != nil { 41 | return &readError{"progress: read ElapsedNS", err} 42 | } 43 | } 44 | 45 | return nil 46 | } 47 | -------------------------------------------------------------------------------- /column/base_big_cpu.go: -------------------------------------------------------------------------------- 1 | //go:build !(386 || amd64 || amd64p32 || arm || arm64 || mipsle || mips64le || mips64p32le || ppc64le || riscv || riscv64) 2 | // +build !386,!amd64,!amd64p32,!arm,!arm64,!mipsle,!mips64le,!mips64p32le,!ppc64le,!riscv,!riscv64 3 | 4 | package column 5 | 6 | // ReadAll read all value in this block and append to the input slice 7 | func (c *Base[T]) readyBufferHook() { 8 | for i := 0; i < c.totalByte; i += c.size { 9 | reverseBuffer(c.b[i : i+c.size]) 10 | } 11 | } 12 | 13 | func reverseBuffer(s []byte) { 14 | for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 { 15 | s[i], s[j] = s[j], s[i] 16 | } 17 | } 18 | 19 | // slice is the runtime representation of a slice. 20 | // It cannot be used safely or portably and its representation may 21 | // change in a later release. 22 | // Moreover, the Data field is not sufficient to guarantee the data 23 | // it references will not be garbage collected, so programs must keep 24 | // a separate, correctly typed pointer to the underlying data. 25 | type slice struct { 26 | Data uintptr 27 | Len int 28 | Cap int 29 | } 30 | 31 | func (c *Base[T]) WriteTo(w io.Writer) (int64, error) { 32 | s := *(*slice)(unsafe.Pointer(&c.values)) 33 | s.Len *= c.size 34 | s.Cap *= c.size 35 | b := *(*[]byte)(unsafe.Pointer(&s)) 36 | for i := 0; i < len(b); i += c.size { 37 | reverseBuffer(b[i : i+c.size]) 38 | } 39 | var n int64 40 | nw, err := w.Write(*(*[]byte)(unsafe.Pointer(&s))) 41 | return int64(nw) + n, err 42 | } 43 | -------------------------------------------------------------------------------- /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | 9 | jobs: 10 | 11 | test-coverage: 12 | name: Test Coverage 13 | runs-on: ubuntu-latest 14 | env: 15 | VERBOSE: 1 16 | GOFLAGS: -mod=readonly 17 | 18 | steps: 19 | - uses: vahid-sohrabloo/clickhouse-action@v1 20 | with: 21 | version: '22.9' 22 | 23 | 24 | - name: Set up Go 25 | uses: actions/setup-go@v3 26 | with: 27 | go-version: 1.19 28 | 29 | - name: Checkout code 30 | uses: actions/checkout@v3.3.0 31 | 32 | - name: Test 33 | run: make test-cover 34 | - name: Send coverage 35 | uses: codecov/codecov-action@v3 36 | with: 37 | file: coverage.out 38 | test: 39 | name: Test 40 | runs-on: ubuntu-latest 41 | strategy: 42 | matrix: 43 | golang-version: [1.18.5, 1.19] 44 | clickhouse-version: ['22.11', '22.10', '22.9', '22.8', '22.7', '22.6', '22.5', '22.4'] 45 | env: 46 | VERBOSE: 1 47 | GOFLAGS: -mod=readonly 48 | 49 | steps: 50 | - uses: vahid-sohrabloo/clickhouse-action@v1 51 | with: 52 | version: '${{ matrix.clickhouse-version }}' 53 | 54 | - name: Set up Go 55 | uses: actions/setup-go@v3 56 | with: 57 | go-version: 1.18.5 58 | 59 | - name: Checkout code 60 | uses: actions/checkout@v3.3.0 61 | 62 | - name: Test 63 | run: make test -------------------------------------------------------------------------------- /internal/readerwriter/consts.go: -------------------------------------------------------------------------------- 1 | package readerwriter 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/go-faster/city" 7 | ) 8 | 9 | // Method is compression codec. 10 | type CompressMethod byte 11 | 12 | const ( 13 | // ChecksumSize is 128bits for cityhash102 checksum 14 | ChecksumSize = 16 15 | // CompressHeaderSize magic + compressed_size + uncompressed_size 16 | CompressHeaderSize = 1 + 4 + 4 17 | 18 | // HeaderSize for compress header 19 | HeaderSize = ChecksumSize + CompressHeaderSize 20 | // BlockMaxSize 1MB 21 | BlockMaxSize = 1024 * 1024 * 128 22 | ) 23 | 24 | // Possible compression methods. 25 | const ( 26 | CompressNone CompressMethod = 0x00 27 | CompressChecksum CompressMethod = 0x02 28 | CompressLZ4 CompressMethod = 0x82 29 | CompressZSTD CompressMethod = 0x90 30 | ) 31 | 32 | // Constants for compression encoding. 33 | // 34 | // See https://go-faster.org/docs/clickhouse/compression for reference. 35 | const ( 36 | checksumSize = 16 37 | compressHeaderSize = 1 + 4 + 4 38 | headerSize = checksumSize + compressHeaderSize 39 | 40 | // Limiting total data/block size to protect from possible OOM. 41 | maxDataSize = 1024 * 1024 * 2 // 2MB 42 | maxBlockSize = maxDataSize 43 | 44 | hRawSize = 17 45 | hDataSize = 21 46 | hMethod = 16 47 | ) 48 | 49 | // CorruptedDataErr means that provided hash mismatch with calculated. 50 | type CorruptedDataErr struct { 51 | Actual city.U128 52 | Reference city.U128 53 | RawSize int 54 | DataSize int 55 | } 56 | 57 | func (c *CorruptedDataErr) Error() string { 58 | return fmt.Sprintf("corrupted data: %d (actual), %d (reference), compressed size: %d, data size: %d", 59 | c.Actual.High, c.Reference.High, c.RawSize, c.DataSize, 60 | ) 61 | } 62 | -------------------------------------------------------------------------------- /column/size.go: -------------------------------------------------------------------------------- 1 | package column 2 | 3 | const ( 4 | // Uint8Size data Size of Uint8 Column 5 | Uint8Size = 1 6 | // Uint16Size data Size of Uint16 Column 7 | Uint16Size = 2 8 | // Uint32Size data Size of Uint32 Column 9 | Uint32Size = 4 10 | // Uint64Size data Size of Uint64 Column 11 | Uint64Size = 8 12 | // Uint128Size data Size of Uint128 Column 13 | Uint128Size = 16 14 | // Uint256Size data Size of Uint256 Column 15 | Uint256Size = 32 16 | // Int8Size data Size of Int8 Column 17 | Int8Size = 1 18 | // Int16Size data Size of Int16 Column 19 | Int16Size = 2 20 | // Int32Size data Size of Int32 Column 21 | Int32Size = 4 22 | // Int64Size data Size of Int64 Column 23 | Int64Size = 8 24 | // Int128Size data Size of Int128 Column 25 | Int128Size = 16 26 | // Int256Size data Size of Int256 Column 27 | Int256Size = 32 28 | // Float32Size data Size of Float32 Column 29 | Float32Size = 4 30 | // Float64Size data Size of Float64 Column 31 | Float64Size = 8 32 | // DateSize data Size of Date Column 33 | DateSize = 2 34 | // Date32Size data Size of Date32 Column 35 | Date32Size = 4 36 | // DatetimeSize data Size of Datetime Column 37 | DatetimeSize = 4 38 | // Datetime64Size data Size of Datetime64 Column 39 | Datetime64Size = 8 40 | // IPv4Size data Size of IPv4 Column 41 | IPv4Size = 4 42 | // IPv6Size data Size of IPv6 Column 43 | IPv6Size = 16 44 | // Decimal32Size data Size of Decimal32 Column 45 | Decimal32Size = 4 46 | // Decimal64Size data Size of Decimal64 Column 47 | Decimal64Size = 8 48 | // Decimal128Size data Size of Decimal128 Column 49 | Decimal128Size = 16 50 | // Decimal256Size data Size of Decimal256 Column 51 | Decimal256Size = 32 52 | // ArraylenSize data Size of Arraylen Column 53 | ArraylenSize = 8 54 | // MaplenSize data Size of Maplen Column 55 | MaplenSize = 8 56 | // UUIDSize data Size of UUID Column 57 | UUIDSize = 16 58 | ) 59 | -------------------------------------------------------------------------------- /column/array2.go: -------------------------------------------------------------------------------- 1 | package column 2 | 3 | // Array2 is a column of Array(Array(T)) ClickHouse data type 4 | type Array2[T any] struct { 5 | ArrayBase 6 | } 7 | 8 | // NewArray create a new array column of Array(Array(T)) ClickHouse data type 9 | func NewArray2[T any](array *Array[T]) *Array2[T] { 10 | a := &Array2[T]{ 11 | ArrayBase: ArrayBase{ 12 | dataColumn: array, 13 | offsetColumn: New[uint64](), 14 | }, 15 | } 16 | return a 17 | } 18 | 19 | // Data get all the data in current block as a slice. 20 | func (c *Array2[T]) Data() [][][]T { 21 | values := make([][][]T, c.offsetColumn.numRow) 22 | for i := range values { 23 | values[i] = c.Row(i) 24 | } 25 | return values 26 | } 27 | 28 | // Read reads all the data in current block and append to the input. 29 | func (c *Array2[T]) Read(value [][][]T) [][][]T { 30 | if cap(value)-len(value) >= c.NumRow() { 31 | value = (value)[:len(value)+c.NumRow()] 32 | } else { 33 | value = append(value, make([][][]T, c.NumRow())...) 34 | } 35 | val := (value)[len(value)-c.NumRow():] 36 | for i := 0; i < c.NumRow(); i++ { 37 | val[i] = c.Row(i) 38 | } 39 | return value 40 | } 41 | 42 | // Row return the value of given row. 43 | // NOTE: Row number start from zero 44 | func (c *Array2[T]) Row(row int) [][]T { 45 | var lastOffset uint64 46 | if row != 0 { 47 | lastOffset = c.offsetColumn.Row(row - 1) 48 | } 49 | var val [][]T 50 | lastRow := c.offsetColumn.Row(row) 51 | for ; lastOffset < lastRow; lastOffset++ { 52 | val = append(val, c.dataColumn.(*Array[T]).Row(int(lastOffset))) 53 | } 54 | return val 55 | } 56 | 57 | // Append value for insert 58 | func (c *Array2[T]) Append(v ...[][]T) { 59 | for _, v := range v { 60 | c.AppendLen(len(v)) 61 | c.dataColumn.(*Array[T]).Append(v...) 62 | } 63 | } 64 | 65 | func (c *Array2[T]) elem(arrayLevel int) ColumnBasic { 66 | if arrayLevel > 0 { 67 | return c.Array().elem(arrayLevel - 1) 68 | } 69 | return c 70 | } 71 | -------------------------------------------------------------------------------- /column/array3.go: -------------------------------------------------------------------------------- 1 | package column 2 | 3 | // Array3 is a column of Array(Array(Array(T))) ClickHouse data type 4 | type Array3[T any] struct { 5 | ArrayBase 6 | } 7 | 8 | // NewArray create a new array column of Array(Array(Array(T))) ClickHouse data type 9 | func NewArray3[T any](array *Array2[T]) *Array3[T] { 10 | a := &Array3[T]{ 11 | ArrayBase: ArrayBase{ 12 | dataColumn: array, 13 | offsetColumn: New[uint64](), 14 | }, 15 | } 16 | return a 17 | } 18 | 19 | // Data get all the data in current block as a slice. 20 | func (c *Array3[T]) Data() [][][][]T { 21 | values := make([][][][]T, c.offsetColumn.numRow) 22 | for i := range values { 23 | values[i] = c.Row(i) 24 | } 25 | return values 26 | } 27 | 28 | // Read reads all the data in current block and append to the input. 29 | func (c *Array3[T]) Read(value [][][][]T) [][][][]T { 30 | if cap(value)-len(value) >= c.NumRow() { 31 | value = (value)[:len(value)+c.NumRow()] 32 | } else { 33 | value = append(value, make([][][][]T, c.NumRow())...) 34 | } 35 | val := (value)[len(value)-c.NumRow():] 36 | for i := 0; i < c.NumRow(); i++ { 37 | val[i] = c.Row(i) 38 | } 39 | return value 40 | } 41 | 42 | // Row return the value of given row. 43 | // NOTE: Row number start from zero 44 | func (c *Array3[T]) Row(row int) [][][]T { 45 | var lastOffset uint64 46 | if row != 0 { 47 | lastOffset = c.offsetColumn.Row(row - 1) 48 | } 49 | var val [][][]T 50 | lastRow := c.offsetColumn.Row(row) 51 | for ; lastOffset < lastRow; lastOffset++ { 52 | val = append(val, c.dataColumn.(*Array2[T]).Row(int(lastOffset))) 53 | } 54 | return val 55 | } 56 | 57 | // Append value for insert 58 | func (c *Array3[T]) Append(v ...[][][]T) { 59 | for _, v := range v { 60 | c.AppendLen(len(v)) 61 | c.dataColumn.(*Array2[T]).Append(v...) 62 | } 63 | } 64 | 65 | // Array return a Array type for this column 66 | func (c *Array2[T]) Array() *Array3[T] { 67 | return NewArray3(c) 68 | } 69 | 70 | func (c *Array3[T]) elem(arrayLevel int) ColumnBasic { 71 | if arrayLevel > 0 { 72 | panic("array level is too deep") 73 | } 74 | return c 75 | } 76 | -------------------------------------------------------------------------------- /internal/ctxwatch/context_watcher.go: -------------------------------------------------------------------------------- 1 | package ctxwatch 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | ) 7 | 8 | // ContextWatcher watches a context and performs an action when the context is canceled. It can watch one context at a 9 | // time. 10 | type ContextWatcher struct { 11 | onCancel func() 12 | onUnwatchAfterCancel func() 13 | unwatchChan chan struct{} 14 | 15 | lock sync.Mutex 16 | watchInProgress bool 17 | onCancelWasCalled bool 18 | } 19 | 20 | // NewContextWatcher returns a ContextWatcher. onCancel will be called when a watched context is canceled. 21 | // OnUnwatchAfterCancel will be called when Unwatch is called and the watched context had already been canceled and 22 | // onCancel called. 23 | func NewContextWatcher(onCancel, onUnwatchAfterCancel func()) *ContextWatcher { 24 | cw := &ContextWatcher{ 25 | onCancel: onCancel, 26 | onUnwatchAfterCancel: onUnwatchAfterCancel, 27 | unwatchChan: make(chan struct{}), 28 | } 29 | 30 | return cw 31 | } 32 | 33 | // Watch starts watching ctx. If ctx is canceled then the onCancel function passed to NewContextWatcher will be called. 34 | func (cw *ContextWatcher) Watch(ctx context.Context) { 35 | cw.lock.Lock() 36 | defer cw.lock.Unlock() 37 | 38 | if cw.watchInProgress { 39 | panic("Watch already in progress") 40 | } 41 | 42 | cw.onCancelWasCalled = false 43 | 44 | if ctx.Done() != nil { 45 | cw.watchInProgress = true 46 | go func() { 47 | select { 48 | case <-ctx.Done(): 49 | cw.onCancel() 50 | cw.onCancelWasCalled = true 51 | <-cw.unwatchChan 52 | case <-cw.unwatchChan: 53 | } 54 | }() 55 | } else { 56 | cw.watchInProgress = false 57 | } 58 | } 59 | 60 | // Unwatch stops watching the previously watched context. If the onCancel function passed to NewContextWatcher was 61 | // called then onUnwatchAfterCancel will also be called. 62 | func (cw *ContextWatcher) Unwatch() { 63 | cw.lock.Lock() 64 | defer cw.lock.Unlock() 65 | 66 | if cw.watchInProgress { 67 | cw.unwatchChan <- struct{}{} 68 | if cw.onCancelWasCalled { 69 | cw.onUnwatchAfterCancel() 70 | } 71 | cw.watchInProgress = false 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /server_info_test.go: -------------------------------------------------------------------------------- 1 | package chconn 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "io" 7 | "os" 8 | "testing" 9 | 10 | "github.com/stretchr/testify/require" 11 | ) 12 | 13 | func TestServerInfoError(t *testing.T) { 14 | startValidReader := 1 15 | 16 | tests := []struct { 17 | name string 18 | wantErr string 19 | numberValid int 20 | }{ 21 | { 22 | name: "server name", 23 | wantErr: "ServerInfo: could not read server name", 24 | numberValid: startValidReader, 25 | }, { 26 | name: "server major version", 27 | wantErr: "ServerInfo: could not read server major version", 28 | numberValid: startValidReader + 2, 29 | }, { 30 | name: "server minor version", 31 | wantErr: "ServerInfo: could not read server minor version", 32 | numberValid: startValidReader + 3, 33 | }, { 34 | name: "server revision", 35 | wantErr: "ServerInfo: could not read server revision", 36 | numberValid: startValidReader + 4, 37 | }, { 38 | name: "server timezone", 39 | wantErr: "ServerInfo: could not read server timezone", 40 | numberValid: startValidReader + 7, 41 | }, { 42 | name: "server display name", 43 | wantErr: "ServerInfo: could not read server display name", 44 | numberValid: startValidReader + 9, 45 | }, { 46 | name: "server version patch", 47 | wantErr: "ServerInfo: could not read server version patch", 48 | numberValid: startValidReader + 11, 49 | }, 50 | } 51 | for _, tt := range tests { 52 | t.Run(tt.name, func(t *testing.T) { 53 | config, err := ParseConfig(os.Getenv("CHX_TEST_TCP_CONN_STRING")) 54 | require.NoError(t, err) 55 | config.ReaderFunc = func(r io.Reader) io.Reader { 56 | return &readErrorHelper{ 57 | err: errors.New("timeout"), 58 | r: r, 59 | numberValid: tt.numberValid, 60 | } 61 | } 62 | 63 | _, err = ConnectConfig(context.Background(), config) 64 | require.Error(t, err) 65 | readErr, ok := err.(*readError) 66 | require.True(t, ok) 67 | require.Equal(t, readErr.msg, tt.wantErr) 68 | require.EqualError(t, readErr.Unwrap(), "timeout") 69 | }) 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /server_info.go: -------------------------------------------------------------------------------- 1 | package chconn 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/vahid-sohrabloo/chconn/v2/internal/helper" 7 | "github.com/vahid-sohrabloo/chconn/v2/internal/readerwriter" 8 | ) 9 | 10 | // ServerInfo detail of server info 11 | type ServerInfo struct { 12 | Name string 13 | Revision uint64 14 | MinorVersion uint64 15 | MajorVersion uint64 16 | ServerDisplayName string 17 | ServerVersionPatch uint64 18 | Timezone string 19 | } 20 | 21 | func (srv *ServerInfo) read(r *readerwriter.Reader) (err error) { 22 | if srv.Name, err = r.String(); err != nil { 23 | return &readError{"ServerInfo: could not read server name", err} 24 | } 25 | if srv.MajorVersion, err = r.Uvarint(); err != nil { 26 | return &readError{"ServerInfo: could not read server major version", err} 27 | } 28 | if srv.MinorVersion, err = r.Uvarint(); err != nil { 29 | return &readError{"ServerInfo: could not read server minor version", err} 30 | } 31 | if srv.Revision, err = r.Uvarint(); err != nil { 32 | return &readError{"ServerInfo: could not read server revision", err} 33 | } 34 | if srv.Revision >= helper.DbmsMinRevisionWithServerTimezone { 35 | if srv.Timezone, err = r.String(); err != nil { 36 | return &readError{"ServerInfo: could not read server timezone", err} 37 | } 38 | } 39 | if srv.Revision >= helper.DbmsMinRevisionWithServerDisplayName { 40 | if srv.ServerDisplayName, err = r.String(); err != nil { 41 | return &readError{"ServerInfo: could not read server display name", err} 42 | } 43 | } 44 | if srv.Revision >= helper.DbmsMinRevisionWithVersionPatch { 45 | if srv.ServerVersionPatch, err = r.Uvarint(); err != nil { 46 | return &readError{"ServerInfo: could not read server version patch", err} 47 | } 48 | } 49 | return nil 50 | } 51 | 52 | func (srv *ServerInfo) String() string { 53 | return fmt.Sprintf("%s %d.%d.%d (%s) %s %d", 54 | srv.Name, 55 | srv.MajorVersion, 56 | srv.MinorVersion, 57 | srv.Revision, 58 | srv.Timezone, 59 | srv.ServerDisplayName, 60 | srv.ServerVersionPatch) 61 | } 62 | 63 | // ServerInfo get server info 64 | func (ch *conn) ServerInfo() *ServerInfo { 65 | return ch.serverInfo 66 | } 67 | -------------------------------------------------------------------------------- /internal/helper/strs.go: -------------------------------------------------------------------------------- 1 | package helper 2 | 3 | const ( 4 | TupleStr = "Tuple(" 5 | LenTupleStr = len(TupleStr) 6 | PointStr = "Point" 7 | ) 8 | 9 | var PointMainTypeStr = []byte("Tuple(Float64, Float64)") 10 | 11 | const PolygonStr = "Polygon" 12 | 13 | var PolygonMainTypeStr = []byte("Array(Array(Tuple(Float64, Float64)))") 14 | 15 | const MultiPolygonStr = "MultiPolygon" 16 | 17 | var MultiPolygonMainTypeStr = []byte("Array(Array(Array(Tuple(Float64, Float64))))") 18 | 19 | const ( 20 | ArrayStr = "Array(" 21 | LenArrayStr = len(ArrayStr) 22 | ArrayTypeStr = "Array()" 23 | NestedStr = "Nested(" 24 | LenNestedStr = len(NestedStr) 25 | NestedToArrayTube = "Array(Nested(" 26 | RingStr = "Ring" 27 | ) 28 | 29 | var RingMainTypeStr = []byte("Array(Tuple(Float64, Float64))") 30 | 31 | const ( 32 | Enum8Str = "Enum8(" 33 | Enum8StrLen = len(Enum8Str) 34 | Enum16Str = "Enum16(" 35 | Enum16StrLen = len(Enum16Str) 36 | DateTimeStr = "DateTime(" 37 | DateTimeStrLen = len(DateTimeStr) 38 | DateTime64Str = "DateTime64(" 39 | DateTime64StrLen = len(DateTime64Str) 40 | DecimalStr = "Decimal(" 41 | DecimalStrLen = len(DecimalStr) 42 | FixedStringStr = "FixedString(" 43 | FixedStringStrLen = len(FixedStringStr) 44 | SimpleAggregateStr = "SimpleAggregateFunction(" 45 | SimpleAggregateStrLen = len(SimpleAggregateStr) 46 | ) 47 | 48 | const ( 49 | LowCardinalityStr = "LowCardinality(" 50 | LenLowCardinalityStr = len(LowCardinalityStr) 51 | LowCardinalityTypeStr = "LowCardinality()" 52 | LowCardinalityNullableStr = "LowCardinality(Nullable(" 53 | LenLowCardinalityNullableStr = len(LowCardinalityNullableStr) 54 | LowCardinalityNullableTypeStr = "LowCardinality(Nullable())" 55 | ) 56 | 57 | const ( 58 | MapStr = "Map(" 59 | LenMapStr = len(MapStr) 60 | MapTypeStr = "Map(, )" 61 | ) 62 | 63 | const ( 64 | NullableStr = "Nullable(" 65 | LenNullableStr = len(NullableStr) 66 | NullableTypeStr = "Nullable()" 67 | ) 68 | 69 | const ( 70 | StringStr = "String" 71 | ) 72 | -------------------------------------------------------------------------------- /column/bench_test.go: -------------------------------------------------------------------------------- 1 | package column_test 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/vahid-sohrabloo/chconn/v2" 8 | "github.com/vahid-sohrabloo/chconn/v2/column" 9 | ) 10 | 11 | func BenchmarkTestChconnSelect100MUint64(b *testing.B) { 12 | // return 13 | ctx := context.Background() 14 | c, err := chconn.Connect(ctx, "password=salam") 15 | if err != nil { 16 | b.Fatal(err) 17 | } 18 | colRead := column.New[uint64]() 19 | for n := 0; n < b.N; n++ { 20 | s, err := c.Select(ctx, "SELECT number FROM system.numbers_mt LIMIT 100000000", colRead) 21 | if err != nil { 22 | b.Fatal(err) 23 | } 24 | 25 | for s.Next() { 26 | colRead.Data() 27 | } 28 | if err := s.Err(); err != nil { 29 | b.Fatal(err) 30 | } 31 | s.Close() 32 | } 33 | } 34 | 35 | func BenchmarkTestChconnSelect1MString(b *testing.B) { 36 | ctx := context.Background() 37 | c, err := chconn.Connect(ctx, "password=salam") 38 | if err != nil { 39 | b.Fatal(err) 40 | } 41 | 42 | colRead := column.NewString() 43 | var data [][]byte 44 | for n := 0; n < b.N; n++ { 45 | s, err := c.Select(ctx, "SELECT randomString(20) FROM system.numbers_mt LIMIT 1000000", colRead) 46 | if err != nil { 47 | b.Fatal(err) 48 | } 49 | 50 | for s.Next() { 51 | data = data[:0] 52 | colRead.DataBytes() 53 | } 54 | if err := s.Err(); err != nil { 55 | b.Fatal(err) 56 | } 57 | s.Close() 58 | } 59 | } 60 | 61 | func BenchmarkTestChconnInsert10M(b *testing.B) { 62 | // return 63 | ctx := context.Background() 64 | c, err := chconn.Connect(ctx, "password=salam") 65 | if err != nil { 66 | b.Fatal(err) 67 | } 68 | err = c.Exec(ctx, "DROP TABLE IF EXISTS test_insert_chconn") 69 | if err != nil { 70 | b.Fatal(err) 71 | } 72 | err = c.Exec(ctx, "CREATE TABLE test_insert_chconn (id UInt64) ENGINE = Null") 73 | if err != nil { 74 | b.Fatal(err) 75 | } 76 | 77 | const ( 78 | rowsInBlock = 10_000_000 79 | ) 80 | 81 | idColumns := column.New[uint64]() 82 | idColumns.SetWriteBufferSize(rowsInBlock) 83 | for n := 0; n < b.N; n++ { 84 | for y := 0; y < rowsInBlock; y++ { 85 | idColumns.Append(1) 86 | } 87 | err := c.Insert(ctx, "INSERT INTO test_insert_chconn VALUES", idColumns) 88 | if err != nil { 89 | b.Fatal(err) 90 | } 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /sqlbuilder/select_test.go: -------------------------------------------------------------------------------- 1 | package sqlbuilder 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | "github.com/stretchr/testify/require" 8 | "github.com/vahid-sohrabloo/chconn/v2" 9 | ) 10 | 11 | func TestSelectBuilder(t *testing.T) { 12 | sb := Select("id", "name", As("COUNT(*)", "t")).Distinct() 13 | sb.Column("age", "birthday") 14 | sb.From("user").Final() 15 | sb.SQL("/* before */") 16 | sb.ArrayJoin("roles").LeftArrayJoin() 17 | sb.SQL("/* after */") 18 | sb.PreWhere("id > 0") 19 | sb.Where( 20 | "id > {id: Int32}", 21 | "name LIKE {name: String}", 22 | ) 23 | sb.Parameters(chconn.IntParameter("id", 1)) 24 | sb.Parameters(chconn.StringParameter("name", "vahid")) 25 | sb.Join("contract c", 26 | "u.id = c.user_id", 27 | "c.status = {status: Array(Int64)}", 28 | ) 29 | sb.Parameters(chconn.IntSliceParameter("status", []int64{1, 2, 3})) 30 | sb.JoinWithOption(RightOuterJoin, "person p", 31 | "u.id = p.user_id", 32 | "p.surname = {surname: String}", 33 | ) 34 | sb.Parameters(chconn.StringParameter("surname", "sohrabloo")) 35 | sb.GroupBy("status").Having("status > 0") 36 | sb.OrderBy("modified_at ASC", "created_at DESC") 37 | sb.Limit(10).Offset(5) 38 | 39 | s, args := sb.Build() 40 | 41 | assert.Equal(t, "SELECT DISTINCT id, name, COUNT(*) AS t, age, birthday /* before */ FROM user FINAL "+ 42 | "LEFT ARRAY JOIN roles /* after */ "+ 43 | "JOIN contract c ON u.id = c.user_id AND c.status = {status: Array(Int64)} "+ 44 | "RIGHT OUTER JOIN person p ON u.id = p.user_id AND p.surname = {surname: String} "+ 45 | "PREWHERE id > 0 "+ 46 | "WHERE id > {id: Int32} AND name LIKE {name: String} "+ 47 | "GROUP BY status HAVING status > 0 "+ 48 | "ORDER BY modified_at ASC, created_at DESC "+ 49 | "LIMIT 10 OFFSET 5", 50 | s, 51 | ) 52 | require.Len(t, args.Params(), 4) 53 | assert.Equal(t, "id", args.Params()[0].Name) 54 | assert.Equal(t, "'1'", args.Params()[0].Value) 55 | assert.Equal(t, "name", args.Params()[1].Name) 56 | assert.Equal(t, "'vahid'", args.Params()[1].Value) 57 | assert.Equal(t, "status", args.Params()[2].Name) 58 | assert.Equal(t, "'[1,2,3]'", args.Params()[2].Value) 59 | assert.Equal(t, "surname", args.Params()[3].Name) 60 | assert.Equal(t, "'sohrabloo'", args.Params()[3].Value) 61 | } 62 | -------------------------------------------------------------------------------- /internal/readerwriter/writer.go: -------------------------------------------------------------------------------- 1 | package readerwriter 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "io" 7 | "reflect" 8 | "unsafe" 9 | ) 10 | 11 | // Writer is a helper to write data into bytes.Buffer 12 | type Writer struct { 13 | output *bytes.Buffer 14 | scratch [binary.MaxVarintLen64]byte 15 | } 16 | 17 | // NewWriter get new writer 18 | func NewWriter() *Writer { 19 | return &Writer{ 20 | output: &bytes.Buffer{}, 21 | } 22 | } 23 | 24 | // Uvarint write a variable uint64 value into writer 25 | func (w *Writer) Uvarint(v uint64) { 26 | ln := binary.PutUvarint(w.scratch[:binary.MaxVarintLen64], v) 27 | w.Write(w.scratch[:ln]) 28 | } 29 | 30 | // Int32 write Int32 value 31 | func (w *Writer) Int32(v int32) { 32 | w.Uint32(uint32(v)) 33 | } 34 | 35 | // Int64 write Int64 value 36 | func (w *Writer) Int64(v int64) { 37 | w.Uint64(uint64(v)) 38 | } 39 | 40 | // Uint8 write Uint8 value 41 | func (w *Writer) Uint8(v uint8) { 42 | w.output.WriteByte(v) 43 | } 44 | 45 | // Uint32 write Uint32 value 46 | func (w *Writer) Uint32(v uint32) { 47 | w.scratch[0] = byte(v) 48 | w.scratch[1] = byte(v >> 8) 49 | w.scratch[2] = byte(v >> 16) 50 | w.scratch[3] = byte(v >> 24) 51 | w.Write(w.scratch[:4]) 52 | } 53 | 54 | // Uint64 write Uint64 value 55 | func (w *Writer) Uint64(v uint64) { 56 | w.scratch[0] = byte(v) 57 | w.scratch[1] = byte(v >> 8) 58 | w.scratch[2] = byte(v >> 16) 59 | w.scratch[3] = byte(v >> 24) 60 | w.scratch[4] = byte(v >> 32) 61 | w.scratch[5] = byte(v >> 40) 62 | w.scratch[6] = byte(v >> 48) 63 | w.scratch[7] = byte(v >> 56) 64 | w.Write(w.scratch[:8]) 65 | } 66 | 67 | // String write string 68 | func (w *Writer) String(v string) { 69 | str := str2Bytes(v) 70 | w.Uvarint(uint64(len(str))) 71 | w.Write(str) 72 | } 73 | 74 | // ByteString write []byte 75 | func (w *Writer) ByteString(v []byte) { 76 | w.Uvarint(uint64(len(v))) 77 | w.Write(v) 78 | } 79 | 80 | // Write write raw []byte data 81 | func (w *Writer) Write(b []byte) { 82 | w.output.Write(b) 83 | } 84 | 85 | // WriteTo implement WriteTo 86 | func (w *Writer) WriteTo(wt io.Writer) (int64, error) { 87 | return w.output.WriteTo(wt) 88 | } 89 | 90 | // Reset reset all data 91 | func (w *Writer) Reset() { 92 | w.output.Reset() 93 | } 94 | 95 | // Output get raw *bytes.Buffer 96 | func (w *Writer) Output() *bytes.Buffer { 97 | return w.output 98 | } 99 | 100 | func str2Bytes(str string) []byte { 101 | header := (*reflect.SliceHeader)(unsafe.Pointer(&str)) 102 | header.Len = len(str) 103 | header.Cap = header.Len 104 | return *(*[]byte)(unsafe.Pointer(header)) 105 | } 106 | -------------------------------------------------------------------------------- /types/date_type.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | type Date uint16 8 | 9 | const minDate32 = int32(-25567) // 1900-01-01 00:00:00 +0000 UTC 10 | 11 | type Date32 int32 12 | 13 | type DateTime uint32 14 | 15 | const minDateTime64 = int64(-2208988800) // 1900-01-01 00:00:00 +0000 UTC 16 | 17 | type DateTime64 int64 18 | 19 | const daySeconds = 24 * 60 * 60 20 | 21 | func TimeToDate(t time.Time) Date { 22 | if t.Unix() <= 0 { 23 | return 0 24 | } 25 | _, offset := t.Zone() 26 | return Date((t.Unix() + int64(offset)) / daySeconds) 27 | } 28 | 29 | func (d Date) FromTime(v time.Time, precision int) Date { 30 | return TimeToDate(v) 31 | } 32 | 33 | func (d Date) ToTime(loc *time.Location, precision int) time.Time { 34 | return time.Unix(d.Unix(), 0).UTC() 35 | } 36 | 37 | func (d Date) Unix() int64 { 38 | return daySeconds * int64(d) 39 | } 40 | 41 | func (d Date32) Unix() int64 { 42 | return daySeconds * int64(d) 43 | } 44 | 45 | func (d Date32) FromTime(v time.Time, precision int) Date32 { 46 | return TimeToDate32(v) 47 | } 48 | 49 | func (d Date32) ToTime(loc *time.Location, precision int) time.Time { 50 | return time.Unix(d.Unix(), 0).UTC() 51 | } 52 | 53 | func TimeToDate32(t time.Time) Date32 { 54 | _, offset := t.Zone() 55 | d := int32((t.Unix() + int64(offset)) / daySeconds) 56 | if d <= minDate32 { 57 | return Date32(minDate32) 58 | } 59 | 60 | return Date32(d) 61 | } 62 | 63 | func TimeToDateTime(t time.Time) DateTime { 64 | if t.Unix() <= 0 { 65 | return 0 66 | } 67 | return DateTime(t.Unix()) 68 | } 69 | 70 | func (d DateTime) FromTime(v time.Time, precision int) DateTime { 71 | return TimeToDateTime(v) 72 | } 73 | 74 | func (d DateTime) ToTime(loc *time.Location, precision int) time.Time { 75 | return time.Unix(int64(d), 0).In(loc) 76 | } 77 | 78 | var precisionFactor = [...]int64{ 79 | 1000000000, 80 | 100000000, 81 | 10000000, 82 | 1000000, 83 | 100000, 84 | 10000, 85 | 1000, 86 | 100, 87 | 10, 88 | 1, 89 | } 90 | 91 | func TimeToDateTime64(t time.Time, precision int) DateTime64 { 92 | if t.Unix() <= minDateTime64 { 93 | return DateTime64(minDateTime64) 94 | } 95 | return DateTime64(t.UnixNano() / precisionFactor[precision]) 96 | } 97 | 98 | func (d DateTime64) FromTime(v time.Time, precision int) DateTime64 { 99 | return TimeToDateTime64(v, precision) 100 | } 101 | 102 | func (d DateTime64) ToTime(loc *time.Location, precision int) time.Time { 103 | if d == 0 { 104 | return time.Time{} 105 | } 106 | nsec := int64(d) * precisionFactor[precision] 107 | return time.Unix(nsec/1e9, nsec%1e9).In(loc) 108 | } 109 | -------------------------------------------------------------------------------- /chpool/stat.go: -------------------------------------------------------------------------------- 1 | package chpool 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/jackc/puddle/v2" 7 | ) 8 | 9 | // Stat is a snapshot of Pool statistics. 10 | type Stat struct { 11 | s *puddle.Stat 12 | newConnsCount int64 13 | lifetimeDestroyCount int64 14 | idleDestroyCount int64 15 | } 16 | 17 | // AcquireCount returns the cumulative count of successful acquires from the pool. 18 | func (s *Stat) AcquireCount() int64 { 19 | return s.s.AcquireCount() 20 | } 21 | 22 | // AcquireDuration returns the total duration of all successful acquires from 23 | // the pool. 24 | func (s *Stat) AcquireDuration() time.Duration { 25 | return s.s.AcquireDuration() 26 | } 27 | 28 | // AcquiredConns returns the number of currently acquired connections in the pool. 29 | func (s *Stat) AcquiredConns() int32 { 30 | return s.s.AcquiredResources() 31 | } 32 | 33 | // CanceledAcquireCount returns the cumulative count of acquires from the pool 34 | // that were canceled by a context. 35 | func (s *Stat) CanceledAcquireCount() int64 { 36 | return s.s.CanceledAcquireCount() 37 | } 38 | 39 | // ConstructingConns returns the number of conns with construction in progress in 40 | // the pool. 41 | func (s *Stat) ConstructingConns() int32 { 42 | return s.s.ConstructingResources() 43 | } 44 | 45 | // EmptyAcquireCount returns the cumulative count of successful acquires from the pool 46 | // that waited for a resource to be released or constructed because the pool was 47 | // empty. 48 | func (s *Stat) EmptyAcquireCount() int64 { 49 | return s.s.EmptyAcquireCount() 50 | } 51 | 52 | // IdleConns returns the number of currently idle conns in the pool. 53 | func (s *Stat) IdleConns() int32 { 54 | return s.s.IdleResources() 55 | } 56 | 57 | // MaxConns returns the maximum size of the pool. 58 | func (s *Stat) MaxConns() int32 { 59 | return s.s.MaxResources() 60 | } 61 | 62 | // TotalConns returns the total number of resources currently in the pool. 63 | // The value is the sum of ConstructingConns, AcquiredConns, and 64 | // IdleConns. 65 | func (s *Stat) TotalConns() int32 { 66 | return s.s.TotalResources() 67 | } 68 | 69 | // NewConnsCount returns the cumulative count of new connections opened. 70 | func (s *Stat) NewConnsCount() int64 { 71 | return s.newConnsCount 72 | } 73 | 74 | // MaxLifetimeDestroyCount returns the cumulative count of connections destroyed 75 | // because they exceeded MaxConnLifetime. 76 | func (s *Stat) MaxLifetimeDestroyCount() int64 { 77 | return s.lifetimeDestroyCount 78 | } 79 | 80 | // MaxIdleDestroyCount returns the cumulative count of connections destroyed because 81 | // they exceeded MaxConnIdleTime. 82 | func (s *Stat) MaxIdleDestroyCount() int64 { 83 | return s.idleDestroyCount 84 | } 85 | -------------------------------------------------------------------------------- /block_test.go: -------------------------------------------------------------------------------- 1 | package chconn 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "io" 7 | "os" 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | func TestBlockReadError(t *testing.T) { 15 | startValidReader := 15 16 | 17 | tests := []struct { 18 | name string 19 | wantErr string 20 | numberValid int 21 | }{ 22 | { 23 | name: "blockInfo: temporary table", 24 | wantErr: "block: temporary table", 25 | numberValid: startValidReader - 1, 26 | }, { 27 | name: "blockInfo: read field1", 28 | wantErr: "blockInfo: read field1", 29 | numberValid: startValidReader, 30 | }, { 31 | name: "blockInfo: read isOverflows", 32 | wantErr: "blockInfo: read isOverflows", 33 | numberValid: startValidReader + 1, 34 | }, { 35 | name: "blockInfo: read field2", 36 | wantErr: "blockInfo: read field2", 37 | numberValid: startValidReader + 2, 38 | }, { 39 | name: "blockInfo: read bucketNum", 40 | wantErr: "blockInfo: read bucketNum", 41 | numberValid: startValidReader + 3, 42 | }, { 43 | name: "blockInfo: read num3", 44 | wantErr: "blockInfo: read num3", 45 | numberValid: startValidReader + 4, 46 | }, { 47 | name: "block: read NumColumns", 48 | wantErr: "block: read NumColumns", 49 | numberValid: startValidReader + 5, 50 | }, { 51 | name: "block: read NumRows", 52 | wantErr: "block: read NumRows", 53 | numberValid: startValidReader + 6, 54 | }, { 55 | name: "block: read column name", 56 | wantErr: "block: read column name", 57 | numberValid: startValidReader + 8, 58 | }, { 59 | name: "block: read column type", 60 | wantErr: "block: read column type", 61 | numberValid: startValidReader + 10, 62 | }, 63 | } 64 | for _, tt := range tests { 65 | t.Run(tt.name, func(t *testing.T) { 66 | config, err := ParseConfig(os.Getenv("CHX_TEST_TCP_CONN_STRING")) 67 | require.NoError(t, err) 68 | config.ReaderFunc = func(r io.Reader) io.Reader { 69 | return &readErrorHelper{ 70 | err: errors.New("timeout"), 71 | r: r, 72 | numberValid: tt.numberValid, 73 | } 74 | } 75 | 76 | c, err := ConnectConfig(context.Background(), config) 77 | assert.NoError(t, err) 78 | stmt, err := c.Select(context.Background(), "SELECT * FROM system.numbers LIMIT 5;") 79 | require.Error(t, err) 80 | require.Nil(t, stmt) 81 | 82 | readErr, ok := err.(*readError) 83 | require.True(t, ok) 84 | require.Equal(t, readErr.msg, tt.wantErr) 85 | require.EqualError(t, readErr.Unwrap(), "timeout") 86 | assert.True(t, c.IsClosed()) 87 | }) 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /client_info.go: -------------------------------------------------------------------------------- 1 | package chconn 2 | 3 | import ( 4 | "os/user" 5 | 6 | "github.com/vahid-sohrabloo/chconn/v2/internal/helper" 7 | ) 8 | 9 | // ClientInfo Information about client for query. 10 | // Some fields are passed explicitly from client and some are calculated automatically. 11 | // Contains info about initial query source, for tracing distributed queries 12 | // where one query initiates many other queries. 13 | type ClientInfo struct { 14 | InitialUser string 15 | InitialQueryID string 16 | 17 | OSUser string 18 | ClientHostname string 19 | ClientName string 20 | 21 | ClientVersionMajor uint64 22 | ClientVersionMinor uint64 23 | ClientVersionPatch uint64 24 | ClientRevision uint64 25 | DistributedDepth uint64 26 | 27 | QuotaKey string 28 | } 29 | 30 | // Write Only values that are not calculated automatically or passed separately are serialized. 31 | // Revisions are passed to use format that server will understand or client was used. 32 | func (c *ClientInfo) write(ch *conn) { 33 | // InitialQuery 34 | ch.writer.Uint8(1) 35 | 36 | ch.writer.String(c.InitialUser) 37 | ch.writer.String(c.InitialQueryID) 38 | 39 | ch.writer.String("[::ffff:127.0.0.1]:0") 40 | 41 | if ch.serverInfo.Revision >= helper.DbmsMinProtocolVersionWithInitialQueryStartTime { 42 | ch.writer.Uint64(0) 43 | } 44 | 45 | // iface type 46 | ch.writer.Uint8(1) // tcp 47 | ch.writer.String(c.OSUser) 48 | ch.writer.String(c.ClientHostname) 49 | ch.writer.String(c.ClientName) 50 | ch.writer.Uvarint(c.ClientVersionMajor) 51 | ch.writer.Uvarint(c.ClientVersionMinor) 52 | ch.writer.Uvarint(c.ClientRevision) 53 | 54 | if ch.serverInfo.Revision >= helper.DbmsMinRevisionWithQuotaKeyInClientInfo { 55 | ch.writer.String(c.QuotaKey) 56 | } 57 | 58 | if ch.serverInfo.Revision >= helper.DbmsMinProtocolVersionWithDistributedDepth { 59 | ch.writer.Uvarint(c.DistributedDepth) 60 | } 61 | 62 | if ch.serverInfo.Revision >= helper.DbmsMinRevisionWithVersionPatch { 63 | ch.writer.Uvarint(c.ClientVersionPatch) 64 | } 65 | 66 | if ch.serverInfo.Revision >= helper.DbmsMinRevisionWithOpenTelemetry { 67 | ch.writer.Uint8(0) 68 | } 69 | 70 | if ch.serverInfo.Revision >= helper.DbmsMinProtocolVersionWithParallelReplicas { 71 | ch.writer.Uvarint(0) // collaborate_with_initiator 72 | ch.writer.Uvarint(0) // count_participating_replicas 73 | ch.writer.Uvarint(0) // number_of_current_replica 74 | } 75 | } 76 | 77 | func (c *ClientInfo) fillOSUserHostNameAndVersionInfo() { 78 | u, err := user.Current() 79 | if err == nil { 80 | c.OSUser = u.Username 81 | } 82 | 83 | c.ClientVersionMajor = dbmsVersionMajor 84 | c.ClientVersionMinor = dbmsVersionMinor 85 | c.ClientVersionPatch = dbmsVersionPatch 86 | c.ClientRevision = dbmsVersionRevision 87 | } 88 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 2 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 3 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= 5 | github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw= 6 | github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= 7 | github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 8 | github.com/jackc/puddle/v2 v2.1.2 h1:0f7vaaXINONKTsxYDn4otOAiJanX/BMeAtY//BXqzlg= 9 | github.com/jackc/puddle/v2 v2.1.2/go.mod h1:2lpufsF5mRHO6SuZkm0fNYxM6SWHfvyFj62KwNzgels= 10 | github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw= 11 | github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4= 12 | github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= 13 | github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= 14 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 15 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 16 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 17 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 18 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 19 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 20 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 21 | github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= 22 | github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 23 | go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= 24 | go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= 25 | golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7 h1:ZrnxWX62AgTKOSagEqxvb3ffipvEDX2pl7E1TdqLqIc= 26 | golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 27 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 28 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 29 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 30 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 31 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 32 | -------------------------------------------------------------------------------- /column/array.go: -------------------------------------------------------------------------------- 1 | package column 2 | 3 | // Array is a column of Array(T) ClickHouse data type 4 | type Array[T any] struct { 5 | ArrayBase 6 | columnData []T 7 | } 8 | 9 | // NewArray create a new array column of Array(T) ClickHouse data type 10 | func NewArray[T any](dataColumn Column[T]) *Array[T] { 11 | a := &Array[T]{ 12 | ArrayBase: ArrayBase{ 13 | dataColumn: dataColumn, 14 | offsetColumn: New[uint64](), 15 | }, 16 | } 17 | a.resetHook = func() { 18 | a.columnData = a.columnData[:0] 19 | } 20 | return a 21 | } 22 | 23 | // Data get all the data in current block as a slice. 24 | func (c *Array[T]) Data() [][]T { 25 | values := make([][]T, c.offsetColumn.numRow) 26 | offsets := c.Offsets() 27 | var lastOffset uint64 28 | columnData := c.getColumnData() 29 | for i, offset := range offsets { 30 | val := make([]T, offset-lastOffset) 31 | copy(val, columnData[lastOffset:offset]) 32 | values[i] = val 33 | lastOffset = offset 34 | } 35 | return values 36 | } 37 | 38 | // Read reads all the data in current block and append to the input. 39 | func (c *Array[T]) Read(value [][]T) [][]T { 40 | offsets := c.Offsets() 41 | var lastOffset uint64 42 | columnData := c.getColumnData() 43 | for _, offset := range offsets { 44 | val := make([]T, offset-lastOffset) 45 | copy(val, columnData[lastOffset:offset]) 46 | value = append(value, val) 47 | lastOffset = offset 48 | } 49 | return value 50 | } 51 | 52 | // Row return the value of given row. 53 | // NOTE: Row number start from zero 54 | func (c *Array[T]) Row(row int) []T { 55 | var lastOffset uint64 56 | if row != 0 { 57 | lastOffset = c.offsetColumn.Row(row - 1) 58 | } 59 | var val []T 60 | val = append(val, c.getColumnData()[lastOffset:c.offsetColumn.Row(row)]...) 61 | return val 62 | } 63 | 64 | // Append value for insert 65 | func (c *Array[T]) Append(v ...[]T) { 66 | for _, v := range v { 67 | c.AppendLen(len(v)) 68 | c.dataColumn.(Column[T]).Append(v...) 69 | } 70 | } 71 | 72 | // Append single item value for insert 73 | // 74 | // it should use with AppendLen 75 | // 76 | // Example: 77 | // 78 | // c.AppendLen(2) // insert 2 items 79 | // c.AppendItem(1, 2) 80 | func (c *Array[T]) AppendItem(v ...T) { 81 | c.dataColumn.(Column[T]).Append(v...) 82 | } 83 | 84 | // Array return a Array type for this column 85 | func (c *Array[T]) Array() *Array2[T] { 86 | return NewArray2(c) 87 | } 88 | 89 | func (c *Array[T]) getColumnData() []T { 90 | if len(c.columnData) == 0 { 91 | c.columnData = c.dataColumn.(Column[T]).Data() 92 | } 93 | return c.columnData 94 | } 95 | 96 | func (c *Array[T]) elem(arrayLevel int) ColumnBasic { 97 | if arrayLevel > 0 { 98 | return c.Array().elem(arrayLevel - 1) 99 | } 100 | return c 101 | } 102 | -------------------------------------------------------------------------------- /column/tuple2_gen.go: -------------------------------------------------------------------------------- 1 | package column 2 | 3 | import ( 4 | "unsafe" 5 | ) 6 | 7 | type tuple2Value[T1, T2 any] struct { 8 | Col1 T1 9 | Col2 T2 10 | } 11 | 12 | // Tuple2 is a column of Tuple(T1, T2) ClickHouse data type 13 | type Tuple2[T ~struct { 14 | Col1 T1 15 | Col2 T2 16 | }, T1, T2 any] struct { 17 | Tuple 18 | col1 Column[T1] 19 | col2 Column[T2] 20 | } 21 | 22 | // NewTuple2 create a new tuple of Tuple(T1, T2) ClickHouse data type 23 | func NewTuple2[T ~struct { 24 | Col1 T1 25 | Col2 T2 26 | }, T1, T2 any]( 27 | column1 Column[T1], 28 | column2 Column[T2], 29 | ) *Tuple2[T, T1, T2] { 30 | return &Tuple2[T, T1, T2]{ 31 | Tuple: Tuple{ 32 | columns: []ColumnBasic{ 33 | column1, 34 | column2, 35 | }, 36 | }, 37 | col1: column1, 38 | col2: column2, 39 | } 40 | } 41 | 42 | // NewNested2 create a new nested of Nested(T1, T2) ClickHouse data type 43 | // 44 | // this is actually an alias for NewTuple2(T1, T2).Array() 45 | func NewNested2[T ~struct { 46 | Col1 T1 47 | Col2 T2 48 | }, T1, T2 any]( 49 | column1 Column[T1], 50 | column2 Column[T2], 51 | ) *Array[T] { 52 | return NewTuple2[T]( 53 | column1, 54 | column2, 55 | ).Array() 56 | } 57 | 58 | // Data get all the data in current block as a slice. 59 | func (c *Tuple2[T, T1, T2]) Data() []T { 60 | val := make([]T, c.NumRow()) 61 | for i := 0; i < c.NumRow(); i++ { 62 | val[i] = T(tuple2Value[T1, T2]{ 63 | Col1: c.col1.Row(i), 64 | Col2: c.col2.Row(i), 65 | }) 66 | } 67 | return val 68 | } 69 | 70 | // Read reads all the data in current block and append to the input. 71 | func (c *Tuple2[T, T1, T2]) Read(value []T) []T { 72 | valTuple := *(*[]tuple2Value[T1, T2])(unsafe.Pointer(&value)) 73 | if cap(valTuple)-len(valTuple) >= c.NumRow() { 74 | valTuple = valTuple[:len(value)+c.NumRow()] 75 | } else { 76 | valTuple = append(valTuple, make([]tuple2Value[T1, T2], c.NumRow())...) 77 | } 78 | 79 | val := valTuple[len(valTuple)-c.NumRow():] 80 | for i := 0; i < c.NumRow(); i++ { 81 | val[i].Col1 = c.col1.Row(i) 82 | val[i].Col2 = c.col2.Row(i) 83 | } 84 | return *(*[]T)(unsafe.Pointer(&valTuple)) 85 | } 86 | 87 | // Row return the value of given row. 88 | // NOTE: Row number start from zero 89 | func (c *Tuple2[T, T1, T2]) Row(row int) T { 90 | return T(tuple2Value[T1, T2]{ 91 | Col1: c.col1.Row(row), 92 | Col2: c.col2.Row(row), 93 | }) 94 | } 95 | 96 | // Append value for insert 97 | func (c *Tuple2[T, T1, T2]) Append(v ...T) { 98 | for _, v := range v { 99 | t := tuple2Value[T1, T2](v) 100 | c.col1.Append(t.Col1) 101 | c.col2.Append(t.Col2) 102 | } 103 | } 104 | 105 | // Array return a Array type for this column 106 | func (c *Tuple2[T, T1, T2]) Array() *Array[T] { 107 | return NewArray[T](c) 108 | } 109 | -------------------------------------------------------------------------------- /column/map_nullable.go: -------------------------------------------------------------------------------- 1 | package column 2 | 3 | import "github.com/vahid-sohrabloo/chconn/v2/internal/readerwriter" 4 | 5 | // MapNullable is a column of Map(K,V) ClickHouse data type where V is nullable. 6 | // Map in clickhouse actually is a array of pair(K,V) 7 | type MapNullable[K comparable, V any] struct { 8 | Map[K, V] 9 | valueColumn NullableColumn[V] 10 | keyColumnData []K 11 | valueColumnData []*V 12 | } 13 | 14 | // NewMapNullable create a new map column of Map(K,V) ClickHouse data type 15 | func NewMapNullable[K comparable, V any]( 16 | keyColumn Column[K], 17 | valueColumn NullableColumn[V], 18 | ) *MapNullable[K, V] { 19 | a := &MapNullable[K, V]{ 20 | valueColumn: valueColumn, 21 | Map: Map[K, V]{ 22 | MapBase: MapBase{ 23 | keyColumn: keyColumn, 24 | valueColumn: valueColumn, 25 | offsetColumn: New[uint64](), 26 | }, 27 | }, 28 | } 29 | return a 30 | } 31 | 32 | // Data get all the data in current block as a slice. 33 | func (c *MapNullable[T, V]) DataP() []map[T]*V { 34 | values := make([]map[T]*V, c.offsetColumn.numRow) 35 | var lastOffset uint64 36 | for i := 0; i < c.offsetColumn.numRow; i++ { 37 | val := make(map[T]*V) 38 | offset := c.offsetColumn.Row(i) 39 | for ki, key := range c.keyColumnData[lastOffset:offset] { 40 | v := c.valueColumnData[lastOffset:offset][ki] 41 | val[key] = v 42 | } 43 | values[i] = val 44 | lastOffset = c.offsetColumn.Row(i) 45 | } 46 | return values 47 | } 48 | 49 | // Read reads all the data in current block and append to column. 50 | func (c *MapNullable[T, V]) ReadP(value []map[T]*V) []map[T]*V { 51 | return append(value, c.DataP()...) 52 | } 53 | 54 | // Row return the value of given row. 55 | // NOTE: Row number start from zero 56 | func (c *MapNullable[T, V]) RowP(row int) map[T]*V { 57 | var lastOffset uint64 58 | if row != 0 { 59 | lastOffset = c.offsetColumn.Row(row - 1) 60 | } 61 | val := make(map[T]*V) 62 | offset := c.offsetColumn.Row(row) 63 | for ki, key := range c.keyColumnData[lastOffset:offset] { 64 | v := c.valueColumnData[lastOffset:offset][ki] 65 | val[key] = v 66 | } 67 | return val 68 | } 69 | 70 | func (c *MapNullable[K, V]) AppendP(v map[K]*V) { 71 | c.AppendLen(len(v)) 72 | for k, d := range v { 73 | c.keyColumn.(Column[K]).Append(k) 74 | c.valueColumn.AppendP(d) 75 | } 76 | } 77 | 78 | // ReadRaw read raw data from the reader. it runs automatically 79 | func (c *MapNullable[K, V]) ReadRaw(num int, r *readerwriter.Reader) error { 80 | err := c.Map.ReadRaw(num, r) 81 | if err != nil { 82 | return err 83 | } 84 | 85 | c.keyColumnData = c.keyColumn.(Column[K]).Data() 86 | c.valueColumnData = c.valueColumn.DataP() 87 | 88 | return nil 89 | } 90 | 91 | // ValueColumn return the value column 92 | func (c *MapNullable[K, V]) ValueColumn() NullableColumn[V] { 93 | return c.valueColumn 94 | } 95 | -------------------------------------------------------------------------------- /internal/readerwriter/reader.go: -------------------------------------------------------------------------------- 1 | package readerwriter 2 | 3 | import ( 4 | "encoding/binary" 5 | "io" 6 | ) 7 | 8 | // Reader is a helper to read data from reader 9 | type Reader struct { 10 | mainReader io.Reader 11 | input io.Reader 12 | compressReader io.Reader 13 | scratch [binary.MaxVarintLen64]byte 14 | } 15 | 16 | // NewReader get new Reader 17 | func NewReader(input io.Reader) *Reader { 18 | return &Reader{ 19 | input: input, 20 | mainReader: input, 21 | } 22 | } 23 | 24 | // SetCompress set compress statusp 25 | func (r *Reader) SetCompress(c bool) { 26 | if c { 27 | if r.compressReader == nil { 28 | r.compressReader = NewCompressReader(r.mainReader) 29 | } 30 | r.input = r.compressReader 31 | return 32 | } 33 | r.input = r.mainReader 34 | } 35 | 36 | // Uvarint read variable uint64 value 37 | func (r *Reader) Uvarint() (uint64, error) { 38 | return binary.ReadUvarint(r) 39 | } 40 | 41 | // Int32 read Int32 value 42 | func (r *Reader) Int32() (int32, error) { 43 | v, err := r.Uint32() 44 | if err != nil { 45 | return 0, err 46 | } 47 | return int32(v), nil 48 | } 49 | 50 | // Uint32 read Uint32 value 51 | func (r *Reader) Uint32() (uint32, error) { 52 | if _, err := io.ReadFull(r.input, r.scratch[:4]); err != nil { 53 | return 0, err 54 | } 55 | return binary.LittleEndian.Uint32(r.scratch[:4]), nil 56 | } 57 | 58 | // Uint64 read Uint64 value 59 | func (r *Reader) Uint64() (uint64, error) { 60 | if _, err := io.ReadFull(r.input, r.scratch[:8]); err != nil { 61 | return 0, err 62 | } 63 | return binary.LittleEndian.Uint64(r.scratch[:8]), nil 64 | } 65 | 66 | // FixedString read FixedString value 67 | func (r *Reader) FixedString(strlen int) ([]byte, error) { 68 | buf := make([]byte, strlen) 69 | 70 | _, err := io.ReadFull(r, buf) 71 | return buf, err 72 | } 73 | 74 | // String read String value 75 | func (r *Reader) String() (string, error) { 76 | strlen, err := r.Uvarint() 77 | if err != nil { 78 | return "", err 79 | } 80 | str, err := r.FixedString(int(strlen)) 81 | if err != nil { 82 | return "", err 83 | } 84 | return string(str), nil 85 | } 86 | 87 | // ByteString read string value as []byte 88 | func (r *Reader) ByteString() ([]byte, error) { 89 | strlen, err := r.Uvarint() 90 | if err != nil { 91 | return nil, err 92 | } 93 | if strlen == 0 { 94 | return []byte{}, nil 95 | } 96 | return r.FixedString(int(strlen)) 97 | } 98 | 99 | // ReadByte read a single byte 100 | func (r *Reader) ReadByte() (byte, error) { 101 | if _, err := r.input.Read(r.scratch[:1]); err != nil { 102 | return 0, err 103 | } 104 | return r.scratch[0], nil 105 | } 106 | 107 | // Read implement Read 108 | func (r *Reader) Read(buf []byte) (int, error) { 109 | return io.ReadFull(r.input, buf) 110 | } 111 | -------------------------------------------------------------------------------- /ping_test.go: -------------------------------------------------------------------------------- 1 | package chconn 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "io" 7 | "os" 8 | "testing" 9 | "time" 10 | 11 | "github.com/stretchr/testify/assert" 12 | "github.com/stretchr/testify/require" 13 | ) 14 | 15 | func TestPing(t *testing.T) { 16 | t.Parallel() 17 | 18 | connString := os.Getenv("CHX_TEST_TCP_CONN_STRING") 19 | 20 | conn, err := Connect(context.Background(), connString) 21 | require.NoError(t, err) 22 | require.NoError(t, conn.Ping(context.Background())) 23 | conn.Close() 24 | } 25 | 26 | func TestPingWriteError(t *testing.T) { 27 | t.Parallel() 28 | 29 | connString := os.Getenv("CHX_TEST_TCP_CONN_STRING") 30 | 31 | config, err := ParseConfig(connString) 32 | require.NoError(t, err) 33 | 34 | config.WriterFunc = func(w io.Writer) io.Writer { 35 | return &writerErrorHelper{ 36 | err: errors.New("timeout"), 37 | w: w, 38 | numberValid: 1, 39 | } 40 | } 41 | c, err := ConnectConfig(context.Background(), config) 42 | require.NoError(t, err) 43 | err = c.Ping(context.Background()) 44 | require.EqualError(t, err, "ping: write packet type (timeout)") 45 | require.EqualError(t, errors.Unwrap(err), "timeout") 46 | 47 | assert.True(t, c.IsClosed()) 48 | 49 | config.WriterFunc = nil 50 | 51 | config.ReaderFunc = func(r io.Reader) io.Reader { 52 | return &readErrorHelper{ 53 | err: errors.New("timeout"), 54 | r: r, 55 | numberValid: 13, 56 | } 57 | } 58 | 59 | c, err = ConnectConfig(context.Background(), config) 60 | require.NoError(t, err) 61 | 62 | require.EqualError(t, c.Ping(context.Background()), "packet: read packet type (timeout)") 63 | assert.True(t, c.IsClosed()) 64 | } 65 | 66 | func TestPingCtxError(t *testing.T) { 67 | t.Parallel() 68 | 69 | connString := os.Getenv("CHX_TEST_TCP_CONN_STRING") 70 | 71 | config, err := ParseConfig(connString) 72 | require.NoError(t, err) 73 | 74 | c, err := ConnectConfig(context.Background(), config) 75 | require.NoError(t, err) 76 | 77 | ctx, cancel := context.WithCancel(context.Background()) 78 | cancel() 79 | 80 | err = c.Ping(ctx) 81 | require.EqualError(t, err, "timeout: context already done: context canceled") 82 | require.EqualError(t, errors.Unwrap(err), "context already done: context canceled") 83 | 84 | assert.False(t, c.IsClosed()) 85 | 86 | config.WriterFunc = func(w io.Writer) io.Writer { 87 | return &writerSlowHelper{ 88 | w: w, 89 | sleep: time.Second, 90 | } 91 | } 92 | c, err = ConnectConfig(context.Background(), config) 93 | require.NoError(t, err) 94 | ctx, cancel = context.WithTimeout(context.Background(), time.Millisecond*50) 95 | defer cancel() 96 | err = c.Ping(ctx) 97 | require.EqualError(t, errors.Unwrap(errors.Unwrap(err)), "context deadline exceeded") 98 | assert.True(t, c.IsClosed()) 99 | } 100 | -------------------------------------------------------------------------------- /types/uint128.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "math" 5 | "math/big" 6 | ) 7 | 8 | // Note, Zero and Max are functions just to make read-only values. 9 | // We cannot define constants for structures, and global variables 10 | // are unacceptable because it will be possible to change them. 11 | 12 | // Zero is the lowest possible Uint128 value. 13 | func Uint128Zero() Uint128 { 14 | return Uint128From64(0) 15 | } 16 | 17 | // Max is the largest possible Uint128 value. 18 | func Uint128Max() Uint128 { 19 | return Uint128{ 20 | Lo: math.MaxUint64, 21 | Hi: math.MaxUint64, 22 | } 23 | } 24 | 25 | // Uint128 is an unsigned 128-bit number. 26 | // All methods are immutable, works just like standard uint64. 27 | type Uint128 struct { 28 | Lo uint64 // lower 64-bit half 29 | Hi uint64 // upper 64-bit half 30 | } 31 | 32 | // Note, there in no New(lo, hi) just not to confuse 33 | // which half goes first: lower or upper. 34 | // Use structure initialization Uint128{Lo: ..., Hi: ...} instead. 35 | 36 | // From64 converts 64-bit value v to a Uint128 value. 37 | // Upper 64-bit half will be zero. 38 | func Uint128From64(v uint64) Uint128 { 39 | return Uint128{Lo: v} 40 | } 41 | 42 | // FromBig converts *big.Int to 128-bit Uint128 value ignoring overflows. 43 | // If input integer is nil or negative then return Zero. 44 | // If input interger overflows 128-bit then return Max. 45 | func Uint128FromBig(i *big.Int) Uint128 { 46 | u, _ := Uint128FromBigEx(i) 47 | return u 48 | } 49 | 50 | // FromBigEx converts *big.Int to 128-bit Uint128 value (eXtended version). 51 | // Provides ok successful flag as a second return value. 52 | // If input integer is negative or overflows 128-bit then ok=false. 53 | // If input is nil then zero 128-bit returned. 54 | func Uint128FromBigEx(i *big.Int) (Uint128, bool) { 55 | switch { 56 | case i == nil: 57 | return Uint128Zero(), true // assuming nil === 0 58 | case i.Sign() < 0: 59 | return Uint128Zero(), false // value cannot be negative! 60 | case i.BitLen() > 128: 61 | return Uint128Max(), false // value overflows 128-bit! 62 | } 63 | 64 | // Note, actually result of big.Int.Uint64 is undefined 65 | // if stored value is greater than 2^64 66 | // but we assume that it just gets lower 64 bits. 67 | t := new(big.Int) 68 | lo := i.Uint64() 69 | hi := t.Rsh(i, 64).Uint64() 70 | return Uint128{ 71 | Lo: lo, 72 | Hi: hi, 73 | }, true 74 | } 75 | 76 | // Big returns 128-bit value as a *big.Int. 77 | func (u Uint128) Big() *big.Int { 78 | i := new(big.Int).SetUint64(u.Hi) 79 | i = i.Lsh(i, 64) 80 | i = i.Or(i, new(big.Int).SetUint64(u.Lo)) 81 | return i 82 | } 83 | 84 | // Equals returns true if two 128-bit values are equal. 85 | // Uint128 values can be compared directly with == operator 86 | // but use of the Equals method is preferred for consistency. 87 | func (u Uint128) Equals(v Uint128) bool { 88 | return (u.Lo == v.Lo) && (u.Hi == v.Hi) 89 | } 90 | -------------------------------------------------------------------------------- /column/array3_nullable.go: -------------------------------------------------------------------------------- 1 | package column 2 | 3 | import "github.com/vahid-sohrabloo/chconn/v2/internal/readerwriter" 4 | 5 | // Array is a column of Array(Array(Nullable(T))) ClickHouse data type 6 | type Array3Nullable[T comparable] struct { 7 | Array3[T] 8 | dataColumn *Array2Nullable[T] 9 | columnData [][][]*T 10 | } 11 | 12 | // NewArrayNullable create a new array column of Array(Nullable(T)) ClickHouse data type 13 | func NewArray3Nullable[T comparable](dataColumn *Array2Nullable[T]) *Array3Nullable[T] { 14 | a := &Array3Nullable[T]{ 15 | dataColumn: dataColumn, 16 | Array3: Array3[T]{ 17 | ArrayBase: ArrayBase{ 18 | dataColumn: dataColumn, 19 | offsetColumn: New[uint64](), 20 | }, 21 | }, 22 | } 23 | a.resetHook = func() { 24 | a.columnData = a.columnData[:0] 25 | } 26 | return a 27 | } 28 | 29 | // Data get all the nullable data in current block as a slice of pointer. 30 | func (c *Array3Nullable[T]) DataP() [][][][]*T { 31 | values := make([][][][]*T, c.offsetColumn.numRow) 32 | var lastOffset uint64 33 | columnData := c.getColumnData() 34 | for i := 0; i < c.offsetColumn.numRow; i++ { 35 | values[i] = columnData[lastOffset:c.offsetColumn.Row(i)] 36 | lastOffset = c.offsetColumn.Row(i) 37 | } 38 | return values 39 | } 40 | 41 | // Read reads all the nullable data in current block as a slice pointer and append to the input. 42 | func (c *Array3Nullable[T]) ReadP(value [][][][]*T) [][][][]*T { 43 | var lastOffset uint64 44 | columnData := c.getColumnData() 45 | for i := 0; i < c.offsetColumn.numRow; i++ { 46 | value = append(value, columnData[lastOffset:c.offsetColumn.Row(i)]) 47 | lastOffset = c.offsetColumn.Row(i) 48 | } 49 | return value 50 | } 51 | 52 | // RowP return the nullable value of given row as a pointer 53 | // NOTE: Row number start from zero 54 | func (c *Array3Nullable[T]) RowP(row int) [][][]*T { 55 | var lastOffset uint64 56 | if row != 0 { 57 | lastOffset = c.offsetColumn.Row(row - 1) 58 | } 59 | var val [][][]*T 60 | val = append(val, c.getColumnData()[lastOffset:c.offsetColumn.Row(row)]...) 61 | return val 62 | } 63 | 64 | // AppendP a nullable value for insert 65 | func (c *Array3Nullable[T]) AppendP(v ...[][][]*T) { 66 | for _, v := range v { 67 | c.AppendLen(len(v)) 68 | c.dataColumn.AppendP(v...) 69 | } 70 | } 71 | 72 | // ReadRaw read raw data from the reader. it runs automatically 73 | func (c *Array3Nullable[T]) ReadRaw(num int, r *readerwriter.Reader) error { 74 | err := c.Array3.ReadRaw(num, r) 75 | if err != nil { 76 | return err 77 | } 78 | c.columnData = c.dataColumn.DataP() 79 | return nil 80 | } 81 | 82 | func (c *Array3Nullable[T]) getColumnData() [][][]*T { 83 | if len(c.columnData) == 0 { 84 | c.columnData = c.dataColumn.DataP() 85 | } 86 | return c.columnData 87 | } 88 | 89 | func (c *Array3Nullable[T]) elem(arrayLevel int) ColumnBasic { 90 | if arrayLevel > 0 { 91 | panic("array level is too deep") 92 | } 93 | return c 94 | } 95 | -------------------------------------------------------------------------------- /profile_test.go: -------------------------------------------------------------------------------- 1 | package chconn 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "os" 9 | "testing" 10 | 11 | "github.com/stretchr/testify/assert" 12 | "github.com/stretchr/testify/require" 13 | "github.com/vahid-sohrabloo/chconn/v2/column" 14 | "github.com/vahid-sohrabloo/chconn/v2/internal/helper" 15 | ) 16 | 17 | func TestProfileReadError(t *testing.T) { 18 | startValidReader := 43 19 | config, err := ParseConfig(os.Getenv("CHX_TEST_TCP_CONN_STRING")) 20 | require.NoError(t, err) 21 | c, err := ConnectConfig(context.Background(), config) 22 | require.NoError(t, err) 23 | if c.ServerInfo().Revision >= helper.DbmsMinProtocolWithServerQueryTimeInProgress { 24 | // todo we need to fix this for clickhouse 22.10 and above 25 | return 26 | } 27 | 28 | tests := []struct { 29 | name string 30 | wantErr string 31 | numberValid int 32 | }{ 33 | { 34 | name: "profile: read Rows", 35 | wantErr: "profile: read Rows", 36 | numberValid: startValidReader, 37 | }, { 38 | name: "profile: read Blocks", 39 | wantErr: "profile: read Blocks", 40 | numberValid: startValidReader + 1, 41 | }, { 42 | name: "profile: read Bytes", 43 | wantErr: "profile: read Bytes", 44 | numberValid: startValidReader + 2, 45 | }, { 46 | name: "profile: read AppliedLimit", 47 | wantErr: "profile: read AppliedLimit", 48 | numberValid: startValidReader + 3, 49 | }, { 50 | name: "profile: read RowsBeforeLimit", 51 | wantErr: "profile: read RowsBeforeLimit", 52 | numberValid: startValidReader + 4, 53 | }, { 54 | name: "profile: read CalculatedRowsBeforeLimit", 55 | wantErr: "profile: read CalculatedRowsBeforeLimit", 56 | numberValid: startValidReader + 5, 57 | }, 58 | } 59 | for _, tt := range tests { 60 | t.Run(tt.name, func(t *testing.T) { 61 | config, err := ParseConfig(os.Getenv("CHX_TEST_TCP_CONN_STRING")) 62 | require.NoError(t, err) 63 | 64 | if c.ServerInfo().Revision >= helper.DbmsMinProtocolWithServerQueryTimeInProgress { 65 | tt.numberValid++ 66 | } 67 | 68 | config.ReaderFunc = func(r io.Reader) io.Reader { 69 | return &readErrorHelper{ 70 | err: errors.New("timeout"), 71 | r: r, 72 | numberValid: tt.numberValid, 73 | } 74 | } 75 | 76 | c, err := ConnectConfig(context.Background(), config) 77 | require.NoError(t, err) 78 | col := column.New[uint64]() 79 | stmt, err := c.Select(context.Background(), "SELECT * FROM system.numbers LIMIT 1;", col) 80 | require.NoError(t, err) 81 | for stmt.Next() { 82 | } 83 | require.Error(t, stmt.Err()) 84 | readErr, ok := stmt.Err().(*readError) 85 | require.True(t, ok) 86 | fmt.Println("readErr.msg:", readErr.msg) 87 | require.Equal(t, tt.wantErr, readErr.msg) 88 | require.EqualError(t, readErr.Unwrap(), "timeout") 89 | assert.True(t, c.IsClosed()) 90 | }) 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /column/column_helper.go: -------------------------------------------------------------------------------- 1 | package column 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | 7 | "github.com/vahid-sohrabloo/chconn/v2/internal/helper" 8 | "github.com/vahid-sohrabloo/chconn/v2/internal/readerwriter" 9 | ) 10 | 11 | type ColumnBasic interface { 12 | ReadRaw(num int, r *readerwriter.Reader) error 13 | HeaderReader(r *readerwriter.Reader, readColumn bool, revision uint64) error 14 | HeaderWriter(*readerwriter.Writer) 15 | WriteTo(io.Writer) (int64, error) 16 | NumRow() int 17 | Reset() 18 | SetType(v []byte) 19 | Type() []byte 20 | SetName(v []byte) 21 | Name() []byte 22 | Validate() error 23 | ColumnType() string 24 | SetWriteBufferSize(int) 25 | } 26 | 27 | type Column[T any] interface { 28 | ColumnBasic 29 | Data() []T 30 | Read([]T) []T 31 | Row(int) T 32 | Append(...T) 33 | } 34 | 35 | type NullableColumn[T any] interface { 36 | Column[T] 37 | DataP() []*T 38 | ReadP([]*T) []*T 39 | RowP(int) *T 40 | AppendP(...*T) 41 | } 42 | 43 | type column struct { 44 | r *readerwriter.Reader 45 | b []byte 46 | totalByte int 47 | name []byte 48 | chType []byte 49 | parent ColumnBasic 50 | } 51 | 52 | func (c *column) readColumn(readColumn bool, revision uint64) error { 53 | if c.parent != nil || !readColumn { 54 | return nil 55 | } 56 | strLen, err := c.r.Uvarint() 57 | if err != nil { 58 | return fmt.Errorf("read column name length: %w", err) 59 | } 60 | if cap(c.name) < int(strLen) { 61 | c.name = make([]byte, strLen) 62 | } else { 63 | c.name = c.name[:strLen] 64 | } 65 | _, err = c.r.Read(c.name) 66 | if err != nil { 67 | return fmt.Errorf("read column name: %w", err) 68 | } 69 | 70 | strLen, err = c.r.Uvarint() 71 | if err != nil { 72 | return fmt.Errorf("read column type length: %w", err) 73 | } 74 | if cap(c.chType) < int(strLen) { 75 | c.chType = make([]byte, strLen) 76 | } else { 77 | c.chType = c.chType[:strLen] 78 | } 79 | _, err = c.r.Read(c.chType) 80 | if err != nil { 81 | return fmt.Errorf("read column type: %w", err) 82 | } 83 | 84 | if revision >= helper.DbmsMinProtocolWithCustomSerialization { 85 | hasCustomSerialization, err := c.r.ReadByte() 86 | if err != nil { 87 | return fmt.Errorf("read custom serialization: %w", err) 88 | } 89 | // todo check with json object 90 | if hasCustomSerialization == 1 { 91 | return fmt.Errorf("custom serialization not supported") 92 | } 93 | } 94 | 95 | return nil 96 | } 97 | 98 | // Name get name of the column 99 | func (c *column) Name() []byte { 100 | return c.name 101 | } 102 | 103 | // Type get clickhouse type 104 | func (c *column) Type() []byte { 105 | return c.chType 106 | } 107 | 108 | // SetName set name of the column 109 | func (c *column) SetName(v []byte) { 110 | c.name = v 111 | } 112 | 113 | // SetType set clickhouse type 114 | func (c *column) SetType(v []byte) { 115 | c.chType = v 116 | } 117 | -------------------------------------------------------------------------------- /doc_test.go: -------------------------------------------------------------------------------- 1 | package chconn_test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "time" 8 | 9 | "github.com/vahid-sohrabloo/chconn/v2/chpool" 10 | "github.com/vahid-sohrabloo/chconn/v2/column" 11 | ) 12 | 13 | func Example() { 14 | conn, err := chpool.New(os.Getenv("DATABASE_URL")) 15 | if err != nil { 16 | panic(err) 17 | } 18 | 19 | defer conn.Close() 20 | 21 | // to check if the connection is alive 22 | err = conn.Ping(context.Background()) 23 | if err != nil { 24 | panic(err) 25 | } 26 | 27 | err = conn.Exec(context.Background(), `DROP TABLE IF EXISTS example_table`) 28 | if err != nil { 29 | panic(err) 30 | } 31 | 32 | err = conn.Exec(context.Background(), `CREATE TABLE example_table ( 33 | uint64 UInt64, 34 | uint64_nullable Nullable(UInt64) 35 | ) Engine=Memory`) 36 | if err != nil { 37 | panic(err) 38 | } 39 | 40 | col1 := column.New[uint64]() 41 | col2 := column.New[uint64]().Nullable() 42 | rows := 1_000_0000 // One hundred million rows- insert in 10 times 43 | numInsert := 10 44 | col1.SetWriteBufferSize(rows) 45 | col2.SetWriteBufferSize(rows) 46 | startInsert := time.Now() 47 | for i := 0; i < numInsert; i++ { 48 | col1.Reset() 49 | col2.Reset() 50 | for y := 0; y < rows; y++ { 51 | col1.Append(uint64(i)) 52 | if i%2 == 0 { 53 | col2.Append(uint64(i)) 54 | } else { 55 | col2.AppendNil() 56 | } 57 | } 58 | 59 | ctxInsert, cancelInsert := context.WithTimeout(context.Background(), time.Second*30) 60 | // insert data 61 | err = conn.Insert(ctxInsert, "INSERT INTO example_table (uint64,uint64_nullable) VALUES", col1, col2) 62 | if err != nil { 63 | cancelInsert() 64 | panic(err) 65 | } 66 | cancelInsert() 67 | } 68 | fmt.Println("inserted 10M rows in ", time.Since(startInsert)) 69 | 70 | // select data 71 | col1Read := column.New[uint64]() 72 | col2Read := column.New[uint64]().Nullable() 73 | 74 | ctxSelect, cancelSelect := context.WithTimeout(context.Background(), time.Second*30) 75 | defer cancelSelect() 76 | 77 | startSelect := time.Now() 78 | selectStmt, err := conn.Select(ctxSelect, "SELECT uint64,uint64_nullable FROM example_table", col1Read, col2Read) 79 | if err != nil { 80 | panic(err) 81 | } 82 | 83 | // make sure the stmt close after select. but it's not necessary 84 | defer selectStmt.Close() 85 | 86 | var col1Data []uint64 87 | var col2DataNil []bool 88 | var col2Data []uint64 89 | // read data block by block 90 | // for more information about block, see: https://clickhouse.com/docs/en/development/architecture/#block 91 | for selectStmt.Next() { 92 | col1Data = col1Data[:0] 93 | col1Data = col1Read.Read(col1Data) 94 | 95 | col2DataNil = col2DataNil[:0] 96 | col2DataNil = col2Read.ReadNil(col2DataNil) 97 | 98 | col2Data = col2Data[:0] 99 | col2Data = col2Read.Read(col2Data) 100 | } 101 | 102 | // check errors 103 | if selectStmt.Err() != nil { 104 | panic(selectStmt.Err()) 105 | } 106 | fmt.Println("selected 10M rows in ", time.Since(startSelect)) 107 | } 108 | -------------------------------------------------------------------------------- /column/map.go: -------------------------------------------------------------------------------- 1 | package column 2 | 3 | // Map is a column of Map(K,V) ClickHouse data type 4 | // Map in clickhouse actually is a array of pair(K,V) 5 | type Map[K comparable, V any] struct { 6 | MapBase 7 | keyColumnData []K 8 | valueColumnData []V 9 | } 10 | 11 | // NewMap create a new map column of Map(K,V) ClickHouse data type 12 | func NewMap[K comparable, V any]( 13 | keyColumn Column[K], 14 | valueColumn Column[V], 15 | ) *Map[K, V] { 16 | a := &Map[K, V]{ 17 | MapBase: MapBase{ 18 | keyColumn: keyColumn, 19 | valueColumn: valueColumn, 20 | offsetColumn: New[uint64](), 21 | }, 22 | } 23 | a.resetHook = func() { 24 | a.keyColumnData = a.keyColumnData[:0] 25 | a.valueColumnData = a.valueColumnData[:0] 26 | } 27 | return a 28 | } 29 | 30 | // Data get all the data in current block as a slice. 31 | func (c *Map[K, V]) Data() []map[K]V { 32 | values := make([]map[K]V, c.offsetColumn.numRow) 33 | offsets := c.Offsets() 34 | if len(offsets) == 0 { 35 | return values 36 | } 37 | keyColumnData := c.getKeyColumnData() 38 | valueColumnData := c.getValueColumnData() 39 | var lastOffset uint64 40 | for i, offset := range offsets { 41 | val := make(map[K]V) 42 | for ki, key := range keyColumnData[lastOffset:offset] { 43 | val[key] = valueColumnData[lastOffset:offset][ki] 44 | } 45 | values[i] = val 46 | lastOffset = offset 47 | } 48 | return values 49 | } 50 | 51 | // Read reads all the data in current block and append to the input. 52 | func (c *Map[K, V]) Read(value []map[K]V) []map[K]V { 53 | return append(value, c.Data()...) 54 | } 55 | 56 | // Row return the value of given row. 57 | // NOTE: Row number start from zero 58 | func (c *Map[K, V]) Row(row int) map[K]V { 59 | var lastOffset uint64 60 | if row != 0 { 61 | lastOffset = c.offsetColumn.Row(row - 1) 62 | } 63 | keyColumnData := c.getKeyColumnData() 64 | valueColumnData := c.getValueColumnData() 65 | 66 | val := make(map[K]V) 67 | offset := c.offsetColumn.Row(row) 68 | for ki, key := range keyColumnData[lastOffset:offset] { 69 | val[key] = valueColumnData[lastOffset:offset][ki] 70 | } 71 | return val 72 | } 73 | 74 | // Append value for insert 75 | func (c *Map[K, V]) Append(v map[K]V) { 76 | c.AppendLen(len(v)) 77 | for k, d := range v { 78 | c.keyColumn.(Column[K]).Append(k) 79 | c.valueColumn.(Column[V]).Append(d) 80 | } 81 | } 82 | 83 | func (c *Map[K, V]) getKeyColumnData() []K { 84 | if len(c.keyColumnData) == 0 { 85 | c.keyColumnData = c.keyColumn.(Column[K]).Data() 86 | } 87 | return c.keyColumnData 88 | } 89 | func (c *Map[K, V]) getValueColumnData() []V { 90 | if len(c.valueColumnData) == 0 { 91 | c.valueColumnData = c.valueColumn.(Column[V]).Data() 92 | } 93 | return c.valueColumnData 94 | } 95 | 96 | // KeyColumn return the key column 97 | func (c *Map[K, V]) KeyColumn() Column[K] { 98 | return c.keyColumn.(Column[K]) 99 | } 100 | 101 | // ValueColumn return the value column 102 | func (c *Map[K, V]) ValueColumn() Column[V] { 103 | return c.valueColumn.(Column[V]) 104 | } 105 | -------------------------------------------------------------------------------- /column/array2_nullable.go: -------------------------------------------------------------------------------- 1 | package column 2 | 3 | import "github.com/vahid-sohrabloo/chconn/v2/internal/readerwriter" 4 | 5 | // Array is a column of Array(Array(Nullable(T))) ClickHouse data type 6 | type Array2Nullable[T comparable] struct { 7 | Array2[T] 8 | dataColumn *ArrayNullable[T] 9 | columnData [][]*T 10 | } 11 | 12 | // NewArrayNullable create a new array column of Array(Nullable(T)) ClickHouse data type 13 | func NewArray2Nullable[T comparable](dataColumn *ArrayNullable[T]) *Array2Nullable[T] { 14 | a := &Array2Nullable[T]{ 15 | dataColumn: dataColumn, 16 | Array2: Array2[T]{ 17 | ArrayBase: ArrayBase{ 18 | dataColumn: dataColumn, 19 | offsetColumn: New[uint64](), 20 | }, 21 | }, 22 | } 23 | a.resetHook = func() { 24 | a.columnData = a.columnData[:0] 25 | } 26 | return a 27 | } 28 | 29 | // Data get all the nullable data in current block as a slice of pointer. 30 | func (c *Array2Nullable[T]) DataP() [][][]*T { 31 | values := make([][][]*T, c.offsetColumn.numRow) 32 | var lastOffset uint64 33 | columnData := c.getColumnData() 34 | for i := 0; i < c.offsetColumn.numRow; i++ { 35 | values[i] = columnData[lastOffset:c.offsetColumn.Row(i)] 36 | lastOffset = c.offsetColumn.Row(i) 37 | } 38 | return values 39 | } 40 | 41 | // Read reads all the nullable data in current block as a slice pointer and append to the input. 42 | func (c *Array2Nullable[T]) ReadP(value [][][]*T) [][][]*T { 43 | var lastOffset uint64 44 | columnData := c.getColumnData() 45 | for i := 0; i < c.offsetColumn.numRow; i++ { 46 | value = append(value, columnData[lastOffset:c.offsetColumn.Row(i)]) 47 | lastOffset = c.offsetColumn.Row(i) 48 | } 49 | return value 50 | } 51 | 52 | // RowP return the nullable value of given row as a pointer 53 | // NOTE: Row number start from zero 54 | func (c *Array2Nullable[T]) RowP(row int) [][]*T { 55 | var lastOffset uint64 56 | if row != 0 { 57 | lastOffset = c.offsetColumn.Row(row - 1) 58 | } 59 | var val [][]*T 60 | val = append(val, c.getColumnData()[lastOffset:c.offsetColumn.Row(row)]...) 61 | return val 62 | } 63 | 64 | // AppendP a nullable value for insert 65 | func (c *Array2Nullable[T]) AppendP(v ...[][]*T) { 66 | for _, v := range v { 67 | c.AppendLen(len(v)) 68 | c.dataColumn.AppendP(v...) 69 | } 70 | } 71 | 72 | // ReadRaw read raw data from the reader. it runs automatically 73 | func (c *Array2Nullable[T]) ReadRaw(num int, r *readerwriter.Reader) error { 74 | err := c.Array2.ReadRaw(num, r) 75 | if err != nil { 76 | return err 77 | } 78 | c.columnData = c.dataColumn.DataP() 79 | return nil 80 | } 81 | 82 | // Array return a Array type for this column 83 | func (c *Array2Nullable[T]) Array() *Array3Nullable[T] { 84 | return NewArray3Nullable(c) 85 | } 86 | 87 | func (c *Array2Nullable[T]) getColumnData() [][]*T { 88 | if len(c.columnData) == 0 { 89 | c.columnData = c.dataColumn.DataP() 90 | } 91 | return c.columnData 92 | } 93 | 94 | func (c *Array2Nullable[T]) elem(arrayLevel int) ColumnBasic { 95 | if arrayLevel > 0 { 96 | return c.Array().elem(arrayLevel - 1) 97 | } 98 | return c 99 | } 100 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | linters-settings: 2 | dupl: 3 | threshold: 100 4 | funlen: 5 | lines: 130 6 | statements: 60 7 | goconst: 8 | min-len: 5 9 | min-occurrences: 3 10 | gocritic: 11 | enabled-tags: 12 | - diagnostic 13 | - experimental 14 | - opinionated 15 | - performance 16 | - style 17 | disabled-checks: 18 | - dupImport # https://github.com/go-critic/go-critic/issues/845 19 | - ifElseChain 20 | - octalLiteral 21 | - whyNoLint 22 | - wrapperFunc 23 | gocyclo: 24 | min-complexity: 20 25 | goimports: 26 | local-prefixes: github.com/golangci/golangci-lint 27 | gomnd: 28 | settings: 29 | mnd: 30 | # don't include the "operation" and "assign" 31 | checks: argument,case,condition,return 32 | ignored-numbers: 1000000 33 | 34 | govet: 35 | check-shadowing: false 36 | lll: 37 | line-length: 140 38 | maligned: 39 | suggest-new: true 40 | misspell: 41 | locale: US 42 | nolintlint: 43 | allow-leading-space: true # don't require machine-readable nolint directives (i.e. with no leading space) 44 | allow-unused: false # report any unused nolint directives 45 | require-explanation: false # don't require an explanation for nolint directives 46 | require-specific: false # don't require nolint directives to be specific about which linter is being skipped 47 | 48 | linters: 49 | disable-all: true 50 | enable: 51 | # - bodyclose 52 | - depguard 53 | - dogsled 54 | - dupl 55 | - errcheck 56 | - exportloopref 57 | - funlen 58 | - gochecknoinits 59 | - goconst 60 | - gocritic 61 | - gocyclo 62 | - gofmt 63 | - goimports 64 | - goprintffuncname 65 | - gosec 66 | - gosimple 67 | - govet 68 | - ineffassign 69 | - lll 70 | - misspell 71 | - nakedret 72 | # - noctx 73 | - nolintlint 74 | - staticcheck 75 | - stylecheck 76 | - typecheck 77 | - unconvert 78 | # - unparam 79 | - unused 80 | - whitespace 81 | 82 | # don't enable: 83 | # - asciicheck 84 | # - scopelint 85 | # - gochecknoglobals 86 | # - gocognit 87 | # - godot 88 | # - godox 89 | # - goerr113 90 | # - interfacer 91 | # - maligned 92 | # - nestif 93 | # - prealloc 94 | # - testpackage 95 | # - revive 96 | # - wsl 97 | # - gomnd 98 | 99 | 100 | issues: 101 | # Excluding configuration per-path, per-linter, per-text and per-source 102 | exclude-rules: 103 | - path: _test\.go 104 | linters: 105 | - goconst 106 | - dupl 107 | - funlen 108 | - gocyclo 109 | - gosec 110 | - goerr113 111 | - maligned 112 | - errcheck 113 | - path: cmd/chgogen 114 | linters: 115 | - goconst 116 | - funlen 117 | - gocyclo 118 | - path: _unsafe\.go 119 | linters: 120 | - dupl 121 | - path: main\.go 122 | linters: 123 | - goconst 124 | - gocritic 125 | - dupl # todo fix later 126 | 127 | run: 128 | skip-dirs: 129 | -------------------------------------------------------------------------------- /internal/readerwriter/compress_writer.go: -------------------------------------------------------------------------------- 1 | package readerwriter 2 | 3 | // copy from https://github.com/ClickHouse/ch-go/blob/4cde4e4bec24211c0bcdc6f385f4212d0ad522d9/compress/writer.go 4 | // some changes to compatible with chconn 5 | 6 | import ( 7 | "encoding/binary" 8 | "fmt" 9 | "io" 10 | 11 | "github.com/go-faster/city" 12 | "github.com/klauspost/compress/zstd" 13 | "github.com/pierrec/lz4/v4" 14 | ) 15 | 16 | type compressWriter struct { 17 | writer io.Writer 18 | // data uncompressed 19 | data []byte 20 | // data position 21 | pos int 22 | // data compressed 23 | zdata []byte 24 | // compression method 25 | method CompressMethod 26 | 27 | lz4 *lz4.Compressor 28 | zstd *zstd.Encoder 29 | } 30 | 31 | // NewCompressWriter wrap the io.Writer 32 | func NewCompressWriter(w io.Writer, method byte) io.Writer { 33 | p := &compressWriter{ 34 | writer: w, 35 | method: CompressMethod(method), 36 | data: make([]byte, maxBlockSize), 37 | } 38 | return p 39 | } 40 | 41 | func (cw *compressWriter) Write(buf []byte) (int, error) { 42 | var n int 43 | for len(buf) > 0 { 44 | // Accumulate the data to be compressed. 45 | m := copy(cw.data[cw.pos:], buf) 46 | cw.pos += m 47 | buf = buf[m:] 48 | if cw.pos == len(cw.data) { 49 | err := cw.Flush() 50 | if err != nil { 51 | return n, err 52 | } 53 | } 54 | n += m 55 | } 56 | 57 | return n, nil 58 | } 59 | 60 | // Compress buf into Data. 61 | func (cw *compressWriter) Flush() error { 62 | if cw.pos == 0 { 63 | return nil 64 | } 65 | maxSize := lz4.CompressBlockBound(len(cw.data[:cw.pos])) 66 | cw.zdata = append(cw.zdata[:0], make([]byte, maxSize+headerSize)...) 67 | _ = cw.zdata[:headerSize] 68 | cw.zdata[hMethod] = byte(cw.method) 69 | 70 | var n int 71 | //nolint:exhaustive 72 | switch cw.method { 73 | case CompressLZ4: 74 | if cw.lz4 == nil { 75 | cw.lz4 = &lz4.Compressor{} 76 | } 77 | compressedSize, err := cw.lz4.CompressBlock(cw.data[:cw.pos], cw.zdata[headerSize:]) 78 | if err != nil { 79 | return fmt.Errorf("lz4 compress error: %v", err) 80 | } 81 | n = compressedSize 82 | case CompressZSTD: 83 | if cw.zstd == nil { 84 | zw, err := zstd.NewWriter(nil, 85 | zstd.WithEncoderLevel(zstd.SpeedDefault), 86 | zstd.WithEncoderConcurrency(1), 87 | zstd.WithLowerEncoderMem(true), 88 | ) 89 | if err != nil { 90 | return fmt.Errorf("zstd new error: %v", err) 91 | } 92 | cw.zstd = zw 93 | } 94 | cw.zdata = cw.zstd.EncodeAll(cw.data[:cw.pos], cw.zdata[:headerSize]) 95 | n = len(cw.zdata) - headerSize 96 | case CompressChecksum: 97 | n = copy(cw.zdata[headerSize:], cw.data[:cw.pos]) 98 | } 99 | 100 | cw.zdata = cw.zdata[:n+headerSize] 101 | 102 | binary.LittleEndian.PutUint32(cw.zdata[hRawSize:], uint32(n+compressHeaderSize)) 103 | binary.LittleEndian.PutUint32(cw.zdata[hDataSize:], uint32(cw.pos)) 104 | h := city.CH128(cw.zdata[hMethod:]) 105 | binary.LittleEndian.PutUint64(cw.zdata[0:8], h.Low) 106 | binary.LittleEndian.PutUint64(cw.zdata[8:16], h.High) 107 | 108 | _, err := cw.writer.Write(cw.zdata) 109 | cw.pos = 0 110 | return err 111 | } 112 | -------------------------------------------------------------------------------- /column/tuple3_gen.go: -------------------------------------------------------------------------------- 1 | package column 2 | 3 | import ( 4 | "unsafe" 5 | ) 6 | 7 | type tuple3Value[T1, T2, T3 any] struct { 8 | Col1 T1 9 | Col2 T2 10 | Col3 T3 11 | } 12 | 13 | // Tuple3 is a column of Tuple(T1, T2, T3) ClickHouse data type 14 | type Tuple3[T ~struct { 15 | Col1 T1 16 | Col2 T2 17 | Col3 T3 18 | }, T1, T2, T3 any] struct { 19 | Tuple 20 | col1 Column[T1] 21 | col2 Column[T2] 22 | col3 Column[T3] 23 | } 24 | 25 | // NewTuple3 create a new tuple of Tuple(T1, T2, T3) ClickHouse data type 26 | func NewTuple3[T ~struct { 27 | Col1 T1 28 | Col2 T2 29 | Col3 T3 30 | }, T1, T2, T3 any]( 31 | column1 Column[T1], 32 | column2 Column[T2], 33 | column3 Column[T3], 34 | ) *Tuple3[T, T1, T2, T3] { 35 | return &Tuple3[T, T1, T2, T3]{ 36 | Tuple: Tuple{ 37 | columns: []ColumnBasic{ 38 | column1, 39 | column2, 40 | column3, 41 | }, 42 | }, 43 | col1: column1, 44 | col2: column2, 45 | col3: column3, 46 | } 47 | } 48 | 49 | // NewNested3 create a new nested of Nested(T1, T2, T3) ClickHouse data type 50 | // 51 | // this is actually an alias for NewTuple3(T1, T2, T3).Array() 52 | func NewNested3[T ~struct { 53 | Col1 T1 54 | Col2 T2 55 | Col3 T3 56 | }, T1, T2, T3 any]( 57 | column1 Column[T1], 58 | column2 Column[T2], 59 | column3 Column[T3], 60 | ) *Array[T] { 61 | return NewTuple3[T]( 62 | column1, 63 | column2, 64 | column3, 65 | ).Array() 66 | } 67 | 68 | // Data get all the data in current block as a slice. 69 | func (c *Tuple3[T, T1, T2, T3]) Data() []T { 70 | val := make([]T, c.NumRow()) 71 | for i := 0; i < c.NumRow(); i++ { 72 | val[i] = T(tuple3Value[T1, T2, T3]{ 73 | Col1: c.col1.Row(i), 74 | Col2: c.col2.Row(i), 75 | Col3: c.col3.Row(i), 76 | }) 77 | } 78 | return val 79 | } 80 | 81 | // Read reads all the data in current block and append to the input. 82 | func (c *Tuple3[T, T1, T2, T3]) Read(value []T) []T { 83 | valTuple := *(*[]tuple3Value[T1, T2, T3])(unsafe.Pointer(&value)) 84 | if cap(valTuple)-len(valTuple) >= c.NumRow() { 85 | valTuple = valTuple[:len(value)+c.NumRow()] 86 | } else { 87 | valTuple = append(valTuple, make([]tuple3Value[T1, T2, T3], c.NumRow())...) 88 | } 89 | 90 | val := valTuple[len(valTuple)-c.NumRow():] 91 | for i := 0; i < c.NumRow(); i++ { 92 | val[i].Col1 = c.col1.Row(i) 93 | val[i].Col2 = c.col2.Row(i) 94 | val[i].Col3 = c.col3.Row(i) 95 | } 96 | return *(*[]T)(unsafe.Pointer(&valTuple)) 97 | } 98 | 99 | // Row return the value of given row. 100 | // NOTE: Row number start from zero 101 | func (c *Tuple3[T, T1, T2, T3]) Row(row int) T { 102 | return T(tuple3Value[T1, T2, T3]{ 103 | Col1: c.col1.Row(row), 104 | Col2: c.col2.Row(row), 105 | Col3: c.col3.Row(row), 106 | }) 107 | } 108 | 109 | // Append value for insert 110 | func (c *Tuple3[T, T1, T2, T3]) Append(v ...T) { 111 | for _, v := range v { 112 | t := tuple3Value[T1, T2, T3](v) 113 | c.col1.Append(t.Col1) 114 | c.col2.Append(t.Col2) 115 | c.col3.Append(t.Col3) 116 | } 117 | } 118 | 119 | // Array return a Array type for this column 120 | func (c *Tuple3[T, T1, T2, T3]) Array() *Array[T] { 121 | return NewArray[T](c) 122 | } 123 | -------------------------------------------------------------------------------- /types/uint256.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "math/big" 5 | ) 6 | 7 | // Note, Zero and Max are functions just to make read-only values. 8 | // We cannot define constants for structures, and global variables 9 | // are unacceptable because it will be possible to change them. 10 | 11 | // Zero is the lowest possible Uint256 value. 12 | func Uint256Zero() Uint256 { 13 | return Uint256From64(0) 14 | } 15 | 16 | // Max is the largest possible Uint256 value. 17 | func Uint256Max() Uint256 { 18 | return Uint256{ 19 | Lo: Uint128Max(), 20 | Hi: Uint128Max(), 21 | } 22 | } 23 | 24 | // Uint256 is an unsigned 256-bit number. 25 | // All methods are immutable, works just like standard uint64. 26 | type Uint256 struct { 27 | Lo Uint128 // lower 128-bit half 28 | Hi Uint128 // upper 128-bit half 29 | } 30 | 31 | // From128 converts 128-bit value v to a Uint256 value. 32 | // Upper 128-bit half will be zero. 33 | func Uint256From128(v Uint128) Uint256 { 34 | return Uint256{Lo: v} 35 | } 36 | 37 | // From64 converts 64-bit value v to a Uint256 value. 38 | // Upper 128-bit half will be zero. 39 | func Uint256From64(v uint64) Uint256 { 40 | return Uint256From128(Uint128From64(v)) 41 | } 42 | 43 | // FromBig converts *big.Int to 256-bit Uint256 value ignoring overflows. 44 | // If input integer is nil or negative then return Zero. 45 | // If input interger overflows 256-bit then return Max. 46 | func Uint256FromBig(i *big.Int) Uint256 { 47 | u, _ := Uint256FromBigEx(i) 48 | return u 49 | } 50 | 51 | // FromBigEx converts *big.Int to 256-bit Uint256 value (eXtended version). 52 | // Provides ok successful flag as a second return value. 53 | // If input integer is negative or overflows 256-bit then ok=false. 54 | // If input is nil then zero 256-bit returned. 55 | func Uint256FromBigEx(i *big.Int) (Uint256, bool) { 56 | switch { 57 | case i == nil: 58 | return Uint256Zero(), true // assuming nil === 0 59 | case i.Sign() < 0: 60 | return Uint256Zero(), false // value cannot be negative! 61 | case i.BitLen() > 256: 62 | return Uint256Max(), false // value overflows 256-bit! 63 | } 64 | 65 | // Note, actually result of big.Int.Uint64 is undefined 66 | // if stored value is greater than 2^64 67 | // but we assume that it just gets lower 64 bits. 68 | t := new(big.Int) 69 | lolo := i.Uint64() 70 | lohi := t.Rsh(i, 64).Uint64() 71 | hilo := t.Rsh(i, 128).Uint64() 72 | hihi := t.Rsh(i, 192).Uint64() 73 | return Uint256{ 74 | Lo: Uint128{Lo: lolo, Hi: lohi}, 75 | Hi: Uint128{Lo: hilo, Hi: hihi}, 76 | }, true 77 | } 78 | 79 | // Big returns 256-bit value as a *big.Int. 80 | // 81 | //nolint:dupl 82 | func (u Uint256) Big() *big.Int { 83 | t := new(big.Int) 84 | i := new(big.Int).SetUint64(u.Hi.Hi) 85 | i = i.Lsh(i, 64) 86 | i = i.Or(i, t.SetUint64(u.Hi.Lo)) 87 | i = i.Lsh(i, 64) 88 | i = i.Or(i, t.SetUint64(u.Lo.Hi)) 89 | i = i.Lsh(i, 64) 90 | i = i.Or(i, t.SetUint64(u.Lo.Lo)) 91 | return i 92 | } 93 | 94 | // Equals returns true if two 256-bit values are equal. 95 | // Uint256 values can be compared directly with == operator 96 | // but use of the Equals method is preferred for consistency. 97 | func (u Uint256) Equals(v Uint256) bool { 98 | return u.Lo.Equals(v.Lo) && u.Hi.Equals(v.Hi) 99 | } 100 | -------------------------------------------------------------------------------- /types/int128.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "math" 5 | "math/big" 6 | ) 7 | 8 | // Note, Zero and Max are functions just to make read-only values. 9 | // We cannot define constants for structures, and global variables 10 | // are unacceptable because it will be possible to change them. 11 | 12 | // Zero is the lowest possible Int128 value. 13 | func Int128Zero() Int128 { 14 | return Int128From64(0) 15 | } 16 | 17 | // Max is the largest possible Int128 value. 18 | func Int128Max() Int128 { 19 | return Int128{ 20 | Lo: math.MaxUint64, 21 | Hi: math.MaxInt64, 22 | } 23 | } 24 | 25 | // Int128 is an unsigned 128-bit number. 26 | // All methods are immutable, works just like standard uint64. 27 | type Int128 struct { 28 | Lo uint64 // lower 64-bit half 29 | Hi int64 // upper 64-bit half 30 | } 31 | 32 | // Note, there in no New(lo, hi) just not to confuse 33 | // which half goes first: lower or upper. 34 | // Use structure initialization Int128{Lo: ..., Hi: ...} instead. 35 | 36 | // From64 converts 64-bit value v to a Int128 value. 37 | // Upper 64-bit half will be zero. 38 | func Int128From64(v int64) Int128 { 39 | var hi int64 40 | if v < 0 { 41 | hi = -1 42 | } 43 | return Int128{Lo: uint64(v), Hi: hi} 44 | } 45 | 46 | // FromBig converts *big.Int to 128-bit Int128 value ignoring overflows. 47 | // If input integer is nil or negative then return Zero. 48 | // If input interger overflows 128-bit then return Max. 49 | func Int128FromBig(i *big.Int) Int128 { 50 | u, _ := Int128FromBigEx(i) 51 | return u 52 | } 53 | 54 | // FromBigEx converts *big.Int to 128-bit Int128 value (eXtended version). 55 | // Provides ok successful flag as a second return value. 56 | // If input integer is negative or overflows 128-bit then ok=false. 57 | // If input is nil then zero 128-bit returned. 58 | func Int128FromBigEx(i *big.Int) (Int128, bool) { 59 | switch { 60 | case i == nil: 61 | return Int128Zero(), true // assuming nil === 0 62 | case i.BitLen() > 128: 63 | return Int128Max(), false // value overflows 128-bit! 64 | } 65 | 66 | neg := false 67 | if i.Sign() == -1 { 68 | i = new(big.Int).Neg(i) 69 | neg = true 70 | } 71 | 72 | // Note, actually result of big.Int.Uint64 is undefined 73 | // if stored value is greater than 2^64 74 | // but we assume that it just gets lower 64 bits. 75 | t := new(big.Int) 76 | lo := i.Uint64() 77 | hi := int64(t.Rsh(i, 64).Uint64()) 78 | val := Int128{ 79 | Lo: lo, 80 | Hi: hi, 81 | } 82 | if neg { 83 | return val.Neg(), true 84 | } 85 | return val, true 86 | } 87 | 88 | // Big returns 128-bit value as a *big.Int. 89 | func (u Int128) Big() *big.Int { 90 | i := new(big.Int).SetInt64(u.Hi) 91 | 92 | i = i.Lsh(i, 64) 93 | i = i.Or(i, new(big.Int).SetUint64(u.Lo)) 94 | return i 95 | } 96 | 97 | // Equals returns true if two 128-bit values are equal. 98 | // Int128 values can be compared directly with == operator 99 | // but use of the Equals method is preferred for consistency. 100 | func (u Int128) Equals(v Int128) bool { 101 | return (u.Lo == v.Lo) && (u.Hi == v.Hi) 102 | } 103 | 104 | // Neg returns the additive inverse of an Int128 105 | func (u Int128) Neg() (z Int128) { 106 | z.Hi = -u.Hi 107 | z.Lo = -u.Lo 108 | if z.Lo > 0 { 109 | z.Hi-- 110 | } 111 | return z 112 | } 113 | -------------------------------------------------------------------------------- /column/array_nullable.go: -------------------------------------------------------------------------------- 1 | package column 2 | 3 | import "github.com/vahid-sohrabloo/chconn/v2/internal/readerwriter" 4 | 5 | // Array is a column of Array(Nullable(T)) ClickHouse data type 6 | type ArrayNullable[T comparable] struct { 7 | Array[T] 8 | dataColumn NullableColumn[T] 9 | columnData []*T 10 | } 11 | 12 | // NewArrayNullable create a new array column of Array(Nullable(T)) ClickHouse data type 13 | func NewArrayNullable[T comparable](dataColumn NullableColumn[T]) *ArrayNullable[T] { 14 | a := &ArrayNullable[T]{ 15 | dataColumn: dataColumn, 16 | Array: Array[T]{ 17 | ArrayBase: ArrayBase{ 18 | dataColumn: dataColumn, 19 | offsetColumn: New[uint64](), 20 | }, 21 | }, 22 | } 23 | a.resetHook = func() { 24 | a.columnData = a.columnData[:0] 25 | } 26 | return a 27 | } 28 | 29 | // Data get all the nullable data in current block as a slice of pointer. 30 | func (c *ArrayNullable[T]) DataP() [][]*T { 31 | values := make([][]*T, c.offsetColumn.numRow) 32 | var lastOffset uint64 33 | columnData := c.getColumnData() 34 | for i := 0; i < c.offsetColumn.numRow; i++ { 35 | values[i] = columnData[lastOffset:c.offsetColumn.Row(i)] 36 | lastOffset = c.offsetColumn.Row(i) 37 | } 38 | return values 39 | } 40 | 41 | // Read reads all the nullable data in current block as a slice pointer and append to the input. 42 | func (c *ArrayNullable[T]) ReadP(value [][]*T) [][]*T { 43 | var lastOffset uint64 44 | columnData := c.getColumnData() 45 | for i := 0; i < c.offsetColumn.numRow; i++ { 46 | value = append(value, columnData[lastOffset:c.offsetColumn.Row(i)]) 47 | lastOffset = c.offsetColumn.Row(i) 48 | } 49 | return value 50 | } 51 | 52 | // RowP return the nullable value of given row as a pointer 53 | // NOTE: Row number start from zero 54 | func (c *ArrayNullable[T]) RowP(row int) []*T { 55 | var lastOffset uint64 56 | if row != 0 { 57 | lastOffset = c.offsetColumn.Row(row - 1) 58 | } 59 | var val []*T 60 | val = append(val, c.getColumnData()[lastOffset:c.offsetColumn.Row(row)]...) 61 | return val 62 | } 63 | 64 | // AppendP a nullable value for insert 65 | func (c *ArrayNullable[T]) AppendP(v ...[]*T) { 66 | for _, v := range v { 67 | c.AppendLen(len(v)) 68 | c.dataColumn.AppendP(v...) 69 | } 70 | } 71 | 72 | // AppendItemP Append nullable item value for insert 73 | // 74 | // it should use with AppendLen 75 | // 76 | // Example: 77 | // 78 | // c.AppendLen(2) // insert 2 items 79 | // c.AppendItemP(val1, val2) // insert item 1 80 | func (c *ArrayNullable[T]) AppendItemP(v ...*T) { 81 | c.dataColumn.AppendP(v...) 82 | } 83 | 84 | // ArrayOf return a Array type for this column 85 | func (c *ArrayNullable[T]) ArrayOf() *Array2Nullable[T] { 86 | return NewArray2Nullable(c) 87 | } 88 | 89 | // ReadRaw read raw data from the reader. it runs automatically 90 | func (c *ArrayNullable[T]) ReadRaw(num int, r *readerwriter.Reader) error { 91 | err := c.Array.ReadRaw(num, r) 92 | if err != nil { 93 | return err 94 | } 95 | c.columnData = c.dataColumn.DataP() 96 | return nil 97 | } 98 | 99 | func (c *ArrayNullable[T]) getColumnData() []*T { 100 | if len(c.columnData) == 0 { 101 | c.columnData = c.dataColumn.DataP() 102 | } 103 | return c.columnData 104 | } 105 | 106 | func (c *ArrayNullable[T]) elem(arrayLevel int) ColumnBasic { 107 | if arrayLevel > 0 { 108 | return c.ArrayOf().elem(arrayLevel - 1) 109 | } 110 | return c 111 | } 112 | -------------------------------------------------------------------------------- /column/tuple4_gen.go: -------------------------------------------------------------------------------- 1 | package column 2 | 3 | import ( 4 | "unsafe" 5 | ) 6 | 7 | type tuple4Value[T1, T2, T3, T4 any] struct { 8 | Col1 T1 9 | Col2 T2 10 | Col3 T3 11 | Col4 T4 12 | } 13 | 14 | // Tuple4 is a column of Tuple(T1, T2, T3, T4) ClickHouse data type 15 | type Tuple4[T ~struct { 16 | Col1 T1 17 | Col2 T2 18 | Col3 T3 19 | Col4 T4 20 | }, T1, T2, T3, T4 any] struct { 21 | Tuple 22 | col1 Column[T1] 23 | col2 Column[T2] 24 | col3 Column[T3] 25 | col4 Column[T4] 26 | } 27 | 28 | // NewTuple4 create a new tuple of Tuple(T1, T2, T3, T4) ClickHouse data type 29 | func NewTuple4[T ~struct { 30 | Col1 T1 31 | Col2 T2 32 | Col3 T3 33 | Col4 T4 34 | }, T1, T2, T3, T4 any]( 35 | column1 Column[T1], 36 | column2 Column[T2], 37 | column3 Column[T3], 38 | column4 Column[T4], 39 | ) *Tuple4[T, T1, T2, T3, T4] { 40 | return &Tuple4[T, T1, T2, T3, T4]{ 41 | Tuple: Tuple{ 42 | columns: []ColumnBasic{ 43 | column1, 44 | column2, 45 | column3, 46 | column4, 47 | }, 48 | }, 49 | col1: column1, 50 | col2: column2, 51 | col3: column3, 52 | col4: column4, 53 | } 54 | } 55 | 56 | // NewNested4 create a new nested of Nested(T1, T2, T3, T4) ClickHouse data type 57 | // 58 | // this is actually an alias for NewTuple4(T1, T2, T3, T4).Array() 59 | func NewNested4[T ~struct { 60 | Col1 T1 61 | Col2 T2 62 | Col3 T3 63 | Col4 T4 64 | }, T1, T2, T3, T4 any]( 65 | column1 Column[T1], 66 | column2 Column[T2], 67 | column3 Column[T3], 68 | column4 Column[T4], 69 | ) *Array[T] { 70 | return NewTuple4[T]( 71 | column1, 72 | column2, 73 | column3, 74 | column4, 75 | ).Array() 76 | } 77 | 78 | // Data get all the data in current block as a slice. 79 | func (c *Tuple4[T, T1, T2, T3, T4]) Data() []T { 80 | val := make([]T, c.NumRow()) 81 | for i := 0; i < c.NumRow(); i++ { 82 | val[i] = T(tuple4Value[T1, T2, T3, T4]{ 83 | Col1: c.col1.Row(i), 84 | Col2: c.col2.Row(i), 85 | Col3: c.col3.Row(i), 86 | Col4: c.col4.Row(i), 87 | }) 88 | } 89 | return val 90 | } 91 | 92 | // Read reads all the data in current block and append to the input. 93 | func (c *Tuple4[T, T1, T2, T3, T4]) Read(value []T) []T { 94 | valTuple := *(*[]tuple4Value[T1, T2, T3, T4])(unsafe.Pointer(&value)) 95 | if cap(valTuple)-len(valTuple) >= c.NumRow() { 96 | valTuple = valTuple[:len(value)+c.NumRow()] 97 | } else { 98 | valTuple = append(valTuple, make([]tuple4Value[T1, T2, T3, T4], c.NumRow())...) 99 | } 100 | 101 | val := valTuple[len(valTuple)-c.NumRow():] 102 | for i := 0; i < c.NumRow(); i++ { 103 | val[i].Col1 = c.col1.Row(i) 104 | val[i].Col2 = c.col2.Row(i) 105 | val[i].Col3 = c.col3.Row(i) 106 | val[i].Col4 = c.col4.Row(i) 107 | } 108 | return *(*[]T)(unsafe.Pointer(&valTuple)) 109 | } 110 | 111 | // Row return the value of given row. 112 | // NOTE: Row number start from zero 113 | func (c *Tuple4[T, T1, T2, T3, T4]) Row(row int) T { 114 | return T(tuple4Value[T1, T2, T3, T4]{ 115 | Col1: c.col1.Row(row), 116 | Col2: c.col2.Row(row), 117 | Col3: c.col3.Row(row), 118 | Col4: c.col4.Row(row), 119 | }) 120 | } 121 | 122 | // Append value for insert 123 | func (c *Tuple4[T, T1, T2, T3, T4]) Append(v ...T) { 124 | for _, v := range v { 125 | t := tuple4Value[T1, T2, T3, T4](v) 126 | c.col1.Append(t.Col1) 127 | c.col2.Append(t.Col2) 128 | c.col3.Append(t.Col3) 129 | c.col4.Append(t.Col4) 130 | } 131 | } 132 | 133 | // Array return a Array type for this column 134 | func (c *Tuple4[T, T1, T2, T3, T4]) Array() *Array[T] { 135 | return NewArray[T](c) 136 | } 137 | -------------------------------------------------------------------------------- /types/Int256.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "math/big" 5 | ) 6 | 7 | // Note, Zero and Max are functions just to make read-only values. 8 | // We cannot define constants for structures, and global variables 9 | // are unacceptable because it will be possible to change them. 10 | 11 | // Zero is the lowest possible Int256 value. 12 | func Int256Zero() Int256 { 13 | return Int256From64(0) 14 | } 15 | 16 | // Max is the largest possible Int256 value. 17 | func Int256Max() Int256 { 18 | return Int256{ 19 | Lo: Uint128Max(), 20 | Hi: Int128Max(), 21 | } 22 | } 23 | 24 | // Int256 is an unsigned 256-bit number. 25 | // All methods are immutable, works just like standard uint64. 26 | type Int256 struct { 27 | Lo Uint128 // lower 128-bit half 28 | Hi Int128 // upper 128-bit half 29 | } 30 | 31 | // From128 converts 128-bit value v to a Int256 value. 32 | // Upper 128-bit half will be zero. 33 | func Int256From128(v Int128) Int256 { 34 | var hi Int128 35 | if v.Hi < 0 { 36 | hi = Int128{Lo: 0, Hi: -1} 37 | v = v.Neg() 38 | } 39 | return Int256{Lo: Uint128{ 40 | Lo: v.Lo, 41 | Hi: uint64(v.Hi), 42 | }, Hi: hi} 43 | } 44 | 45 | // From64 converts 64-bit value v to a Int256 value. 46 | // Upper 128-bit half will be zero. 47 | func Int256From64(v int64) Int256 { 48 | return Int256From128(Int128From64(v)) 49 | } 50 | 51 | // FromBig converts *big.Int to 256-bit Int256 value ignoring overflows. 52 | // If input integer is nil or negative then return Zero. 53 | // If input integer overflows 256-bit then return Max. 54 | func Int256FromBig(i *big.Int) Int256 { 55 | u, _ := Int256FromBigEx(i) 56 | return u 57 | } 58 | 59 | // FromBigEx converts *big.Int to 256-bit Int256 value (eXtended version). 60 | // Provides ok successful flag as a second return value. 61 | // If input integer is negative or overflows 256-bit then ok=false. 62 | // If input is nil then zero 256-bit returned. 63 | func Int256FromBigEx(i *big.Int) (Int256, bool) { 64 | switch { 65 | case i == nil: 66 | return Int256Zero(), true // assuming nil === 0 67 | 68 | case i.BitLen() > 256: 69 | return Int256Max(), false // value overflows 256-bit! 70 | } 71 | 72 | neg := false 73 | if i.Sign() == -1 { 74 | i = new(big.Int).Neg(i) 75 | neg = true 76 | } 77 | 78 | t := new(big.Int) 79 | lolo := i.Uint64() 80 | lohi := t.Rsh(i, 64).Uint64() 81 | hilo := t.Rsh(i, 128).Uint64() 82 | hihi := int64(t.Rsh(i, 192).Uint64()) 83 | val := Int256{ 84 | Lo: Uint128{Lo: lolo, Hi: lohi}, 85 | Hi: Int128{Lo: hilo, Hi: hihi}, 86 | } 87 | if neg { 88 | val = val.Neg() 89 | } 90 | return val, true 91 | } 92 | 93 | // Big returns 256-bit value as a *big.Int. 94 | // 95 | //nolint:dupl 96 | func (u Int256) Big() *big.Int { 97 | t := new(big.Int) 98 | i := new(big.Int).SetInt64(u.Hi.Hi) 99 | i = i.Lsh(i, 64) 100 | i = i.Or(i, t.SetUint64(u.Hi.Lo)) 101 | i = i.Lsh(i, 64) 102 | i = i.Or(i, t.SetUint64(u.Lo.Hi)) 103 | i = i.Lsh(i, 64) 104 | i = i.Or(i, t.SetUint64(u.Lo.Lo)) 105 | return i 106 | } 107 | 108 | // Equals returns true if two 256-bit values are equal. 109 | // Int256 values can be compared directly with == operator 110 | // but use of the Equals method is preferred for consistency. 111 | func (u Int256) Equals(v Int256) bool { 112 | return u.Lo.Equals(v.Lo) && u.Hi.Equals(v.Hi) 113 | } 114 | 115 | // Neg returns the additive inverse of an Int256 116 | func (u Int256) Neg() (z Int256) { 117 | z.Hi = u.Hi.Neg() 118 | z.Lo.Lo = -u.Lo.Lo 119 | z.Lo.Hi = -u.Lo.Hi 120 | // TODO, I'm not sure here. 121 | if z.Lo.Hi > 0 || z.Lo.Lo > 0 { 122 | z.Hi.Lo-- 123 | } 124 | return z 125 | } 126 | -------------------------------------------------------------------------------- /column/lc_nullable.go: -------------------------------------------------------------------------------- 1 | package column 2 | 3 | // LowCardinalityNullable for LowCardinality(Nullable(T)) ClickHouse DataTypes 4 | type LowCardinalityNullable[T comparable] struct { 5 | LowCardinality[T] 6 | } 7 | 8 | // NewLowCardinalityNullable return new LowCardinalityNullable for nullable LowCardinality ClickHouse DataTypes 9 | func NewLowCardinalityNullable[T comparable](dictColumn Column[T]) *LowCardinalityNullable[T] { 10 | return NewLCNullable(dictColumn) 11 | } 12 | 13 | // NewLCNullable return new LowCardinalityNullable for nullable LowCardinality ClickHouse DataTypes 14 | func NewLCNullable[T comparable](dictColumn Column[T]) *LowCardinalityNullable[T] { 15 | var empty T 16 | dictColumn.Append(empty) 17 | l := &LowCardinalityNullable[T]{ 18 | LowCardinality: LowCardinality[T]{ 19 | nullable: true, 20 | dict: make(map[T]int), 21 | dictColumn: dictColumn, 22 | }, 23 | } 24 | return l 25 | } 26 | 27 | // Data get all nullable data in current block as a slice. 28 | // 29 | // NOTE: the return slice only valid in current block, if you want to use it after, you should copy it. or use Read 30 | func (c *LowCardinalityNullable[T]) DataP() []*T { 31 | result := make([]*T, c.NumRow()) 32 | for i, k := range c.readedKeys { 33 | if k == 0 { 34 | result[i] = nil 35 | } else { 36 | val := c.readedDict[k] 37 | result[i] = &val 38 | } 39 | } 40 | return result 41 | } 42 | 43 | // Read reads all nullable data in current block and append to the input. 44 | func (c *LowCardinalityNullable[T]) ReadP(value []*T) []*T { 45 | for _, k := range c.readedKeys { 46 | if k == 0 { 47 | value = append(value, nil) 48 | } else { 49 | val := c.readedDict[k] 50 | value = append(value, &val) 51 | } 52 | } 53 | return value 54 | } 55 | 56 | // Row return nullable value of given row 57 | // NOTE: Row number start from zero 58 | func (c *LowCardinalityNullable[T]) RowP(row int) *T { 59 | if c.readedKeys[row] == 0 { 60 | return nil 61 | } 62 | val := c.readedDict[c.readedKeys[row]] 63 | return &val 64 | } 65 | 66 | // Append value for insert 67 | func (c *LowCardinalityNullable[T]) Append(v ...T) { 68 | for _, v := range v { 69 | key, ok := c.dict[v] 70 | if !ok { 71 | key = len(c.dict) 72 | c.dict[v] = key 73 | c.dictColumn.Append(v) 74 | } 75 | c.keys = append(c.keys, key+1) 76 | } 77 | 78 | c.numRow += len(v) 79 | } 80 | 81 | // Append nil value for insert 82 | func (c *LowCardinalityNullable[T]) AppendNil() { 83 | c.keys = append(c.keys, 0) 84 | c.numRow++ 85 | } 86 | 87 | // Append nullable value for insert 88 | // 89 | // as an alternative (for better performance), you can use `Append` and `AppendNil` to insert a value 90 | func (c *LowCardinalityNullable[T]) AppendP(v ...*T) { 91 | for _, v := range v { 92 | if v == nil { 93 | c.keys = append(c.keys, 0) 94 | continue 95 | } 96 | key, ok := c.dict[*v] 97 | if !ok { 98 | key = len(c.dict) 99 | c.dict[*v] = key 100 | c.dictColumn.Append(*v) 101 | } 102 | c.keys = append(c.keys, key+1) 103 | } 104 | 105 | c.numRow += len(v) 106 | } 107 | 108 | // Array return a Array type for this column 109 | func (c *LowCardinalityNullable[T]) Array() *ArrayNullable[T] { 110 | return NewArrayNullable[T](c) 111 | } 112 | 113 | // Reset all statuses and buffered data 114 | // 115 | // After each reading, the reading data does not need to be reset. It will be automatically reset. 116 | // 117 | // When inserting, buffers are reset only after the operation is successful. 118 | // If an error occurs, you can safely call insert again. 119 | func (c *LowCardinalityNullable[T]) Reset() { 120 | c.LowCardinality.Reset() 121 | var empty T 122 | c.dictColumn.Append(empty) 123 | } 124 | 125 | func (c *LowCardinalityNullable[T]) elem(arrayLevel int) ColumnBasic { 126 | if arrayLevel > 0 { 127 | return c.Array().elem(arrayLevel - 1) 128 | } 129 | return c 130 | } 131 | -------------------------------------------------------------------------------- /column/lc_test.go: -------------------------------------------------------------------------------- 1 | package column_test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/assert" 10 | "github.com/stretchr/testify/require" 11 | "github.com/vahid-sohrabloo/chconn/v2" 12 | "github.com/vahid-sohrabloo/chconn/v2/column" 13 | ) 14 | 15 | func TestLcIndicator16(t *testing.T) { 16 | tableName := "lc_indicator_16" 17 | 18 | t.Parallel() 19 | 20 | connString := os.Getenv("CHX_TEST_TCP_CONN_STRING") 21 | 22 | conn, err := chconn.Connect(context.Background(), connString) 23 | require.NoError(t, err) 24 | 25 | err = conn.Exec(context.Background(), 26 | fmt.Sprintf(`DROP TABLE IF EXISTS test_%s`, tableName), 27 | ) 28 | require.NoError(t, err) 29 | set := chconn.Settings{ 30 | { 31 | Name: "allow_suspicious_low_cardinality_types", 32 | Value: "true", 33 | }, 34 | } 35 | 36 | err = conn.ExecWithOption(context.Background(), fmt.Sprintf(`CREATE TABLE test_%[1]s ( 37 | %[1]s_lc LowCardinality(Int64) 38 | ) Engine=Memory`, tableName), &chconn.QueryOptions{ 39 | Settings: set, 40 | }) 41 | 42 | require.NoError(t, err) 43 | 44 | col := column.New[int64]().LC() 45 | 46 | var colInsert []int64 47 | 48 | rows := int(^uint8(0)) + 10 49 | for i := 0; i < rows; i++ { 50 | val := int64(i + 1) 51 | col.Append(val) 52 | colInsert = append(colInsert, val) 53 | } 54 | 55 | err = conn.Insert(context.Background(), fmt.Sprintf(`INSERT INTO 56 | test_%[1]s ( 57 | %[1]s_lc 58 | ) 59 | VALUES`, tableName), 60 | col, 61 | ) 62 | require.NoError(t, err) 63 | 64 | // test read row 65 | colRead := column.New[int64]().LC() 66 | 67 | selectStmt, err := conn.Select(context.Background(), fmt.Sprintf(`SELECT 68 | %[1]s_lc 69 | FROM test_%[1]s`, tableName), 70 | colRead, 71 | ) 72 | 73 | require.NoError(t, err) 74 | require.True(t, conn.IsBusy()) 75 | 76 | var colData []int64 77 | 78 | for selectStmt.Next() { 79 | colData = colRead.Read(colData) 80 | } 81 | 82 | require.NoError(t, selectStmt.Err()) 83 | assert.Equal(t, colInsert, colData) 84 | } 85 | 86 | func TestLcIndicator32(t *testing.T) { 87 | tableName := "lc_indicator_32" 88 | 89 | t.Parallel() 90 | 91 | connString := os.Getenv("CHX_TEST_TCP_CONN_STRING") 92 | 93 | conn, err := chconn.Connect(context.Background(), connString) 94 | require.NoError(t, err) 95 | 96 | err = conn.Exec(context.Background(), 97 | fmt.Sprintf(`DROP TABLE IF EXISTS test_%s`, tableName), 98 | ) 99 | require.NoError(t, err) 100 | set := chconn.Settings{ 101 | { 102 | Name: "allow_suspicious_low_cardinality_types", 103 | Value: "true", 104 | }, 105 | } 106 | 107 | err = conn.ExecWithOption(context.Background(), fmt.Sprintf(`CREATE TABLE test_%[1]s ( 108 | %[1]s_lc LowCardinality(Int64) 109 | ) Engine=Memory`, tableName), &chconn.QueryOptions{ 110 | Settings: set, 111 | }) 112 | 113 | require.NoError(t, err) 114 | 115 | col := column.New[int64]().LC() 116 | 117 | var colInsert []int64 118 | 119 | rows := int(^uint16(0)) + 10 120 | for i := 0; i < rows; i++ { 121 | val := int64(i + 1) 122 | col.Append(val) 123 | colInsert = append(colInsert, val) 124 | } 125 | 126 | err = conn.Insert(context.Background(), fmt.Sprintf(`INSERT INTO 127 | test_%[1]s ( 128 | %[1]s_lc 129 | ) 130 | VALUES`, tableName), 131 | col, 132 | ) 133 | require.NoError(t, err) 134 | 135 | // test read row 136 | colRead := column.New[int64]().LC() 137 | 138 | selectStmt, err := conn.Select(context.Background(), fmt.Sprintf(`SELECT 139 | %[1]s_lc 140 | FROM test_%[1]s`, tableName), 141 | colRead, 142 | ) 143 | 144 | require.NoError(t, err) 145 | require.True(t, conn.IsBusy()) 146 | 147 | var colData []int64 148 | 149 | for selectStmt.Next() { 150 | colData = colRead.Read(colData) 151 | } 152 | 153 | require.NoError(t, selectStmt.Err()) 154 | assert.Equal(t, colInsert, colData) 155 | } 156 | -------------------------------------------------------------------------------- /errors_test.go: -------------------------------------------------------------------------------- 1 | package chconn 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "io" 7 | "os" 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | func TestChErrorReadError(t *testing.T) { 15 | startValidReader := 14 16 | 17 | tests := []struct { 18 | name string 19 | wantErr string 20 | numberValid int 21 | }{ 22 | { 23 | name: "ChError: read code", 24 | wantErr: "ChError: read code", 25 | numberValid: startValidReader, 26 | }, { 27 | name: "ChError: read name", 28 | wantErr: "ChError: read name", 29 | numberValid: startValidReader + 1, 30 | }, { 31 | name: "ChError: read message", 32 | wantErr: "ChError: read message", 33 | numberValid: startValidReader + 3, 34 | }, { 35 | name: "ChError: read StackTrace", 36 | wantErr: "ChError: read StackTrace", 37 | numberValid: startValidReader + 5, 38 | }, { 39 | name: "ChError: read hasNested", 40 | wantErr: "ChError: read hasNested", 41 | numberValid: startValidReader + 8, 42 | }, 43 | } 44 | for _, tt := range tests { 45 | t.Run(tt.name, func(t *testing.T) { 46 | config, err := ParseConfig(os.Getenv("CHX_TEST_TCP_CONN_STRING")) 47 | require.NoError(t, err) 48 | config.ReaderFunc = func(r io.Reader) io.Reader { 49 | return &readErrorHelper{ 50 | err: errors.New("timeout"), 51 | r: r, 52 | numberValid: tt.numberValid, 53 | } 54 | } 55 | 56 | c, err := ConnectConfig(context.Background(), config) 57 | require.NoError(t, err) 58 | err = c.Exec(context.Background(), "SELECT * FROM invalid_table LIMIT 5;") 59 | require.Error(t, err) 60 | readErr, ok := err.(*readError) 61 | require.True(t, ok) 62 | require.Equal(t, readErr.msg, tt.wantErr) 63 | require.EqualError(t, readErr.Unwrap(), "timeout") 64 | assert.True(t, c.IsClosed()) 65 | }) 66 | } 67 | } 68 | 69 | func NewParseConfigError(conn, msg string, err error) error { 70 | return &parseConfigError{ 71 | connString: conn, 72 | msg: msg, 73 | err: err, 74 | } 75 | } 76 | 77 | func TestConfigError(t *testing.T) { 78 | tests := []struct { 79 | name string 80 | err error 81 | expectedMsg string 82 | }{ 83 | { 84 | name: "url with password", 85 | err: NewParseConfigError("clickhouse://foo:password@host", "msg", nil), 86 | expectedMsg: "cannot parse `clickhouse://foo:xxxxx@host`: msg", 87 | }, 88 | { 89 | name: "dsn with password unquoted", 90 | err: NewParseConfigError("host=host password=password user=user", "msg", nil), 91 | expectedMsg: "cannot parse `host=host password=xxxxx user=user`: msg", 92 | }, 93 | { 94 | name: "dsn with password quoted", 95 | err: NewParseConfigError("host=host password='pass word' user=user", "msg", nil), 96 | expectedMsg: "cannot parse `host=host password=xxxxx user=user`: msg", 97 | }, 98 | { 99 | name: "weird url", 100 | err: NewParseConfigError("clickhouse://foo::pasword@host:1:", "msg", nil), 101 | expectedMsg: "cannot parse `clickhouse://foo:xxxxx@host:1:`: msg", 102 | }, 103 | { 104 | name: "weird url with slash in password", 105 | err: NewParseConfigError("clickhouse://user:pass/word@host:5432/db_name", "msg", nil), 106 | expectedMsg: "cannot parse `clickhouse://user:xxxxxx@host:5432/db_name`: msg", 107 | }, 108 | { 109 | name: "url without password", 110 | err: NewParseConfigError("clickhouse://other@host/db", "msg", nil), 111 | expectedMsg: "cannot parse `clickhouse://other@host/db`: msg", 112 | }, 113 | } 114 | for _, tt := range tests { 115 | tt := tt 116 | t.Run(tt.name, func(t *testing.T) { 117 | t.Parallel() 118 | assert.EqualError(t, tt.err, tt.expectedMsg) 119 | }) 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /column/tuple5_gen.go: -------------------------------------------------------------------------------- 1 | package column 2 | 3 | import ( 4 | "unsafe" 5 | ) 6 | 7 | type tuple5Value[T1, T2, T3, T4, T5 any] struct { 8 | Col1 T1 9 | Col2 T2 10 | Col3 T3 11 | Col4 T4 12 | Col5 T5 13 | } 14 | 15 | // Tuple5 is a column of Tuple(T1, T2, T3, T4, T5) ClickHouse data type 16 | type Tuple5[T ~struct { 17 | Col1 T1 18 | Col2 T2 19 | Col3 T3 20 | Col4 T4 21 | Col5 T5 22 | }, T1, T2, T3, T4, T5 any] struct { 23 | Tuple 24 | col1 Column[T1] 25 | col2 Column[T2] 26 | col3 Column[T3] 27 | col4 Column[T4] 28 | col5 Column[T5] 29 | } 30 | 31 | // NewTuple5 create a new tuple of Tuple(T1, T2, T3, T4, T5) ClickHouse data type 32 | func NewTuple5[T ~struct { 33 | Col1 T1 34 | Col2 T2 35 | Col3 T3 36 | Col4 T4 37 | Col5 T5 38 | }, T1, T2, T3, T4, T5 any]( 39 | column1 Column[T1], 40 | column2 Column[T2], 41 | column3 Column[T3], 42 | column4 Column[T4], 43 | column5 Column[T5], 44 | ) *Tuple5[T, T1, T2, T3, T4, T5] { 45 | return &Tuple5[T, T1, T2, T3, T4, T5]{ 46 | Tuple: Tuple{ 47 | columns: []ColumnBasic{ 48 | column1, 49 | column2, 50 | column3, 51 | column4, 52 | column5, 53 | }, 54 | }, 55 | col1: column1, 56 | col2: column2, 57 | col3: column3, 58 | col4: column4, 59 | col5: column5, 60 | } 61 | } 62 | 63 | // NewNested5 create a new nested of Nested(T1, T2, T3, T4, T5) ClickHouse data type 64 | // 65 | // this is actually an alias for NewTuple5(T1, T2, T3, T4, T5).Array() 66 | func NewNested5[T ~struct { 67 | Col1 T1 68 | Col2 T2 69 | Col3 T3 70 | Col4 T4 71 | Col5 T5 72 | }, T1, T2, T3, T4, T5 any]( 73 | column1 Column[T1], 74 | column2 Column[T2], 75 | column3 Column[T3], 76 | column4 Column[T4], 77 | column5 Column[T5], 78 | ) *Array[T] { 79 | return NewTuple5[T]( 80 | column1, 81 | column2, 82 | column3, 83 | column4, 84 | column5, 85 | ).Array() 86 | } 87 | 88 | // Data get all the data in current block as a slice. 89 | func (c *Tuple5[T, T1, T2, T3, T4, T5]) Data() []T { 90 | val := make([]T, c.NumRow()) 91 | for i := 0; i < c.NumRow(); i++ { 92 | val[i] = T(tuple5Value[T1, T2, T3, T4, T5]{ 93 | Col1: c.col1.Row(i), 94 | Col2: c.col2.Row(i), 95 | Col3: c.col3.Row(i), 96 | Col4: c.col4.Row(i), 97 | Col5: c.col5.Row(i), 98 | }) 99 | } 100 | return val 101 | } 102 | 103 | // Read reads all the data in current block and append to the input. 104 | func (c *Tuple5[T, T1, T2, T3, T4, T5]) Read(value []T) []T { 105 | valTuple := *(*[]tuple5Value[T1, T2, T3, T4, T5])(unsafe.Pointer(&value)) 106 | if cap(valTuple)-len(valTuple) >= c.NumRow() { 107 | valTuple = valTuple[:len(value)+c.NumRow()] 108 | } else { 109 | valTuple = append(valTuple, make([]tuple5Value[T1, T2, T3, T4, T5], c.NumRow())...) 110 | } 111 | 112 | val := valTuple[len(valTuple)-c.NumRow():] 113 | for i := 0; i < c.NumRow(); i++ { 114 | val[i].Col1 = c.col1.Row(i) 115 | val[i].Col2 = c.col2.Row(i) 116 | val[i].Col3 = c.col3.Row(i) 117 | val[i].Col4 = c.col4.Row(i) 118 | val[i].Col5 = c.col5.Row(i) 119 | } 120 | return *(*[]T)(unsafe.Pointer(&valTuple)) 121 | } 122 | 123 | // Row return the value of given row. 124 | // NOTE: Row number start from zero 125 | func (c *Tuple5[T, T1, T2, T3, T4, T5]) Row(row int) T { 126 | return T(tuple5Value[T1, T2, T3, T4, T5]{ 127 | Col1: c.col1.Row(row), 128 | Col2: c.col2.Row(row), 129 | Col3: c.col3.Row(row), 130 | Col4: c.col4.Row(row), 131 | Col5: c.col5.Row(row), 132 | }) 133 | } 134 | 135 | // Append value for insert 136 | func (c *Tuple5[T, T1, T2, T3, T4, T5]) Append(v ...T) { 137 | for _, v := range v { 138 | t := tuple5Value[T1, T2, T3, T4, T5](v) 139 | c.col1.Append(t.Col1) 140 | c.col2.Append(t.Col2) 141 | c.col3.Append(t.Col3) 142 | c.col4.Append(t.Col4) 143 | c.col5.Append(t.Col5) 144 | } 145 | } 146 | 147 | // Array return a Array type for this column 148 | func (c *Tuple5[T, T1, T2, T3, T4, T5]) Array() *Array[T] { 149 | return NewArray[T](c) 150 | } 151 | -------------------------------------------------------------------------------- /internal/readerwriter/compress_reader.go: -------------------------------------------------------------------------------- 1 | package readerwriter 2 | 3 | // copy from https://github.com/ClickHouse/ch-go/blob/4cde4e4bec24211c0bcdc6f385f4212d0ad522d9/compress/reader.go 4 | // some changes to compatible with chconn 5 | import ( 6 | "encoding/binary" 7 | "fmt" 8 | "io" 9 | 10 | "github.com/go-faster/city" 11 | "github.com/klauspost/compress/zstd" 12 | "github.com/pierrec/lz4/v4" 13 | ) 14 | 15 | type invalidCompressErr struct { 16 | method CompressMethod 17 | } 18 | 19 | func (e *invalidCompressErr) Error() string { 20 | return fmt.Sprintf("unknown compression method: 0x%02x ", e.method) 21 | } 22 | 23 | type compressReader struct { 24 | reader io.Reader 25 | data []byte 26 | pos int64 27 | raw []byte 28 | header []byte 29 | zstd *zstd.Decoder 30 | } 31 | 32 | // NewCompressReader wrap the io.Reader 33 | func NewCompressReader(r io.Reader) io.Reader { 34 | return &compressReader{ 35 | zstd: nil, // lazily initialized 36 | reader: r, 37 | header: make([]byte, headerSize), 38 | } 39 | } 40 | 41 | func (r *compressReader) Read(buf []byte) (n int, err error) { 42 | if r.pos >= int64(len(r.data)) { 43 | if err := r.readBlock(); err != nil { 44 | return 0, fmt.Errorf("read block: %w", err) 45 | } 46 | } 47 | n = copy(buf, r.data[r.pos:]) 48 | r.pos += int64(n) 49 | return n, nil 50 | } 51 | 52 | // readBlock reads next compressed data into raw and decompresses into data. 53 | func (r *compressReader) readBlock() error { 54 | r.pos = 0 55 | 56 | _ = r.header[headerSize-1] 57 | if _, err := io.ReadFull(r.reader, r.header); err != nil { 58 | return fmt.Errorf("read header: %w", err) 59 | } 60 | 61 | var ( 62 | rawSize = int(binary.LittleEndian.Uint32(r.header[hRawSize:])) - compressHeaderSize 63 | dataSize = int(binary.LittleEndian.Uint32(r.header[hDataSize:])) 64 | ) 65 | if dataSize < 0 || dataSize > maxDataSize { 66 | return fmt.Errorf("data size should be %d < %d < %d", 0, dataSize, maxDataSize) 67 | } 68 | if rawSize < 0 || rawSize > maxBlockSize { 69 | return fmt.Errorf("raw size should be %d < %d < %d", 0, rawSize, maxBlockSize) 70 | } 71 | 72 | r.data = append(r.data[:0], make([]byte, dataSize)...) 73 | r.raw = append(r.raw[:0], r.header...) 74 | r.raw = append(r.raw, make([]byte, rawSize)...) 75 | _ = r.raw[:rawSize+headerSize-1] 76 | 77 | if _, err := io.ReadFull(r.reader, r.raw[headerSize:]); err != nil { 78 | return fmt.Errorf("read raw: %w", err) 79 | } 80 | hGot := city.U128{ 81 | Low: binary.LittleEndian.Uint64(r.raw[0:8]), 82 | High: binary.LittleEndian.Uint64(r.raw[8:16]), 83 | } 84 | h := city.CH128(r.raw[hMethod:]) 85 | if hGot != h { 86 | return &CorruptedDataErr{ 87 | Actual: h, 88 | Reference: hGot, 89 | RawSize: rawSize, 90 | DataSize: dataSize, 91 | } 92 | } 93 | //nolint:exhaustive 94 | switch m := CompressMethod(r.header[hMethod]); m { 95 | case CompressLZ4: 96 | n, err := lz4.UncompressBlock(r.raw[headerSize:], r.data) 97 | if err != nil { 98 | return fmt.Errorf("lz4 decompress: %w", err) 99 | } 100 | if n != dataSize { 101 | return fmt.Errorf("unexpected uncompressed data size: %d (actual) != %d (got in header)", 102 | n, dataSize, 103 | ) 104 | } 105 | case CompressZSTD: 106 | if r.zstd == nil { 107 | // Lazily initializing to prevent spawning goroutines in NewReader. 108 | // See https://github.com/golang/go/issues/47056#issuecomment-997436820 109 | zstdReader, err := zstd.NewReader(nil, 110 | zstd.WithDecoderConcurrency(1), 111 | zstd.WithDecoderLowmem(true), 112 | ) 113 | if err != nil { 114 | return fmt.Errorf("zstd new: %w", err) 115 | } 116 | r.zstd = zstdReader 117 | } 118 | data, err := r.zstd.DecodeAll(r.raw[headerSize:], r.data[:0]) 119 | if err != nil { 120 | return fmt.Errorf("zstd decompress: %w", err) 121 | } 122 | if len(data) != dataSize { 123 | return fmt.Errorf("unexpected uncompressed data size: %d (actual) != %d (got in header)", 124 | len(data), dataSize, 125 | ) 126 | } 127 | r.data = data 128 | case CompressChecksum: 129 | copy(r.data, r.raw[headerSize:]) 130 | default: 131 | return &invalidCompressErr{m} 132 | } 133 | 134 | return nil 135 | } 136 | -------------------------------------------------------------------------------- /column/base.go: -------------------------------------------------------------------------------- 1 | package column 2 | 3 | import ( 4 | "fmt" 5 | "unsafe" 6 | 7 | "github.com/vahid-sohrabloo/chconn/v2/internal/readerwriter" 8 | ) 9 | 10 | // Column use for most (fixed size) ClickHouse Columns type 11 | type Base[T comparable] struct { 12 | column 13 | size int 14 | numRow int 15 | values []T 16 | params []interface{} 17 | } 18 | 19 | // New create a new column 20 | func New[T comparable]() *Base[T] { 21 | var tmpValue T 22 | size := int(unsafe.Sizeof(tmpValue)) 23 | return &Base[T]{ 24 | size: size, 25 | } 26 | } 27 | 28 | // Data get all the data in current block as a slice. 29 | // 30 | // NOTE: the return slice only valid in current block, if you want to use it after, you should copy it. or use Read 31 | func (c *Base[T]) Data() []T { 32 | value := *(*[]T)(unsafe.Pointer(&c.b)) 33 | return value[:c.numRow] 34 | } 35 | 36 | // Read reads all the data in current block and append to the input. 37 | func (c *Base[T]) Read(value []T) []T { 38 | return append(value, c.Data()...) 39 | } 40 | 41 | // Row return the value of given row. 42 | // NOTE: Row number start from zero 43 | func (c *Base[T]) Row(row int) T { 44 | i := row * c.size 45 | return *(*T)(unsafe.Pointer(&c.b[i])) 46 | } 47 | 48 | // Append value for insert 49 | func (c *Base[T]) Append(v ...T) { 50 | c.values = append(c.values, v...) 51 | c.numRow += len(v) 52 | } 53 | 54 | // NumRow return number of row for this block 55 | func (c *Base[T]) NumRow() int { 56 | return c.numRow 57 | } 58 | 59 | // Array return a Array type for this column 60 | func (c *Base[T]) Array() *Array[T] { 61 | return NewArray[T](c) 62 | } 63 | 64 | // Nullable return a nullable type for this column 65 | func (c *Base[T]) Nullable() *Nullable[T] { 66 | return NewNullable[T](c) 67 | } 68 | 69 | // LC return a low cardinality type for this column 70 | func (c *Base[T]) LC() *LowCardinality[T] { 71 | return NewLC[T](c) 72 | } 73 | 74 | // LowCardinality return a low cardinality type for this column 75 | func (c *Base[T]) LowCardinality() *LowCardinality[T] { 76 | return NewLowCardinality[T](c) 77 | } 78 | 79 | // appendEmpty append empty value for insert 80 | // this use internally for nullable and low cardinality nullable column 81 | func (c *Base[T]) appendEmpty() { 82 | var emptyValue T 83 | c.Append(emptyValue) 84 | } 85 | 86 | // Reset all statuses and buffered data 87 | // 88 | // After each reading, the reading data does not need to be reset. It will be automatically reset. 89 | // 90 | // When inserting, buffers are reset only after the operation is successful. 91 | // If an error occurs, you can safely call insert again. 92 | func (c *Base[T]) Reset() { 93 | c.numRow = 0 94 | c.values = c.values[:0] 95 | } 96 | 97 | // SetWriteBufferSize set write buffer (number of rows) 98 | // this buffer only used for writing. 99 | // By setting this buffer, you will avoid allocating the memory several times. 100 | func (c *Base[T]) SetWriteBufferSize(row int) { 101 | if cap(c.values) < row { 102 | c.values = make([]T, 0, row) 103 | } 104 | } 105 | 106 | // ReadRaw read raw data from the reader. it runs automatically 107 | func (c *Base[T]) ReadRaw(num int, r *readerwriter.Reader) error { 108 | c.Reset() 109 | c.r = r 110 | c.numRow = num 111 | c.totalByte = num * c.size 112 | err := c.readBuffer() 113 | if err != nil { 114 | err = fmt.Errorf("read data: %w", err) 115 | } 116 | c.readyBufferHook() 117 | return err 118 | } 119 | 120 | func (c *Base[T]) readBuffer() error { 121 | if cap(c.b) < c.totalByte { 122 | c.b = make([]byte, c.totalByte) 123 | } else { 124 | c.b = c.b[:c.totalByte] 125 | } 126 | _, err := c.r.Read(c.b) 127 | return err 128 | } 129 | 130 | // HeaderReader reads header data from reader 131 | // it uses internally 132 | func (c *Base[T]) HeaderReader(r *readerwriter.Reader, readColumn bool, revision uint64) error { 133 | c.r = r 134 | return c.readColumn(readColumn, revision) 135 | } 136 | 137 | // HeaderWriter writes header data to writer 138 | // it uses internally 139 | func (c *Base[T]) HeaderWriter(w *readerwriter.Writer) { 140 | } 141 | 142 | func (c *Base[T]) Elem(arrayLevel int, nullable, lc bool) ColumnBasic { 143 | if nullable { 144 | return c.Nullable().elem(arrayLevel, lc) 145 | } 146 | if lc { 147 | return c.LowCardinality().elem(arrayLevel) 148 | } 149 | if arrayLevel > 0 { 150 | return c.Array().elem(arrayLevel - 1) 151 | } 152 | return c 153 | } 154 | -------------------------------------------------------------------------------- /column/nested_test.go: -------------------------------------------------------------------------------- 1 | package column_test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/assert" 10 | "github.com/stretchr/testify/require" 11 | "github.com/vahid-sohrabloo/chconn/v2" 12 | "github.com/vahid-sohrabloo/chconn/v2/column" 13 | "github.com/vahid-sohrabloo/chconn/v2/types" 14 | ) 15 | 16 | func TestNestedNoFlattened(t *testing.T) { 17 | tableName := "nested_no_flattened" 18 | 19 | t.Parallel() 20 | 21 | connString := os.Getenv("CHX_TEST_TCP_CONN_STRING") 22 | 23 | conn, err := chconn.Connect(context.Background(), connString) 24 | require.NoError(t, err) 25 | 26 | err = conn.Exec(context.Background(), 27 | fmt.Sprintf(`DROP TABLE IF EXISTS test_%s`, tableName), 28 | ) 29 | require.NoError(t, err) 30 | set := chconn.Settings{ 31 | { 32 | Name: "allow_suspicious_low_cardinality_types", 33 | Value: "true", 34 | }, 35 | { 36 | Name: "flatten_nested", 37 | Value: "false", 38 | }, 39 | } 40 | 41 | err = conn.ExecWithOption(context.Background(), fmt.Sprintf(`CREATE TABLE test_%[1]s ( 42 | col1 Nested(col1_n1 Int64, col2_n1 String), 43 | col2 Nested(col1_n2 Int64, col2_n2 Nested(col1_n2_n1 Int64, col2_n2_n2 String)) 44 | ) Engine=Memory`, tableName), &chconn.QueryOptions{ 45 | Settings: set, 46 | }) 47 | 48 | require.NoError(t, err) 49 | type Col1Type types.Tuple2[int64, string] 50 | col1 := column.NewNested2[Col1Type, int64, string](column.New[int64](), column.NewString()) 51 | 52 | type Col2Type types.Tuple2[int64, []Col1Type] 53 | 54 | col2N2 := column.NewNested2[Col1Type, int64, string](column.New[int64](), column.NewString()) 55 | col2 := column.NewNested2[Col2Type, int64, []Col1Type](column.New[int64](), col2N2) 56 | 57 | var col1Insert [][]Col1Type 58 | var col2Insert [][]Col2Type 59 | 60 | for insertN := 0; insertN < 2; insertN++ { 61 | rows := 10 62 | for i := 0; i < rows; i++ { 63 | valString := fmt.Sprintf("string %d", i) 64 | valInt := int64(i) 65 | val2String := fmt.Sprintf("string %d", i+1) 66 | val2Int := int64(i + 1) 67 | col1.Append([]Col1Type{ 68 | { 69 | Col1: valInt, 70 | Col2: valString, 71 | }, 72 | }, 73 | ) 74 | col1Insert = append(col1Insert, []Col1Type{ 75 | { 76 | Col1: valInt, 77 | Col2: valString, 78 | }, 79 | }) 80 | 81 | col2.Append([]Col2Type{ 82 | { 83 | Col1: valInt, 84 | Col2: []Col1Type{ 85 | { 86 | Col1: val2Int, 87 | Col2: val2String, 88 | }, 89 | }, 90 | }, 91 | }) 92 | col2Insert = append(col2Insert, []Col2Type{ 93 | { 94 | Col1: valInt, 95 | Col2: []Col1Type{ 96 | { 97 | Col1: val2Int, 98 | Col2: val2String, 99 | }, 100 | }, 101 | }, 102 | }) 103 | } 104 | 105 | err = conn.Insert(context.Background(), fmt.Sprintf(`INSERT INTO 106 | test_%[1]s ( 107 | col1, 108 | col2 109 | ) 110 | VALUES`, tableName), 111 | col1, 112 | col2, 113 | ) 114 | require.NoError(t, err) 115 | } 116 | 117 | // example read all 118 | 119 | col1Read := column.NewTuple2[Col1Type, int64, string](column.New[int64](), column.NewString()).Array() 120 | 121 | col2N2Read := column.NewNested2[Col1Type, int64, string](column.New[int64](), column.NewString()) 122 | col2Read := column.NewNested2[Col2Type, int64, []Col1Type](column.New[int64](), col2N2Read) 123 | selectStmt, err := conn.Select(context.Background(), fmt.Sprintf(`SELECT 124 | col1,col2 125 | FROM test_%[1]s`, tableName), 126 | col1Read, 127 | col2Read) 128 | 129 | require.NoError(t, err) 130 | require.True(t, conn.IsBusy()) 131 | var col1Data [][]Col1Type 132 | var col2Data [][]Col2Type 133 | 134 | for selectStmt.Next() { 135 | col1Data = col1Read.Read(col1Data) 136 | col2Data = col2Read.Read(col2Data) 137 | } 138 | 139 | require.NoError(t, selectStmt.Err()) 140 | 141 | assert.Equal(t, col1Insert, col1Data) 142 | assert.Equal(t, col2Insert, col2Data) 143 | 144 | // // check dynamic column 145 | selectStmt, err = conn.Select(context.Background(), fmt.Sprintf(`SELECT 146 | col1, col2 147 | FROM test_%[1]s`, tableName)) 148 | 149 | require.NoError(t, err) 150 | autoColumns := selectStmt.Columns() 151 | 152 | assert.Len(t, autoColumns, 2) 153 | 154 | assert.Equal(t, column.NewTuple(column.New[int64](), column.NewString()).Array().ColumnType(), autoColumns[0].ColumnType()) 155 | assert.Equal(t, 156 | column.NewTuple(column.New[int64](), 157 | column.NewTuple(column.New[int64](), column.NewString()).Array()).Array(). 158 | ColumnType(), autoColumns[1].ColumnType()) 159 | 160 | for selectStmt.Next() { 161 | } 162 | require.NoError(t, selectStmt.Err()) 163 | selectStmt.Close() 164 | } 165 | -------------------------------------------------------------------------------- /internal/ctxwatch/context_watcher_test.go: -------------------------------------------------------------------------------- 1 | package ctxwatch_test 2 | 3 | import ( 4 | "context" 5 | "sync/atomic" 6 | "testing" 7 | "time" 8 | 9 | "github.com/stretchr/testify/require" 10 | "github.com/vahid-sohrabloo/chconn/v2/internal/ctxwatch" 11 | ) 12 | 13 | func TestContextWatcherContextCancelled(t *testing.T) { 14 | canceledChan := make(chan struct{}) 15 | cleanupCalled := false 16 | cw := ctxwatch.NewContextWatcher(func() { 17 | canceledChan <- struct{}{} 18 | }, func() { 19 | cleanupCalled = true 20 | }) 21 | 22 | ctx, cancel := context.WithCancel(context.Background()) 23 | cw.Watch(ctx) 24 | cancel() 25 | 26 | select { 27 | case <-canceledChan: 28 | case <-time.NewTimer(time.Second).C: 29 | t.Fatal("Timed out waiting for cancel func to be called") 30 | } 31 | 32 | cw.Unwatch() 33 | 34 | require.True(t, cleanupCalled, "Cleanup func was not called") 35 | } 36 | 37 | func TestContextWatcherUnwatchdBeforeContextCancelled(t *testing.T) { 38 | cw := ctxwatch.NewContextWatcher(func() { 39 | t.Error("cancel func should not have been called") 40 | }, func() { 41 | t.Error("cleanup func should not have been called") 42 | }) 43 | 44 | ctx, cancel := context.WithCancel(context.Background()) 45 | cw.Watch(ctx) 46 | cw.Unwatch() 47 | cancel() 48 | } 49 | 50 | func TestContextWatcherMultipleWatchPanics(t *testing.T) { 51 | cw := ctxwatch.NewContextWatcher(func() {}, func() {}) 52 | 53 | ctx, cancel := context.WithCancel(context.Background()) 54 | defer cancel() 55 | cw.Watch(ctx) 56 | 57 | ctx2, cancel2 := context.WithCancel(context.Background()) 58 | defer cancel2() 59 | require.Panics(t, func() { cw.Watch(ctx2) }, "Expected panic when Watch called multiple times") 60 | } 61 | 62 | func TestContextWatcherUnwatchWhenNotWatchingIsSafe(t *testing.T) { 63 | cw := ctxwatch.NewContextWatcher(func() {}, func() {}) 64 | cw.Unwatch() // unwatch when not / never watching 65 | 66 | ctx, cancel := context.WithCancel(context.Background()) 67 | defer cancel() 68 | cw.Watch(ctx) 69 | cw.Unwatch() 70 | cw.Unwatch() // double unwatch 71 | } 72 | 73 | func TestContextWatcherUnwatchIsConcurrencySafe(t *testing.T) { 74 | cw := ctxwatch.NewContextWatcher(func() {}, func() {}) 75 | 76 | ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) 77 | defer cancel() 78 | cw.Watch(ctx) 79 | 80 | go cw.Unwatch() 81 | go cw.Unwatch() 82 | 83 | <-ctx.Done() 84 | } 85 | 86 | //nolint:govet 87 | func TestContextWatcherStress(t *testing.T) { 88 | var cancelFuncCalls int64 89 | var cleanupFuncCalls int64 90 | 91 | cw := ctxwatch.NewContextWatcher(func() { 92 | atomic.AddInt64(&cancelFuncCalls, 1) 93 | }, func() { 94 | atomic.AddInt64(&cleanupFuncCalls, 1) 95 | }) 96 | 97 | cycleCount := 100000 98 | 99 | for i := 0; i < cycleCount; i++ { 100 | //nolint:govet 101 | ctx, cancel := context.WithCancel(context.Background()) 102 | cw.Watch(ctx) 103 | if i%2 == 0 { 104 | cancel() 105 | } 106 | 107 | // Without time.Sleep, cw.Unwatch will almost always run before the cancel func which means cancel will never happen. 108 | // This gives us a better mix. 109 | if i%3 == 0 { 110 | time.Sleep(time.Nanosecond) 111 | } 112 | 113 | cw.Unwatch() 114 | if i%2 == 1 { 115 | cancel() 116 | } 117 | } 118 | 119 | actualCancelFuncCalls := atomic.LoadInt64(&cancelFuncCalls) 120 | actualCleanupFuncCalls := atomic.LoadInt64(&cleanupFuncCalls) 121 | 122 | if actualCancelFuncCalls == 0 { 123 | t.Fatal("actualCancelFuncCalls == 0") 124 | } 125 | 126 | maxCancelFuncCalls := int64(cycleCount) / 2 127 | if actualCancelFuncCalls > maxCancelFuncCalls { 128 | t.Errorf("cancel func calls should be no more than %d but was %d", actualCancelFuncCalls, maxCancelFuncCalls) 129 | } 130 | 131 | if actualCancelFuncCalls != actualCleanupFuncCalls { 132 | t.Errorf("cancel func calls (%d) should be equal to cleanup func calls (%d) but was not", actualCancelFuncCalls, actualCleanupFuncCalls) 133 | } 134 | } 135 | 136 | func BenchmarkContextWatcherUncancellable(b *testing.B) { 137 | cw := ctxwatch.NewContextWatcher(func() {}, func() {}) 138 | 139 | for i := 0; i < b.N; i++ { 140 | cw.Watch(context.Background()) 141 | cw.Unwatch() 142 | } 143 | } 144 | 145 | func BenchmarkContextWatcherCancelled(b *testing.B) { 146 | cw := ctxwatch.NewContextWatcher(func() {}, func() {}) 147 | 148 | for i := 0; i < b.N; i++ { 149 | ctx, cancel := context.WithCancel(context.Background()) 150 | cw.Watch(ctx) 151 | cancel() 152 | cw.Unwatch() 153 | } 154 | } 155 | 156 | func BenchmarkContextWatcherCancellable(b *testing.B) { 157 | cw := ctxwatch.NewContextWatcher(func() {}, func() {}) 158 | 159 | ctx, cancel := context.WithCancel(context.Background()) 160 | defer cancel() 161 | 162 | for i := 0; i < b.N; i++ { 163 | cw.Watch(ctx) 164 | cw.Unwatch() 165 | } 166 | } 167 | -------------------------------------------------------------------------------- /column/tuples_template/tuple.go.tmpl: -------------------------------------------------------------------------------- 1 | package column 2 | 3 | import ( 4 | "unsafe" 5 | ) 6 | 7 | type tuple{{.Numbrer}}Value[T1{{- range $val := iterate .Numbrer "2" }}, T{{ $val }}{{end }} any] struct { 8 | {{- range $val := iterate .Numbrer "1" }} 9 | Col{{ $val }} T{{ $val }}{{end }} 10 | } 11 | 12 | // Tuple{{.Numbrer}} is a column of Tuple(T1{{- range $val := iterate .Numbrer "2" }}, T{{ $val }}{{end }}) ClickHouse data type 13 | type Tuple{{.Numbrer}}[T ~struct { 14 | {{- range $val := iterate .Numbrer "1" }} 15 | Col{{ $val }} T{{ $val }}{{end }} 16 | }{{- range $val := iterate .Numbrer "1" }}, T{{ $val }}{{end }} any] struct { 17 | Tuple 18 | {{- range $val := iterate .Numbrer "1" }} 19 | col{{ $val }} Column[T{{ $val }}]{{end }} 20 | } 21 | 22 | // NewTuple{{.Numbrer}} create a new tuple of Tuple(T1{{- range $val := iterate .Numbrer "2" }}, T{{ $val }}{{end }}) ClickHouse data type 23 | func NewTuple{{.Numbrer}}[T ~struct { 24 | {{- range $val := iterate .Numbrer "1" }} 25 | Col{{ $val }} T{{ $val }}{{end }} 26 | }{{- range $val := iterate .Numbrer "1" }}, T{{ $val }}{{end }} any]( 27 | {{- range $val := iterate .Numbrer "1" }} 28 | column{{ $val }} Column[T{{ $val }}],{{end }} 29 | ) *Tuple{{.Numbrer}}[T{{- range $val := iterate .Numbrer "1" }} ,T{{$val}}{{end}}] { 30 | return &Tuple{{.Numbrer}}[T{{- range $val := iterate .Numbrer "1" }} ,T{{$val}}{{end}}]{ 31 | Tuple: Tuple{ 32 | columns: []ColumnBasic{ 33 | {{- range $val := iterate .Numbrer "1" }} 34 | column{{ $val }},{{end }} 35 | }, 36 | }, 37 | {{- range $val := iterate .Numbrer "1" }} 38 | col{{ $val }}: column{{ $val }},{{end }} 39 | } 40 | } 41 | 42 | // NewNested{{.Numbrer}} create a new nested of Nested(T1{{- range $val := iterate .Numbrer "2" }}, T{{ $val }}{{end }}) ClickHouse data type 43 | // 44 | // this is actually an alias for NewTuple{{.Numbrer}}(T1{{- range $val := iterate .Numbrer "2" }}, T{{ $val }}{{end }}).Array() 45 | func NewNested{{.Numbrer}}[T ~struct { 46 | {{- range $val := iterate .Numbrer "1" }} 47 | Col{{ $val }} T{{ $val }}{{end }} 48 | }{{- range $val := iterate .Numbrer "1" }}, T{{ $val }}{{end }} any]( 49 | {{- range $val := iterate .Numbrer "1" }} 50 | column{{ $val }} Column[T{{ $val }}],{{end }} 51 | ) *Array[T] { 52 | return NewTuple{{.Numbrer}}[T]( 53 | {{- range $val := iterate .Numbrer "1" }} 54 | column{{ $val }},{{end}} 55 | ).Array() 56 | } 57 | 58 | // Data get all the data in current block as a slice. 59 | func (c *Tuple{{.Numbrer}}[T{{- range $val := iterate .Numbrer "1" }} ,T{{$val}}{{end}}]) Data() []T { 60 | val := make([]T, c.NumRow()) 61 | for i := 0; i < c.NumRow(); i++ { 62 | val[i] = T(tuple{{.Numbrer}}Value[T1{{- range $val := iterate .Numbrer "2" }}, T{{ $val }}{{end }}]{ 63 | {{- range $val := iterate .Numbrer "1" }} 64 | Col{{ $val }}: c.col{{ $val }}.Row(i),{{end }} 65 | }) 66 | } 67 | return val 68 | } 69 | 70 | // Read reads all the data in current block and append to the input. 71 | func (c *Tuple{{.Numbrer}}[T{{- range $val := iterate .Numbrer "1" }} ,T{{$val}}{{end}}]) Read(value []T) []T { 72 | valTuple := *(*[]tuple{{.Numbrer}}Value[T1{{- range $val := iterate .Numbrer "2" }}, T{{ $val }}{{end }}])(unsafe.Pointer(&value)) 73 | if cap(valTuple)-len(valTuple) >= c.NumRow() { 74 | valTuple = valTuple[:len(value)+c.NumRow()] 75 | } else { 76 | valTuple = append(valTuple, make([]tuple{{.Numbrer}}Value[T1{{- range $val := iterate .Numbrer "2" }}, T{{ $val }}{{end }}], c.NumRow())...) 77 | } 78 | 79 | val := valTuple[len(valTuple)-c.NumRow():] 80 | for i := 0; i < c.NumRow(); i++ { 81 | {{- range $val := iterate .Numbrer "1" }} 82 | val[i].Col{{ $val }} = c.col{{ $val }}.Row(i){{end }} 83 | } 84 | return *(*[]T)(unsafe.Pointer(&valTuple)) 85 | } 86 | 87 | // Row return the value of given row. 88 | // NOTE: Row number start from zero 89 | func (c *Tuple{{.Numbrer}}[T{{- range $val := iterate .Numbrer "1" }} ,T{{$val}}{{end}}]) Row(row int) T { 90 | return T(tuple{{.Numbrer}}Value[T1{{- range $val := iterate .Numbrer "2" }}, T{{ $val }}{{end }}]{ 91 | {{- range $val := iterate .Numbrer "1" }} 92 | Col{{ $val }}: c.col{{ $val }}.Row(row),{{end }} 93 | }) 94 | } 95 | 96 | // Append value for insert 97 | func (c *Tuple{{.Numbrer}}[T{{- range $val := iterate .Numbrer "1" }} ,T{{$val}}{{end}}]) Append(v ...T) { 98 | for _, v := range v { 99 | t := tuple{{.Numbrer}}Value[T1{{- range $val := iterate .Numbrer "2" }}, T{{ $val }}{{end }}](v) 100 | {{- range $val := iterate .Numbrer "1" }} 101 | c.col{{ $val }}.Append(t.Col{{ $val }}){{end }} 102 | } 103 | } 104 | 105 | // Array return a Array type for this column 106 | func (c *Tuple{{.Numbrer}}[T{{- range $val := iterate .Numbrer "1" }} ,T{{$val}}{{end}}]) Array() *Array[T] { 107 | return NewArray[T](c) 108 | } 109 | -------------------------------------------------------------------------------- /column/tuple.go: -------------------------------------------------------------------------------- 1 | package column 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | 7 | "github.com/vahid-sohrabloo/chconn/v2/internal/helper" 8 | "github.com/vahid-sohrabloo/chconn/v2/internal/readerwriter" 9 | ) 10 | 11 | // Tuple is a column of Tuple(T1,T2,.....,Tn) ClickHouse data type 12 | // 13 | // this is actually a group of columns. it doesn't have any method for read or write data 14 | // 15 | // You MUST use this on Select and Insert methods and for append and read data use the sub columns 16 | type Tuple struct { 17 | column 18 | columns []ColumnBasic 19 | } 20 | 21 | // NewTuple create a new tuple of Tuple(T1,T2,.....,Tn) ClickHouse data type 22 | // 23 | // this is actually a group of columns. it doesn't have any method for read or write data 24 | // 25 | // You MUST use this on Select and Insert methods and for append and read data use the sub columns 26 | func NewTuple(columns ...ColumnBasic) *Tuple { 27 | if len(columns) < 1 { 28 | panic("tuple must have at least one column") 29 | } 30 | return &Tuple{ 31 | columns: columns, 32 | } 33 | } 34 | 35 | // NumRow return number of row for this block 36 | func (c *Tuple) NumRow() int { 37 | return c.columns[0].NumRow() 38 | } 39 | 40 | // Array return a Array type for this column 41 | func (c *Tuple) Array() *ArrayBase { 42 | return NewArrayBase(c) 43 | } 44 | 45 | // Reset all statuses and buffered data 46 | // 47 | // After each reading, the reading data does not need to be reset. It will be automatically reset. 48 | // 49 | // When inserting, buffers are reset only after the operation is successful. 50 | // If an error occurs, you can safely call insert again. 51 | func (c *Tuple) Reset() { 52 | for _, col := range c.columns { 53 | col.Reset() 54 | } 55 | } 56 | 57 | // SetWriteBufferSize set write buffer (number of rows) 58 | // this buffer only used for writing. 59 | // By setting this buffer, you will avoid allocating the memory several times. 60 | func (c *Tuple) SetWriteBufferSize(row int) { 61 | for _, col := range c.columns { 62 | col.SetWriteBufferSize(row) 63 | } 64 | } 65 | 66 | // ReadRaw read raw data from the reader. it runs automatically 67 | func (c *Tuple) ReadRaw(num int, r *readerwriter.Reader) error { 68 | for i, col := range c.columns { 69 | err := col.ReadRaw(num, r) 70 | if err != nil { 71 | return fmt.Errorf("tuple: read column index %d: %w", i, err) 72 | } 73 | } 74 | return nil 75 | } 76 | 77 | // HeaderReader reads header data from reader. 78 | // it uses internally 79 | func (c *Tuple) HeaderReader(r *readerwriter.Reader, readColumn bool, revision uint64) error { 80 | c.r = r 81 | err := c.readColumn(readColumn, revision) 82 | if err != nil { 83 | return err 84 | } 85 | 86 | for i, col := range c.columns { 87 | err = col.HeaderReader(r, false, revision) 88 | if err != nil { 89 | return fmt.Errorf("tuple: read column header index %d: %w", i, err) 90 | } 91 | } 92 | 93 | return nil 94 | } 95 | 96 | // Column returns the all sub columns 97 | func (c *Tuple) Columns() []ColumnBasic { 98 | return c.columns 99 | } 100 | 101 | func (c *Tuple) Validate() error { 102 | chType := helper.FilterSimpleAggregate(c.chType) 103 | if helper.IsPoint(chType) { 104 | chType = helper.PointMainTypeStr 105 | } 106 | 107 | if !helper.IsTuple(chType) { 108 | return ErrInvalidType{ 109 | column: c, 110 | } 111 | } 112 | 113 | columnsTuple, err := helper.TypesInParentheses(chType[helper.LenTupleStr : len(chType)-1]) 114 | if err != nil { 115 | return fmt.Errorf("tuple invalid types %w", err) 116 | } 117 | if len(columnsTuple) != len(c.columns) { 118 | //nolint:goerr113 119 | return fmt.Errorf("columns number for %s (%s) is not equal to tuple columns number: %d != %d", 120 | string(c.name), 121 | string(c.Type()), 122 | len(columnsTuple), 123 | len(c.columns), 124 | ) 125 | } 126 | 127 | for i, col := range c.columns { 128 | col.SetType(columnsTuple[i].ChType) 129 | col.SetName(columnsTuple[i].Name) 130 | if col.Validate() != nil { 131 | return ErrInvalidType{ 132 | column: c, 133 | } 134 | } 135 | } 136 | return nil 137 | } 138 | 139 | func (c *Tuple) ColumnType() string { 140 | str := helper.TupleStr 141 | for _, col := range c.columns { 142 | str += col.ColumnType() + "," 143 | } 144 | return str[:len(str)-1] + ")" 145 | } 146 | 147 | // WriteTo write data to ClickHouse. 148 | // it uses internally 149 | func (c *Tuple) WriteTo(w io.Writer) (int64, error) { 150 | var n int64 151 | for i, col := range c.columns { 152 | nw, err := col.WriteTo(w) 153 | if err != nil { 154 | return n, fmt.Errorf("tuple: write column index %d: %w", i, err) 155 | } 156 | n += nw 157 | } 158 | return n, nil 159 | } 160 | 161 | // HeaderWriter writes header data to writer 162 | // it uses internally 163 | func (c *Tuple) HeaderWriter(w *readerwriter.Writer) { 164 | for _, col := range c.columns { 165 | col.HeaderWriter(w) 166 | } 167 | } 168 | 169 | func (c *Tuple) Elem(arrayLevel int) ColumnBasic { 170 | if arrayLevel > 0 { 171 | return c.Array().elem(arrayLevel - 1) 172 | } 173 | return c 174 | } 175 | -------------------------------------------------------------------------------- /column/date.go: -------------------------------------------------------------------------------- 1 | package column 2 | 3 | import ( 4 | "strings" 5 | "time" 6 | "unsafe" 7 | ) 8 | 9 | // DateType is an interface to handle convert between time.Time and T. 10 | type DateType[T any] interface { 11 | comparable 12 | FromTime(val time.Time, precision int) T 13 | ToTime(val *time.Location, precision int) time.Time 14 | } 15 | 16 | // Date is a date column of ClickHouse date type (Date, Date32, DateTime, DateTime64). 17 | // it is a wrapper of time.Time. but if you want to work with the raw data like unix timestamp 18 | // you can directly use `Column` (`New[T]()`) 19 | // 20 | // `uint16` or `types.Date` or any 16 bits data types For `Date`. 21 | // 22 | // `uint32` or `types.Date32` or any 32 bits data types For `Date32` 23 | // 24 | // `uint32` or `types.DateTime` or any 32 bits data types For `DateTime` 25 | // 26 | // `uint64` or `types.DateTime64` or any 64 bits data types For `DateTime64` 27 | type Date[T DateType[T]] struct { 28 | Base[T] 29 | loc *time.Location 30 | precision int 31 | } 32 | 33 | // NewDate create a new date column of ClickHouse date type (Date, Date32, DateTime, DateTime64). 34 | // it is a wrapper of time.Time. but if you want to work with the raw data like unix timestamp 35 | // you can directly use `Column` (`New[T]()``) 36 | // 37 | // `uint16` or `types.Date` or any 16 bits data types For `Date`. 38 | // 39 | // `uint32` or `types.Date32` or any 32 bits data types For `Date32` 40 | // 41 | // `uint32` or `types.DateTime` or any 32 bits data types For `DateTime` 42 | // 43 | // `uint64` or `types.DateTime64` or any 64 bits data types For `DateTime64` 44 | // 45 | // ONLY ON SELECT, timezone set automatically for `DateTime` and `DateTime64` if not set and present in clickhouse datatype) 46 | 47 | func NewDate[T DateType[T]]() *Date[T] { 48 | var tmpValue T 49 | size := int(unsafe.Sizeof(tmpValue)) 50 | return &Date[T]{ 51 | Base: Base[T]{ 52 | size: size, 53 | }, 54 | } 55 | } 56 | 57 | // SetLocation set the location of the time.Time. Only use for `DateTime` and `DateTime64` 58 | func (c *Date[T]) SetLocation(loc *time.Location) *Date[T] { 59 | c.loc = loc 60 | return c 61 | } 62 | 63 | // Location get location 64 | // 65 | // ONLY ON SELECT, set automatically for `DateTime` and `DateTime64` if not set and present in clickhouse datatype) 66 | func (c *Date[T]) Location() *time.Location { 67 | if c.loc == nil && len(c.params) >= 2 && len(c.params[1].([]byte)) > 0 { 68 | loc, err := time.LoadLocation(strings.Trim(string(c.params[1].([]byte)), "'")) 69 | if err == nil { 70 | c.SetLocation(loc) 71 | } else { 72 | c.SetLocation(time.Local) 73 | } 74 | } 75 | if c.loc == nil { 76 | c.SetLocation(time.Local) 77 | } 78 | return c.loc 79 | } 80 | 81 | // SetPrecision set the precision of the time.Time. Only use for `DateTime64` 82 | func (c *Date[T]) SetPrecision(precision int) *Date[T] { 83 | c.precision = precision 84 | return c 85 | } 86 | 87 | // Data get all the data in current block as a slice. 88 | func (c *Date[T]) Data() []time.Time { 89 | values := make([]time.Time, c.numRow) 90 | for i := 0; i < c.numRow; i++ { 91 | values[i] = c.Row(i) 92 | } 93 | return values 94 | } 95 | 96 | // Read reads all the data in current block and append to the input. 97 | func (c *Date[T]) Read(value []time.Time) []time.Time { 98 | if cap(value)-len(value) >= c.NumRow() { 99 | value = (value)[:len(value)+c.NumRow()] 100 | } else { 101 | value = append(value, make([]time.Time, c.NumRow())...) 102 | } 103 | val := (value)[len(value)-c.NumRow():] 104 | for i := 0; i < c.NumRow(); i++ { 105 | val[i] = c.Row(i) 106 | } 107 | return value 108 | } 109 | 110 | // Row return the value of given row 111 | // NOTE: Row number start from zero 112 | func (c *Date[T]) Row(row int) time.Time { 113 | i := row * c.size 114 | return (*(*T)(unsafe.Pointer(&c.b[i]))).ToTime(c.Location(), c.precision) 115 | } 116 | 117 | // Append value for insert 118 | func (c *Date[T]) Append(v ...time.Time) { 119 | var val T 120 | for _, v := range v { 121 | c.values = append(c.values, val.FromTime(v, c.precision)) 122 | } 123 | c.numRow += len(v) 124 | } 125 | 126 | // Array return a Array type for this column 127 | func (c *Date[T]) Array() *Array[time.Time] { 128 | return NewArray[time.Time](c) 129 | } 130 | 131 | // Nullable return a nullable type for this column 132 | func (c *Date[T]) Nullable() *Nullable[time.Time] { 133 | return NewNullable[time.Time](c) 134 | } 135 | 136 | // LC return a low cardinality type for this column 137 | func (c *Date[T]) LC() *LowCardinality[time.Time] { 138 | return NewLC[time.Time](c) 139 | } 140 | 141 | // LowCardinality return a low cardinality type for this column 142 | func (c *Date[T]) LowCardinality() *LowCardinality[time.Time] { 143 | return NewLC[time.Time](c) 144 | } 145 | 146 | func (c *Date[T]) Elem(arrayLevel int, nullable, lc bool) ColumnBasic { 147 | if nullable { 148 | return c.Nullable().elem(arrayLevel, lc) 149 | } 150 | if lc { 151 | return c.LowCardinality().elem(arrayLevel) 152 | } 153 | if arrayLevel > 0 { 154 | return c.Array().elem(arrayLevel - 1) 155 | } 156 | return c 157 | } 158 | -------------------------------------------------------------------------------- /column/array_base.go: -------------------------------------------------------------------------------- 1 | package column 2 | 3 | import ( 4 | "encoding/binary" 5 | "fmt" 6 | "io" 7 | "strings" 8 | 9 | "github.com/vahid-sohrabloo/chconn/v2/internal/helper" 10 | "github.com/vahid-sohrabloo/chconn/v2/internal/readerwriter" 11 | ) 12 | 13 | // ArrayBase is a column of Array(T) ClickHouse data type 14 | // 15 | // ArrayBase is a base class for other arrays or use for none generic use 16 | type ArrayBase struct { 17 | column 18 | offsetColumn *Base[uint64] 19 | dataColumn ColumnBasic 20 | offset uint64 21 | resetHook func() 22 | } 23 | 24 | // NewArray create a new array column of Array(T) ClickHouse data type 25 | func NewArrayBase(dataColumn ColumnBasic) *ArrayBase { 26 | a := &ArrayBase{ 27 | dataColumn: dataColumn, 28 | offsetColumn: New[uint64](), 29 | } 30 | return a 31 | } 32 | 33 | // AppendLen Append len of array for insert 34 | func (c *ArrayBase) AppendLen(v int) { 35 | c.offset += uint64(v) 36 | c.offsetColumn.Append(c.offset) 37 | } 38 | 39 | // NumRow return number of row for this block 40 | func (c *ArrayBase) NumRow() int { 41 | return c.offsetColumn.NumRow() 42 | } 43 | 44 | // Array return a Array type for this column 45 | func (c *ArrayBase) Array() *ArrayBase { 46 | return NewArrayBase(c) 47 | } 48 | 49 | // Reset all statuses and buffered data 50 | // 51 | // After each reading, the reading data does not need to be reset. It will be automatically reset. 52 | // 53 | // When inserting, buffers are reset only after the operation is successful. 54 | // If an error occurs, you can safely call insert again. 55 | func (c *ArrayBase) Reset() { 56 | c.offsetColumn.Reset() 57 | c.dataColumn.Reset() 58 | c.offset = 0 59 | } 60 | 61 | // Offsets return all the offsets in current block 62 | // Note: Only available in the current block 63 | func (c *ArrayBase) Offsets() []uint64 { 64 | return c.offsetColumn.Data() 65 | } 66 | 67 | // TotalRows return total rows on this block of array data 68 | func (c *ArrayBase) TotalRows() int { 69 | if c.offsetColumn.totalByte == 0 { 70 | return 0 71 | } 72 | return int(binary.LittleEndian.Uint64(c.offsetColumn.b[c.offsetColumn.totalByte-8 : c.offsetColumn.totalByte])) 73 | } 74 | 75 | // SetWriteBufferSize set write buffer (number of rows) 76 | // this buffer only used for writing. 77 | // By setting this buffer, you will avoid allocating the memory several times. 78 | func (c *ArrayBase) SetWriteBufferSize(row int) { 79 | c.offsetColumn.SetWriteBufferSize(row) 80 | c.dataColumn.SetWriteBufferSize(row) 81 | } 82 | 83 | // ReadRaw read raw data from the reader. it runs automatically 84 | func (c *ArrayBase) ReadRaw(num int, r *readerwriter.Reader) error { 85 | c.offsetColumn.Reset() 86 | err := c.offsetColumn.ReadRaw(num, r) 87 | if err != nil { 88 | return fmt.Errorf("array: read offset column: %w", err) 89 | } 90 | err = c.dataColumn.ReadRaw(c.TotalRows(), r) 91 | if err != nil { 92 | return fmt.Errorf("array: read data column: %w", err) 93 | } 94 | 95 | if c.resetHook != nil { 96 | c.resetHook() 97 | } 98 | return nil 99 | } 100 | 101 | // HeaderReader reads header data from reader 102 | // it uses internally 103 | func (c *ArrayBase) HeaderReader(r *readerwriter.Reader, readColumn bool, revision uint64) error { 104 | c.r = r 105 | err := c.readColumn(readColumn, revision) 106 | if err != nil { 107 | return err 108 | } 109 | 110 | // never return error 111 | //nolint:errcheck 112 | c.offsetColumn.HeaderReader(r, false, revision) 113 | 114 | return c.dataColumn.HeaderReader(r, false, revision) 115 | } 116 | 117 | // Column returns the sub column 118 | func (c *ArrayBase) Column() ColumnBasic { 119 | return c.dataColumn 120 | } 121 | 122 | func (c *ArrayBase) Validate() error { 123 | chType := helper.FilterSimpleAggregate(c.chType) 124 | switch { 125 | case helper.IsRing(chType): 126 | chType = helper.RingMainTypeStr 127 | case helper.IsPolygon(chType): 128 | chType = helper.PolygonMainTypeStr 129 | case helper.IsMultiPolygon(chType): 130 | chType = helper.MultiPolygonMainTypeStr 131 | } 132 | 133 | chType = helper.NestedToArrayType(chType) 134 | 135 | if !helper.IsArray(chType) { 136 | return ErrInvalidType{ 137 | column: c, 138 | } 139 | } 140 | c.dataColumn.SetType(chType[helper.LenArrayStr : len(chType)-1]) 141 | if c.dataColumn.Validate() != nil { 142 | return ErrInvalidType{ 143 | column: c, 144 | } 145 | } 146 | return nil 147 | } 148 | 149 | func (c *ArrayBase) ColumnType() string { 150 | return strings.ReplaceAll(helper.ArrayTypeStr, "", c.dataColumn.ColumnType()) 151 | } 152 | 153 | // WriteTo write data to ClickHouse. 154 | // it uses internally 155 | func (c *ArrayBase) WriteTo(w io.Writer) (int64, error) { 156 | nw, err := c.offsetColumn.WriteTo(w) 157 | if err != nil { 158 | return 0, fmt.Errorf("write len data: %w", err) 159 | } 160 | n, errDataColumn := c.dataColumn.WriteTo(w) 161 | 162 | return nw + n, errDataColumn 163 | } 164 | 165 | // HeaderWriter writes header data to writer 166 | // it uses internally 167 | func (c *ArrayBase) HeaderWriter(w *readerwriter.Writer) { 168 | c.dataColumn.HeaderWriter(w) 169 | } 170 | 171 | func (c *ArrayBase) elem(arrayLevel int) ColumnBasic { 172 | if arrayLevel > 0 { 173 | return c.Array().elem(arrayLevel - 1) 174 | } 175 | return c 176 | } 177 | -------------------------------------------------------------------------------- /chpool/conn.go: -------------------------------------------------------------------------------- 1 | package chpool 2 | 3 | import ( 4 | "context" 5 | "sync/atomic" 6 | 7 | puddle "github.com/jackc/puddle/v2" 8 | "github.com/vahid-sohrabloo/chconn/v2" 9 | "github.com/vahid-sohrabloo/chconn/v2/column" 10 | ) 11 | 12 | // Conn is an acquired *chconn.Conn from a Pool. 13 | type Conn interface { 14 | Release() 15 | // ExecWithOption executes a query without returning any rows with Query options. 16 | // NOTE: don't use it for insert and select query 17 | ExecWithOption( 18 | ctx context.Context, 19 | query string, 20 | queryOptions *chconn.QueryOptions, 21 | ) error 22 | // Select executes a query with the the query options and return select stmt. 23 | // NOTE: only use for select query 24 | SelectWithOption( 25 | ctx context.Context, 26 | query string, 27 | queryOptions *chconn.QueryOptions, 28 | columns ...column.ColumnBasic, 29 | ) (chconn.SelectStmt, error) 30 | // InsertWithSetting executes a query with the query options and commit all columns data. 31 | // NOTE: only use for insert query 32 | InsertWithOption(ctx context.Context, query string, queryOptions *chconn.QueryOptions, columns ...column.ColumnBasic) error 33 | // InsertWithSetting executes a query with the query options and commit all columns data. 34 | // NOTE: only use for insert query 35 | InsertStreamWithOption(ctx context.Context, query string, queryOptions *chconn.QueryOptions) (chconn.InsertStmt, error) 36 | // Conn get the underlying chconn.Conn 37 | Conn() chconn.Conn 38 | // Hijack assumes ownership of the connection from the pool. Caller is responsible for closing the connection. Hijack 39 | // will panic if called on an already released or hijacked connection. 40 | Hijack() chconn.Conn 41 | Ping(ctx context.Context) error 42 | } 43 | type conn struct { 44 | res *puddle.Resource[*connResource] 45 | p *pool 46 | } 47 | 48 | // Release returns c to the pool it was acquired from. Once Release has been called, other methods must not be called. 49 | // However, it is safe to call Release multiple times. Subsequent calls after the first will be ignored. 50 | func (c *conn) Release() { 51 | if c.res == nil { 52 | return 53 | } 54 | 55 | conn := c.Conn() 56 | res := c.res 57 | c.res = nil 58 | 59 | if conn.IsClosed() || conn.IsBusy() { 60 | res.Destroy() 61 | // Signal to the health check to run since we just destroyed a connections 62 | // and we might be below minConns now 63 | c.p.triggerHealthCheck() 64 | return 65 | } 66 | 67 | // If the pool is consistently being used, we might never get to check the 68 | // lifetime of a connection since we only check idle connections in checkConnsHealth 69 | // so we also check the lifetime here and force a health check 70 | if c.p.isExpired(res) { 71 | atomic.AddInt64(&c.p.lifetimeDestroyCount, 1) 72 | res.Destroy() 73 | // Signal to the health check to run since we just destroyed a connections 74 | // and we might be below minConns now 75 | c.p.triggerHealthCheck() 76 | return 77 | } 78 | 79 | if c.p.afterRelease == nil { 80 | res.Release() 81 | return 82 | } 83 | 84 | go func() { 85 | if c.p.afterRelease(conn) { 86 | res.Release() 87 | } else { 88 | res.Destroy() 89 | // Signal to the health check to run since we just destroyed a connections 90 | // and we might be below minConns now 91 | c.p.triggerHealthCheck() 92 | } 93 | }() 94 | } 95 | 96 | // Hijack assumes ownership of the connection from the pool. Caller is responsible for closing the connection. Hijack 97 | // will panic if called on an already released or hijacked connection. 98 | func (c *conn) Hijack() chconn.Conn { 99 | if c.res == nil { 100 | panic("cannot hijack already released or hijacked connection") 101 | } 102 | 103 | conn := c.Conn() 104 | res := c.res 105 | c.res = nil 106 | 107 | res.Hijack() 108 | 109 | return conn 110 | } 111 | 112 | func (c *conn) ExecWithOption( 113 | ctx context.Context, 114 | query string, 115 | queryOptions *chconn.QueryOptions, 116 | ) error { 117 | return c.Conn().ExecWithOption(ctx, query, queryOptions) 118 | } 119 | 120 | func (c *conn) Ping(ctx context.Context) error { 121 | return c.Conn().Ping(ctx) 122 | } 123 | 124 | func (c *conn) SelectWithOption( 125 | ctx context.Context, 126 | query string, 127 | queryOptions *chconn.QueryOptions, 128 | columns ...column.ColumnBasic, 129 | ) (chconn.SelectStmt, error) { 130 | s, err := c.Conn().SelectWithOption(ctx, query, queryOptions, columns...) 131 | if err != nil { 132 | return nil, err 133 | } 134 | return &selectStmt{ 135 | SelectStmt: s, 136 | conn: c, 137 | }, nil 138 | } 139 | 140 | func (c *conn) InsertWithOption(ctx context.Context, query string, queryOptions *chconn.QueryOptions, columns ...column.ColumnBasic) error { 141 | return c.Conn().InsertWithOption(ctx, query, queryOptions, columns...) 142 | } 143 | func (c *conn) InsertStreamWithOption(ctx context.Context, query string, queryOptions *chconn.QueryOptions) (chconn.InsertStmt, error) { 144 | s, err := c.Conn().InsertStreamWithOption(ctx, query, queryOptions) 145 | if err != nil { 146 | return nil, err 147 | } 148 | return &insertStmt{ 149 | InsertStmt: s, 150 | conn: c, 151 | }, nil 152 | } 153 | 154 | func (c *conn) Conn() chconn.Conn { 155 | return c.connResource().conn 156 | } 157 | 158 | func (c *conn) connResource() *connResource { 159 | return c.res.Value() 160 | } 161 | -------------------------------------------------------------------------------- /column/base_validate.go: -------------------------------------------------------------------------------- 1 | package column 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "strconv" 7 | 8 | "github.com/vahid-sohrabloo/chconn/v2/internal/helper" 9 | ) 10 | 11 | var chColumnByteSize = map[string]int{ 12 | "Bool": 1, 13 | "Int8": 1, 14 | "Int16": 2, 15 | "Int32": 4, 16 | "Int64": 8, 17 | "Int128": 16, 18 | "Int256": 32, 19 | "UInt8": 1, 20 | "UInt16": 2, 21 | "UInt32": 4, 22 | "UInt64": 8, 23 | "UInt128": 16, 24 | "UInt256": 32, 25 | "Float32": 4, 26 | "Float64": 8, 27 | "Date": 2, 28 | "Date32": 4, 29 | "DateTime": 4, 30 | "DateTime64": 8, 31 | "UUID": 16, 32 | "IPv4": 4, 33 | "IPv6": 16, 34 | } 35 | 36 | var byteChColumnType = map[int]string{ 37 | 1: "Int8|UInt8|Enum8", 38 | 2: "Int16|UInt16|Enum16|Date", 39 | 4: "Int32|UInt32|Float32|Decimal32|Date32|DateTime|IPv4", 40 | 8: "Int64|UInt64|Float64|Decimal64|DateTime64", 41 | 16: "Int128|UInt128|Decimal128|IPv6|UUID", 42 | 32: "Int256|UInt256|Decimal256", 43 | } 44 | 45 | func (c *Base[T]) Validate() error { 46 | chType := helper.FilterSimpleAggregate(c.chType) 47 | if byteSize, ok := chColumnByteSize[string(chType)]; ok { 48 | if byteSize != c.size { 49 | return &ErrInvalidType{ 50 | column: c, 51 | } 52 | } 53 | return nil 54 | } 55 | 56 | if ok, err := c.checkEnum8(chType); ok { 57 | return err 58 | } 59 | 60 | if ok, err := c.checkEnum16(chType); ok { 61 | return err 62 | } 63 | 64 | if ok, err := c.checkDateTime(chType); ok { 65 | return err 66 | } 67 | 68 | if ok, err := c.checkDateTime(chType); ok { 69 | return err 70 | } 71 | 72 | if ok, err := c.checkDateTime64(chType); ok { 73 | return err 74 | } 75 | if ok, err := c.checkFixedString(chType); ok { 76 | return err 77 | } 78 | if ok, err := c.checkDecimal(chType); ok { 79 | return err 80 | } 81 | 82 | return &ErrInvalidType{ 83 | column: c, 84 | } 85 | } 86 | 87 | func (c *Base[T]) checkEnum8(chType []byte) (bool, error) { 88 | if helper.IsEnum8(chType) { 89 | if c.size != Uint8Size { 90 | return true, &ErrInvalidType{ 91 | column: c, 92 | } 93 | } 94 | return true, nil 95 | } 96 | return false, nil 97 | } 98 | 99 | func (c *Base[T]) checkEnum16(chType []byte) (bool, error) { 100 | if helper.IsEnum16(chType) { 101 | if c.size != Uint16Size { 102 | return true, &ErrInvalidType{ 103 | column: c, 104 | } 105 | } 106 | return true, nil 107 | } 108 | return false, nil 109 | } 110 | 111 | func (c *Base[T]) checkDateTime(chType []byte) (bool, error) { 112 | if helper.IsDateTimeWithParam(chType) { 113 | if c.size != 4 { 114 | return true, &ErrInvalidType{ 115 | column: c, 116 | } 117 | } 118 | c.params = []interface{}{ 119 | // precision 120 | 0, 121 | // timezone 122 | chType[helper.DateTimeStrLen : len(chType)-1], 123 | } 124 | return true, nil 125 | } 126 | return false, nil 127 | } 128 | 129 | func (c *Base[T]) checkDateTime64(chType []byte) (bool, error) { 130 | if helper.IsDateTime64(chType) { 131 | if c.size != 8 { 132 | return true, &ErrInvalidType{ 133 | column: c, 134 | } 135 | } 136 | parts := bytes.Split(chType[helper.DecimalStrLen:len(chType)-1], []byte(", ")) 137 | c.params = []interface{}{ 138 | parts[0], 139 | []byte{}, 140 | } 141 | if len(parts) > 1 { 142 | c.params[1] = parts[1] 143 | } 144 | return true, nil 145 | } 146 | return false, nil 147 | } 148 | 149 | func (c *Base[T]) checkFixedString(chType []byte) (bool, error) { 150 | if helper.IsFixedString(chType) { 151 | size, err := strconv.Atoi(string(chType[helper.FixedStringStrLen : len(chType)-1])) 152 | if err != nil { 153 | return true, fmt.Errorf("invalid size: %s", err) 154 | } 155 | if c.size != size { 156 | return true, &ErrInvalidType{ 157 | column: c, 158 | } 159 | } 160 | return true, nil 161 | } 162 | return false, nil 163 | } 164 | 165 | func (c *Base[T]) checkDecimal(chType []byte) (bool, error) { 166 | if helper.IsDecimal(chType) { 167 | parts := bytes.Split(chType[helper.DecimalStrLen:len(chType)-1], []byte(", ")) 168 | if len(parts) != 2 { 169 | return true, fmt.Errorf("invalid decimal type (should have precision and scale): %s", c.chType) 170 | } 171 | 172 | precision, err := strconv.Atoi(string(parts[0])) 173 | if err != nil { 174 | return true, fmt.Errorf("invalid precision: %s", err) 175 | } 176 | scale, err := strconv.Atoi(string(parts[1])) 177 | if err != nil { 178 | return true, fmt.Errorf("invalid scale: %s", err) 179 | } 180 | c.params = []interface{}{precision, scale} 181 | var size int 182 | switch { 183 | case precision >= 1 && precision <= 9: 184 | size = 4 185 | case precision >= 10 && precision <= 18: 186 | size = 8 187 | case precision >= 19 && precision <= 38: 188 | size = 16 189 | case precision >= 39 && precision <= 76: 190 | size = 32 191 | default: 192 | return true, fmt.Errorf("invalid precision: %d. it should be between 1 and 76", precision) 193 | } 194 | if c.size != size { 195 | return true, &ErrInvalidType{ 196 | column: c, 197 | } 198 | } 199 | return true, nil 200 | } 201 | return false, nil 202 | } 203 | 204 | func (c *Base[T]) ColumnType() string { 205 | if ok, _ := c.checkFixedString(c.chType); !ok { 206 | if str, ok := byteChColumnType[c.size]; ok { 207 | return str 208 | } 209 | } 210 | return fmt.Sprintf("T(%d bytes size)", c.size) 211 | } 212 | -------------------------------------------------------------------------------- /column/nullable_test.go: -------------------------------------------------------------------------------- 1 | package column_test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/assert" 10 | "github.com/stretchr/testify/require" 11 | "github.com/vahid-sohrabloo/chconn/v2" 12 | "github.com/vahid-sohrabloo/chconn/v2/column" 13 | ) 14 | 15 | func TestNullableAsNormal(t *testing.T) { 16 | tableName := "nullable" 17 | 18 | t.Parallel() 19 | 20 | connString := os.Getenv("CHX_TEST_TCP_CONN_STRING") 21 | 22 | conn, err := chconn.Connect(context.Background(), connString) 23 | require.NoError(t, err) 24 | 25 | err = conn.Exec(context.Background(), 26 | fmt.Sprintf(`DROP TABLE IF EXISTS test_%s`, tableName), 27 | ) 28 | require.NoError(t, err) 29 | set := chconn.Settings{ 30 | { 31 | Name: "allow_suspicious_low_cardinality_types", 32 | Value: "true", 33 | }, 34 | } 35 | err = conn.ExecWithOption(context.Background(), fmt.Sprintf(`CREATE TABLE test_%[1]s ( 36 | block_id UInt8, 37 | %[1]s_nullable Nullable(Int64), 38 | %[1]s_array_nullable Array(Nullable(Int64)), 39 | %[1]s_nullable_lc LowCardinality(Nullable(Int64)), 40 | %[1]s_array_lc_nullable Array(LowCardinality(Nullable(Int64))) 41 | ) Engine=Memory`, tableName), &chconn.QueryOptions{ 42 | Settings: set, 43 | }) 44 | 45 | require.NoError(t, err) 46 | 47 | blockID := column.New[uint8]() 48 | colNullable := column.New[int64]().Nullable() 49 | colNullableArray := column.New[int64]().Nullable().Array() 50 | colLCNullable := column.New[int64]().Nullable().LC() 51 | colArrayLCNullable := column.New[int64]().Nullable().LC().Array() 52 | 53 | var colInsert []int64 54 | var colArrayInsert [][]int64 55 | 56 | for insertN := 0; insertN < 2; insertN++ { 57 | rows := 10 58 | for i := 0; i < rows; i++ { 59 | val := int64(i + 1) 60 | blockID.Append(uint8(insertN)) 61 | colNullable.Append(val) 62 | colNullableArray.Append([]int64{val, val + 1}) 63 | colLCNullable.Append(val) 64 | colArrayLCNullable.Append([]int64{val, val + 1}) 65 | colInsert = append(colInsert, val) 66 | colArrayInsert = append(colArrayInsert, []int64{val, val + 1}) 67 | } 68 | 69 | err = conn.Insert(context.Background(), fmt.Sprintf(`INSERT INTO 70 | test_%[1]s ( 71 | block_id, 72 | %[1]s_nullable, 73 | %[1]s_array_nullable, 74 | %[1]s_nullable_lc, 75 | %[1]s_array_lc_nullable 76 | ) 77 | VALUES`, tableName), 78 | blockID, 79 | colNullable, 80 | colNullableArray, 81 | colLCNullable, 82 | colArrayLCNullable, 83 | ) 84 | require.NoError(t, err) 85 | } 86 | 87 | // test read row 88 | colNullableRead := column.New[int64]().Nullable() 89 | colNullableArrayRead := column.New[int64]().Nullable().Array() 90 | colLCNullableRead := column.New[int64]().Nullable().LC() 91 | colArrayLCNullableRead := column.New[int64]().Nullable().LC().Array() 92 | 93 | selectStmt, err := conn.Select(context.Background(), fmt.Sprintf(`SELECT 94 | %[1]s_nullable, 95 | %[1]s_array_nullable, 96 | %[1]s_nullable_lc, 97 | %[1]s_array_lc_nullable 98 | FROM test_%[1]s order by block_id`, tableName), 99 | colNullableRead, 100 | colNullableArrayRead, 101 | colLCNullableRead, 102 | colArrayLCNullableRead, 103 | ) 104 | 105 | require.NoError(t, err) 106 | require.True(t, conn.IsBusy()) 107 | 108 | var colData []int64 109 | var colArrayData [][]int64 110 | var colLCData []int64 111 | var colLCArrayData [][]int64 112 | var colDataNilRead []bool 113 | var colDataNilData []bool 114 | 115 | for selectStmt.Next() { 116 | colData = colNullableRead.Read(colData) 117 | colDataNilRead = colNullableRead.ReadNil(colDataNilRead) 118 | colDataNilData = append(colDataNilData, colNullableRead.DataNil()...) 119 | colArrayData = colNullableArrayRead.Read(colArrayData) 120 | colLCData = colLCNullableRead.Read(colLCData) 121 | colLCArrayData = colArrayLCNullableRead.Read(colLCArrayData) 122 | } 123 | 124 | require.NoError(t, selectStmt.Err()) 125 | assert.Equal(t, colInsert, colData) 126 | assert.Equal(t, colArrayInsert, colArrayData) 127 | assert.Equal(t, colInsert, colLCData) 128 | assert.Equal(t, colArrayInsert, colLCArrayData) 129 | assert.Equal(t, colDataNilRead, colDataNilData) 130 | assert.Equal(t, make([]bool, len(colInsert)), colDataNilRead) 131 | 132 | // test read all 133 | colNullableRead = column.New[int64]().Nullable() 134 | colNullableArrayRead = column.New[int64]().Nullable().Array() 135 | colLCNullableRead = column.New[int64]().Nullable().LC() 136 | colArrayLCNullableRead = column.New[int64]().Nullable().LC().Array() 137 | selectStmt, err = conn.Select(context.Background(), fmt.Sprintf(`SELECT 138 | %[1]s_nullable, 139 | %[1]s_array_nullable, 140 | %[1]s_nullable_lc, 141 | %[1]s_array_lc_nullable 142 | FROM test_%[1]s order by block_id`, tableName), 143 | colNullableRead, 144 | colNullableArrayRead, 145 | colLCNullableRead, 146 | colArrayLCNullableRead, 147 | ) 148 | 149 | require.NoError(t, err) 150 | require.True(t, conn.IsBusy()) 151 | 152 | colData = colData[:0] 153 | colArrayData = colArrayData[:0] 154 | colLCData = colLCData[:0] 155 | colLCArrayData = colLCArrayData[:0] 156 | 157 | for selectStmt.Next() { 158 | for i := 0; i < selectStmt.RowsInBlock(); i++ { 159 | colData = append(colData, colNullableRead.Row(i)) 160 | colArrayData = append(colArrayData, colNullableArrayRead.Row(i)) 161 | colLCData = append(colLCData, colLCNullableRead.Row(i)) 162 | colLCArrayData = append(colLCArrayData, colArrayLCNullableRead.Row(i)) 163 | } 164 | } 165 | 166 | require.NoError(t, selectStmt.Err()) 167 | assert.Equal(t, colInsert, colData) 168 | assert.Equal(t, colArrayInsert, colArrayData) 169 | assert.Equal(t, colInsert, colLCData) 170 | assert.Equal(t, colArrayInsert, colLCArrayData) 171 | } 172 | -------------------------------------------------------------------------------- /chpool/common_test.go: -------------------------------------------------------------------------------- 1 | package chpool 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "time" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | "github.com/vahid-sohrabloo/chconn/v2" 11 | "github.com/vahid-sohrabloo/chconn/v2/column" 12 | ) 13 | 14 | // Conn.Release is an asynchronous process that returns immediately. There is no signal when the actual work is 15 | // completed. To test something that relies on the actual work for Conn.Release being completed we must simply wait. 16 | // This function wraps the sleep so there is more meaning for the callers. 17 | func waitForReleaseToComplete() { 18 | time.Sleep(500 * time.Millisecond) 19 | } 20 | 21 | type execer interface { 22 | Exec(ctx context.Context, sql string) error 23 | } 24 | 25 | func testExec(t *testing.T, db execer) { 26 | err := db.Exec(context.Background(), "SET enable_http_compression=1") 27 | require.NoError(t, err) 28 | } 29 | 30 | type selecter interface { 31 | Select(ctx context.Context, query string, columns ...column.ColumnBasic) (chconn.SelectStmt, error) 32 | } 33 | 34 | func testSelect(t *testing.T, db selecter) { 35 | var ( 36 | num []uint64 37 | ) 38 | col := column.New[uint64]() 39 | stmt, err := db.Select(context.Background(), "SELECT * FROM system.numbers LIMIT 5;", col) 40 | require.NoError(t, err) 41 | for stmt.Next() { 42 | assert.NoError(t, err) 43 | num = col.Read(num) 44 | assert.NoError(t, err) 45 | } 46 | assert.NoError(t, stmt.Err()) 47 | assert.Equal(t, 5, len(num)) 48 | stmt.Close() 49 | assert.ElementsMatch(t, []uint64{0, 1, 2, 3, 4}, num) 50 | } 51 | 52 | func assertConfigsEqual(t *testing.T, expected, actual *Config, testName string) { 53 | if !assert.NotNil(t, expected) { 54 | return 55 | } 56 | if !assert.NotNil(t, actual) { 57 | return 58 | } 59 | 60 | assert.Equalf(t, expected.ConnString(), actual.ConnString(), "%s - ConnString", testName) 61 | 62 | // Can't test function equality, so just test that they are set or not. 63 | assert.Equalf(t, expected.AfterConnect == nil, actual.AfterConnect == nil, "%s - AfterConnect", testName) 64 | assert.Equalf(t, expected.BeforeAcquire == nil, actual.BeforeAcquire == nil, "%s - BeforeAcquire", testName) 65 | assert.Equalf(t, expected.AfterRelease == nil, actual.AfterRelease == nil, "%s - AfterRelease", testName) 66 | 67 | assert.Equalf(t, expected.MaxConnLifetime, actual.MaxConnLifetime, "%s - MaxConnLifetime", testName) 68 | assert.Equalf(t, expected.MaxConnIdleTime, actual.MaxConnIdleTime, "%s - MaxConnIdleTime", testName) 69 | assert.Equalf(t, expected.MaxConns, actual.MaxConns, "%s - MaxConns", testName) 70 | assert.Equalf(t, expected.MinConns, actual.MinConns, "%s - MinConns", testName) 71 | assert.Equalf(t, expected.HealthCheckPeriod, actual.HealthCheckPeriod, "%s - HealthCheckPeriod", testName) 72 | 73 | assertConnConfigsEqual(t, expected.ConnConfig, actual.ConnConfig, testName) 74 | } 75 | 76 | func assertConnConfigsEqual(t *testing.T, expected, actual *chconn.Config, testName string) { 77 | if !assert.NotNil(t, expected) { 78 | return 79 | } 80 | if !assert.NotNil(t, actual) { 81 | return 82 | } 83 | 84 | assert.Equalf(t, expected.ConnString(), actual.ConnString(), "%s - ConnString", testName) 85 | 86 | assert.Equalf(t, expected.Host, actual.Host, "%s - Host", testName) 87 | assert.Equalf(t, expected.Database, actual.Database, "%s - Database", testName) 88 | assert.Equalf(t, expected.Port, actual.Port, "%s - Port", testName) 89 | assert.Equalf(t, expected.User, actual.User, "%s - User", testName) 90 | assert.Equalf(t, expected.Password, actual.Password, "%s - Password", testName) 91 | assert.Equalf(t, expected.ConnectTimeout, actual.ConnectTimeout, "%s - ConnectTimeout", testName) 92 | assert.Equalf(t, expected.RuntimeParams, actual.RuntimeParams, "%s - RuntimeParams", testName) 93 | 94 | // Can't test function equality, so just test that they are set or not. 95 | assert.Equalf(t, expected.ValidateConnect == nil, actual.ValidateConnect == nil, "%s - ValidateConnect", testName) 96 | assert.Equalf(t, expected.AfterConnect == nil, actual.AfterConnect == nil, "%s - AfterConnect", testName) 97 | 98 | if assert.Equalf(t, expected.TLSConfig == nil, actual.TLSConfig == nil, "%s - TLSConfig", testName) { 99 | if expected.TLSConfig != nil { 100 | assert.Equalf(t, 101 | expected.TLSConfig.InsecureSkipVerify, 102 | actual.TLSConfig.InsecureSkipVerify, 103 | "%s - TLSConfig InsecureSkipVerify", testName) 104 | assert.Equalf(t, 105 | expected.TLSConfig.ServerName, 106 | actual.TLSConfig.ServerName, 107 | "%s - TLSConfig ServerName", testName) 108 | } 109 | } 110 | 111 | if assert.Equalf(t, len(expected.Fallbacks), len(actual.Fallbacks), "%s - Fallbacks", testName) { 112 | for i := range expected.Fallbacks { 113 | assert.Equalf(t, 114 | expected.Fallbacks[i].Host, 115 | actual.Fallbacks[i].Host, 116 | "%s - Fallback %d - Host", testName, i) 117 | assert.Equalf(t, 118 | expected.Fallbacks[i].Port, 119 | actual.Fallbacks[i].Port, 120 | "%s - Fallback %d - Port", testName, i) 121 | 122 | if assert.Equalf(t, 123 | expected.Fallbacks[i].TLSConfig == nil, 124 | actual.Fallbacks[i].TLSConfig == nil, 125 | "%s - Fallback %d - TLSConfig", testName, i) { 126 | if expected.Fallbacks[i].TLSConfig != nil { 127 | assert.Equalf(t, 128 | expected.Fallbacks[i].TLSConfig.InsecureSkipVerify, 129 | actual.Fallbacks[i].TLSConfig.InsecureSkipVerify, 130 | "%s - Fallback %d - TLSConfig InsecureSkipVerify", testName) 131 | assert.Equalf(t, 132 | expected.Fallbacks[i].TLSConfig.ServerName, 133 | actual.Fallbacks[i].TLSConfig.ServerName, 134 | "%s - Fallback %d - TLSConfig ServerName", testName) 135 | } 136 | } 137 | } 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /internal/helper/validator.go: -------------------------------------------------------------------------------- 1 | package helper 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "strconv" 7 | ) 8 | 9 | func IsEnum8(chType []byte) bool { 10 | return len(chType) > Enum8StrLen && (string(chType[:Enum8StrLen]) == Enum8Str) 11 | } 12 | 13 | func ExtractEnum(data []byte) (intToStringMap map[int16]string, stringToIntMap map[string]int16, err error) { 14 | enums := bytes.Split(data, []byte(", ")) 15 | intToStringMap = make(map[int16]string) 16 | stringToIntMap = make(map[string]int16) 17 | for _, enum := range enums { 18 | parts := bytes.SplitN(enum, []byte(" = "), 2) 19 | if len(parts) != 2 { 20 | return nil, nil, fmt.Errorf("invalid enum: %s", enum) 21 | } 22 | 23 | id, err := strconv.ParseInt(string(parts[1]), 10, 8) 24 | if err != nil { 25 | return nil, nil, fmt.Errorf("invalid enum id: %s", parts[1]) 26 | } 27 | 28 | val := string(parts[0][1 : len(parts[0])-1]) 29 | intToStringMap[int16(id)] = val 30 | stringToIntMap[val] = int16(id) 31 | } 32 | return intToStringMap, stringToIntMap, nil 33 | } 34 | 35 | func IsEnum16(chType []byte) bool { 36 | return len(chType) > Enum16StrLen && (string(chType[:Enum16StrLen]) == Enum16Str) 37 | } 38 | 39 | func IsDateTimeWithParam(chType []byte) bool { 40 | return len(chType) > DateTimeStrLen && (string(chType[:DateTimeStrLen]) == DateTimeStr) 41 | } 42 | 43 | func IsDateTime64(chType []byte) bool { 44 | return len(chType) > DateTime64StrLen && (string(chType[:DateTime64StrLen]) == DateTime64Str) 45 | } 46 | 47 | func IsFixedString(chType []byte) bool { 48 | return len(chType) > FixedStringStrLen && (string(chType[:FixedStringStrLen]) == FixedStringStr) 49 | } 50 | 51 | func IsDecimal(chType []byte) bool { 52 | return len(chType) > DecimalStrLen && (string(chType[:DecimalStrLen]) == DecimalStr) 53 | } 54 | 55 | func IsRing(chType []byte) bool { 56 | return string(chType) == RingStr 57 | } 58 | 59 | func IsMultiPolygon(chType []byte) bool { 60 | return string(chType) == MultiPolygonStr 61 | } 62 | 63 | func IsNested(chType []byte) bool { 64 | return len(chType) > LenNestedStr && string(chType[:LenNestedStr]) == NestedStr 65 | } 66 | 67 | func NestedToArrayType(chType []byte) []byte { 68 | if IsNested(chType) { 69 | newChType := make([]byte, 0, len(chType)-LenNestedStr+LenArrayStr+LenTupleStr+1) 70 | newChType = append(newChType, "Array(Tuple("...) 71 | newChType = append(newChType, chType[LenNestedStr:]...) 72 | newChType = append(newChType, ')') 73 | return newChType 74 | } 75 | return chType 76 | } 77 | 78 | func IsArray(chType []byte) bool { 79 | return len(chType) > LenArrayStr && string(chType[:LenArrayStr]) == ArrayStr 80 | } 81 | 82 | func IsPolygon(chType []byte) bool { 83 | return string(chType) == PolygonStr 84 | } 85 | 86 | func IsString(chType []byte) bool { 87 | return string(chType) == StringStr 88 | } 89 | 90 | func IsLowCardinality(chType []byte) bool { 91 | return len(chType) > LenLowCardinalityStr && string(chType[:LenLowCardinalityStr]) == LowCardinalityStr 92 | } 93 | 94 | func IsNullableLowCardinality(chType []byte) bool { 95 | return len(chType) > LenLowCardinalityNullableStr && 96 | string(chType[:LenLowCardinalityNullableStr]) == LowCardinalityNullableStr 97 | } 98 | 99 | func IsMap(chType []byte) bool { 100 | return len(chType) > LenMapStr && string(chType[:LenMapStr]) == MapStr 101 | } 102 | 103 | func IsNullable(chType []byte) bool { 104 | return len(chType) > LenNullableStr && string(chType[:LenNullableStr]) == NullableStr 105 | } 106 | 107 | func IsPoint(chType []byte) bool { 108 | return string(chType) == PointStr 109 | } 110 | 111 | func IsTuple(chType []byte) bool { 112 | return len(chType) > LenTupleStr && string(chType[:LenTupleStr]) == TupleStr 113 | } 114 | 115 | type ColumnData struct { 116 | ChType, Name []byte 117 | } 118 | 119 | func TypesInParentheses(b []byte) ([]ColumnData, error) { 120 | var columns []ColumnData 121 | var openFunc int 122 | var hasBacktick bool 123 | cur := 0 124 | for i, char := range b { 125 | if char == '`' { 126 | if !hasBacktick { 127 | hasBacktick = true 128 | continue 129 | } 130 | if b[i-1] != '\\' { 131 | hasBacktick = false 132 | } 133 | continue 134 | } 135 | if hasBacktick { 136 | continue 137 | } 138 | if char == ',' { 139 | if openFunc == 0 { 140 | colData, err := SplitNameType(b[cur:i]) 141 | if err != nil { 142 | return nil, err 143 | } 144 | columns = append(columns, colData) 145 | // add 2 to skip the ', ' 146 | cur = i + 2 147 | } 148 | continue 149 | } 150 | if char == '(' { 151 | openFunc++ 152 | continue 153 | } 154 | if char == ')' { 155 | openFunc-- 156 | continue 157 | } 158 | } 159 | colData, err := SplitNameType(b[cur:]) 160 | if err != nil { 161 | return nil, err 162 | } 163 | return append(columns, colData), nil 164 | } 165 | 166 | func SplitNameType(b []byte) (ColumnData, error) { 167 | // for example: `date f` Array(String) 168 | if b[0] == '`' { 169 | b = b[1:] 170 | for i, char := range b { 171 | if char == '`' && b[i-1] != '\\' { 172 | return ColumnData{ 173 | Name: b[1 : i+1], 174 | ChType: b[i+2:], 175 | }, nil 176 | } 177 | } 178 | return ColumnData{}, fmt.Errorf("cannot find closing backtick in %s", b) 179 | } 180 | for i, char := range b { 181 | if char == '(' { 182 | break 183 | } 184 | if char == ' ' { 185 | return ColumnData{ 186 | Name: b[1 : i+1], 187 | ChType: b[i+1:], 188 | }, nil 189 | } 190 | } 191 | return ColumnData{ 192 | ChType: b, 193 | }, nil 194 | } 195 | 196 | func FilterSimpleAggregate(chType []byte) []byte { 197 | if len(chType) <= SimpleAggregateStrLen || (string(chType[:SimpleAggregateStrLen]) != SimpleAggregateStr) { 198 | return chType 199 | } 200 | chType = chType[SimpleAggregateStrLen:] 201 | for i, v := range chType { 202 | if v == ',' { 203 | return chType[i+2 : len(chType)-1] 204 | } 205 | } 206 | panic("Cannot found nested type of " + string(chType)) 207 | } 208 | -------------------------------------------------------------------------------- /settings.go: -------------------------------------------------------------------------------- 1 | package chconn 2 | 3 | import ( 4 | "strconv" 5 | "strings" 6 | 7 | "github.com/vahid-sohrabloo/chconn/v2/internal/readerwriter" 8 | ) 9 | 10 | // Setting is a setting for the clickhouse query. 11 | // 12 | // The list of setting is here: https://clickhouse.com/docs/en/operations/settings/settings/ 13 | // Some of settings doesn't have effect. for example `http_zlib_compression_level` 14 | // because chconn use TCP connection to send data not HTTP. 15 | type Setting struct { 16 | Name, Value string 17 | Important, Custom, Obsolete bool 18 | } 19 | 20 | const ( 21 | settingFlagImportant = 0x01 22 | settingFlagCustom = 0x02 23 | settingFlagObsolete = 0x04 24 | ) 25 | 26 | // Settings is a list of settings for the clickhouse query. 27 | type Settings []Setting 28 | 29 | func (st Setting) write(w *readerwriter.Writer) { 30 | w.String(st.Name) 31 | 32 | var flag uint8 33 | if st.Important { 34 | flag |= settingFlagImportant 35 | } 36 | if st.Custom { 37 | flag |= settingFlagCustom 38 | } 39 | if st.Obsolete { 40 | flag |= settingFlagObsolete 41 | } 42 | w.Uint8(flag) 43 | 44 | w.String(st.Value) 45 | } 46 | 47 | func (s Settings) write(w *readerwriter.Writer) { 48 | for _, st := range s { 49 | st.write(w) 50 | } 51 | } 52 | 53 | // Parameters is a list of params for the clickhouse query. 54 | type Parameters struct { 55 | params []Setting 56 | } 57 | 58 | type Parameter func() Setting 59 | 60 | func NewParameters(input ...Parameter) *Parameters { 61 | params := make([]Setting, len(input)) 62 | for i, p := range input { 63 | params[i] = p() 64 | } 65 | return &Parameters{ 66 | params: params, 67 | } 68 | } 69 | 70 | // IntParameter get int query parameter. 71 | func IntParameter[T ~int | ~int8 | ~int16 | ~int32 | ~int64](name string, v T) Parameter { 72 | return func() Setting { 73 | return Setting{ 74 | Name: name, 75 | Value: "'" + strconv.FormatInt(int64(v), 10) + "'", 76 | Custom: true, 77 | } 78 | } 79 | } 80 | 81 | // IntSliceParameter get int query parameter. 82 | func IntSliceParameter[T ~int | ~int8 | ~int16 | ~int32 | ~int64](name string, v []T) Parameter { 83 | return func() Setting { 84 | var b strings.Builder 85 | b.WriteString("[") 86 | for i, v := range v { 87 | if i > 0 { 88 | b.WriteString(",") 89 | } 90 | b.WriteString(strconv.FormatInt(int64(v), 10)) 91 | } 92 | b.WriteString("]") 93 | return Setting{ 94 | Name: name, 95 | Value: "'" + b.String() + "'", 96 | Custom: true, 97 | } 98 | } 99 | } 100 | 101 | // UintParameter get uint query parameter. 102 | func UintParameter[T ~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64](name string, v T) Parameter { 103 | return func() Setting { 104 | return Setting{ 105 | Name: name, 106 | Value: "'" + strconv.FormatUint(uint64(v), 10) + "'", 107 | Custom: true, 108 | } 109 | } 110 | } 111 | 112 | // UintSliceParameter get uint slice query parameter. 113 | func UintSliceParameter[T ~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64](name string, v []T) Parameter { 114 | return func() Setting { 115 | var b strings.Builder 116 | b.WriteString("[") 117 | for i, v := range v { 118 | if i > 0 { 119 | b.WriteString(",") 120 | } 121 | b.WriteString(strconv.FormatUint(uint64(v), 10)) 122 | } 123 | b.WriteString("]") 124 | 125 | return Setting{ 126 | Name: name, 127 | Value: "'" + b.String() + "'", 128 | Custom: true, 129 | } 130 | } 131 | } 132 | 133 | // Float32Parameter get float32 query parameter. 134 | func Float32Parameter[T ~float32](name string, v T) Parameter { 135 | return func() Setting { 136 | return Setting{ 137 | Name: name, 138 | Value: "'" + strconv.FormatFloat(float64(v), 'f', -1, 32) + "'", 139 | Custom: true, 140 | } 141 | } 142 | } 143 | 144 | // Float32SliceParameter get float32 slice query parameter. 145 | func Float32SliceParameter[T ~float32](name string, v []T) Parameter { 146 | return func() Setting { 147 | var b strings.Builder 148 | b.WriteString("[") 149 | for i, v := range v { 150 | if i > 0 { 151 | b.WriteString(",") 152 | } 153 | b.WriteString(strconv.FormatFloat(float64(v), 'f', -1, 32)) 154 | } 155 | b.WriteString("]") 156 | 157 | return Setting{ 158 | Name: name, 159 | Value: "'" + b.String() + "'", 160 | Custom: true, 161 | } 162 | } 163 | } 164 | 165 | // Float64Parameter get float64 query parameter. 166 | func Float64Parameter[T ~float64](name string, v T) Parameter { 167 | return func() Setting { 168 | return Setting{ 169 | Name: name, 170 | Value: "'" + strconv.FormatFloat(float64(v), 'f', -1, 64) + "'", 171 | Custom: true, 172 | } 173 | } 174 | } 175 | 176 | // Float64SliceParameter get float64 slice query parameter. 177 | func Float64SliceParameter[T ~float64](name string, v []T) Parameter { 178 | return func() Setting { 179 | var b strings.Builder 180 | b.WriteString("[") 181 | for i, v := range v { 182 | if i > 0 { 183 | b.WriteString(",") 184 | } 185 | b.WriteString(strconv.FormatFloat(float64(v), 'f', -1, 64)) 186 | } 187 | b.WriteString("]") 188 | 189 | return Setting{ 190 | Name: name, 191 | Value: "'" + b.String() + "'", 192 | Custom: true, 193 | } 194 | } 195 | } 196 | 197 | func addSlashes(str string) string { 198 | var tmpRune []rune 199 | for _, ch := range str { 200 | switch ch { 201 | case '\\', '\'': 202 | tmpRune = append(tmpRune, '\\', ch) 203 | default: 204 | tmpRune = append(tmpRune, ch) 205 | } 206 | } 207 | return string(tmpRune) 208 | } 209 | 210 | // StringParameter get string query parameter. 211 | func StringParameter(name, v string) Parameter { 212 | return func() Setting { 213 | return Setting{ 214 | Name: name, 215 | Value: "'" + addSlashes(v) + "'", 216 | Custom: true, 217 | } 218 | } 219 | } 220 | 221 | // StringSliceParameter get string array query parameter. 222 | func StringSliceParameter(name string, v []string) Parameter { 223 | return func() Setting { 224 | var b strings.Builder 225 | b.WriteString("[") 226 | for i, v := range v { 227 | if i > 0 { 228 | b.WriteString(",") 229 | } 230 | b.WriteString("'" + addSlashes(v) + "'") 231 | } 232 | b.WriteString("]") 233 | return Setting{ 234 | Name: name, 235 | Value: "'" + addSlashes(b.String()) + "'", 236 | Custom: true, 237 | } 238 | } 239 | } 240 | 241 | func (p *Parameters) Params() []Setting { 242 | return p.params 243 | } 244 | 245 | func (p *Parameters) hasParam() bool { 246 | return p != nil && len(p.params) > 0 247 | } 248 | 249 | func (p *Parameters) write(w *readerwriter.Writer) { 250 | if p == nil { 251 | return 252 | } 253 | for _, st := range p.params { 254 | st.write(w) 255 | } 256 | } 257 | --------------------------------------------------------------------------------