├── .gitignore ├── docker ├── tini └── entrypoint.sh ├── .github ├── pull_request_template.md ├── ISSUE_TEMPLATE │ ├── question.md │ ├── bug_report.md │ └── feature_request.md └── workflows │ ├── dockerhub-publish.yml │ ├── check.yml │ └── publish-nightly-image.yml ├── server ├── coordinator │ ├── scheduler │ │ ├── static │ │ │ ├── error.go │ │ │ └── scheduler_test.go │ │ ├── manager │ │ │ ├── error.go │ │ │ └── scheduler_manager_test.go │ │ ├── nodepicker │ │ │ └── error.go │ │ ├── scheduler.go │ │ ├── reopen │ │ │ ├── scheduler_test.go │ │ │ └── scheduler.go │ │ └── rebalanced │ │ │ └── scheduler_test.go │ ├── error.go │ ├── procedure │ │ ├── manager.go │ │ ├── storage.go │ │ ├── util.go │ │ ├── operation │ │ │ ├── transferleader │ │ │ │ ├── trasnfer_leader_test.go │ │ │ │ └── batch_transfer_leader_test.go │ │ │ └── split │ │ │ │ └── split_test.go │ │ ├── ddl │ │ │ ├── createtable │ │ │ │ └── create_table_test.go │ │ │ └── createpartitiontable │ │ │ │ └── create_partition_table_test.go │ │ ├── error.go │ │ ├── procedure.go │ │ ├── delay_queue_test.go │ │ ├── storage_test.go │ │ └── delay_queue.go │ ├── lock │ │ ├── entry_lock_test.go │ │ └── entry_lock.go │ ├── eventdispatch │ │ └── dispatch.go │ ├── persist_shard_picker.go │ ├── watch │ │ └── watch_test.go │ ├── shard_picker.go │ ├── persist_shard_picker_test.go │ └── shard_picker_test.go ├── id │ ├── id.go │ ├── error.go │ ├── id_test.go │ ├── reusable_id_test.go │ └── reusable_id_impl.go ├── etcdutil │ ├── get_leader.go │ ├── error.go │ ├── config.go │ └── util_test.go ├── config │ ├── error.go │ └── util.go ├── error.go ├── service │ ├── grpc │ │ ├── error.go │ │ └── forward.go │ ├── util.go │ └── http │ │ ├── service.go │ │ ├── error.go │ │ ├── forward.go │ │ └── route.go ├── status │ └── status.go ├── member │ ├── error.go │ └── watch_leader_test.go ├── storage │ ├── error.go │ └── meta.go ├── cluster │ ├── metadata │ │ ├── error.go │ │ ├── compare_benchmark_test.go │ │ └── table_manager_test.go │ └── cluster.go └── limiter │ ├── limiter.go │ └── limiter_test.go ├── licenserc.toml ├── scripts └── run-integration-test.sh ├── config ├── example-standalone.toml ├── example-cluster0.toml ├── example-cluster1.toml └── example-cluster2.toml ├── pkg ├── assert │ └── assert.go ├── coderr │ ├── error_test.go │ ├── code.go │ └── error.go └── log │ ├── config.go │ ├── log.go │ └── global.go ├── Makefile ├── Dockerfile ├── .asf.yaml ├── .golangci.yml ├── README.md ├── docs └── style_guide.md ├── CONTRIBUTING.md ├── go.mod └── cmd └── horaemeta-server └── main.go /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | .tools 3 | .vscode 4 | bin 5 | coverage.txt 6 | -------------------------------------------------------------------------------- /docker/tini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/incubator-horaedb-meta/HEAD/docker/tini -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | ## Rationale 2 | 3 | 4 | ## Detailed Changes 5 | 6 | 7 | ## Test Plan 8 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/question.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Question 3 | about: Ask question about HoraeDB 4 | labels: question 5 | --- 6 | 7 | **Which part is this question about** 8 | 9 | 12 | 13 | **Describe your question** 14 | 15 | 18 | 19 | **Additional context** 20 | 21 | 24 | 25 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | labels: bug 5 | --- 6 | 7 | **Describe this problem** 8 | 9 | 12 | 13 | **Steps to reproduce** 14 | 15 | 18 | 19 | **Expected behavior** 20 | 21 | 24 | 25 | **Additional Information** 26 | 27 | 30 | 31 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for HoraeDB 4 | labels: enhancement 5 | --- 6 | 7 | **Description** 8 | 9 | 13 | 14 | **Proposal** 15 | 16 | 19 | 20 | **Additional context** 21 | 22 | 25 | 26 | -------------------------------------------------------------------------------- /server/coordinator/scheduler/static/error.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package static 21 | 22 | import "github.com/apache/incubator-horaedb-meta/pkg/coderr" 23 | 24 | var ErrNotImplemented = coderr.NewCodeError(coderr.ErrNotImplemented, "no") 25 | -------------------------------------------------------------------------------- /server/coordinator/scheduler/manager/error.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package manager 21 | 22 | import "github.com/apache/incubator-horaedb-meta/pkg/coderr" 23 | 24 | var ErrInvalidTopologyType = coderr.NewCodeError(coderr.InvalidParams, "invalid topology type") 25 | -------------------------------------------------------------------------------- /server/coordinator/scheduler/nodepicker/error.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package nodepicker 21 | 22 | import "github.com/apache/incubator-horaedb-meta/pkg/coderr" 23 | 24 | var ErrNoAliveNodes = coderr.NewCodeError(coderr.InvalidParams, "no alive nodes is found") 25 | -------------------------------------------------------------------------------- /licenserc.toml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | headerPath = "Apache-2.0-ASF.txt" 19 | 20 | excludes = [ 21 | # Derived 22 | "server/service/http/route.go", 23 | "server/coordinator/scheduler/nodepicker/hash/consistent_uniform_test.go", 24 | "server/coordinator/scheduler/nodepicker/hash/consistent_uniform.go", 25 | ] 26 | -------------------------------------------------------------------------------- /server/coordinator/error.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package coordinator 21 | 22 | import "github.com/apache/incubator-horaedb-meta/pkg/coderr" 23 | 24 | var ( 25 | ErrNodeNumberNotEnough = coderr.NewCodeError(coderr.Internal, "node number not enough") 26 | ErrPickNode = coderr.NewCodeError(coderr.Internal, "no node is picked") 27 | ) 28 | -------------------------------------------------------------------------------- /docker/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | 20 | 21 | 22 | set -exo pipefail 23 | 24 | ## init varibles 25 | USER="horae" 26 | DATA_DIR="/tmp/horaemeta0" 27 | CONFIG_FILE="/etc/horaemeta/horaemeta.toml" 28 | 29 | ## data dir 30 | mkdir -p ${DATA_DIR} 31 | chmod +777 -R ${DATA_DIR} 32 | chown -R ${USER}.${USER} ${DATA_DIR} 33 | 34 | exec /usr/bin/horaemeta-server --config ${CONFIG_FILE} 35 | -------------------------------------------------------------------------------- /server/id/id.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package id 21 | 22 | import "context" 23 | 24 | // Allocator defines the id allocator on the horaedb cluster meta info. 25 | type Allocator interface { 26 | // Alloc allocs a unique id. 27 | Alloc(ctx context.Context) (uint64, error) 28 | 29 | // Collect collect unused id to reused in alloc 30 | Collect(ctx context.Context, id uint64) error 31 | } 32 | -------------------------------------------------------------------------------- /server/etcdutil/get_leader.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package etcdutil 21 | 22 | import ( 23 | "go.etcd.io/etcd/server/v3/etcdserver" 24 | ) 25 | 26 | type EtcdLeaderGetter interface { 27 | EtcdLeaderID() (uint64, error) 28 | } 29 | 30 | type LeaderGetterWrapper struct { 31 | Server *etcdserver.EtcdServer 32 | } 33 | 34 | func (w *LeaderGetterWrapper) EtcdLeaderID() (uint64, error) { 35 | return w.Server.Lead(), nil 36 | } 37 | -------------------------------------------------------------------------------- /scripts/run-integration-test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | 20 | 21 | 22 | set -exo 23 | 24 | META_BIN_PATH="$(pwd)/bin/horaemeta-server" 25 | INTEGRATION_TEST_PATH=$(mktemp -d) 26 | 27 | # Download HoraeDB Code 28 | cd $INTEGRATION_TEST_PATH 29 | git clone --depth 1 https://github.com/apache/incubator-horaedb.git horaedb --branch main 30 | 31 | # Run integration_test 32 | cd horaedb/integration_tests 33 | 34 | META_BIN_PATH=$META_BIN_PATH make run-cluster 35 | -------------------------------------------------------------------------------- /config/example-standalone.toml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | etcd-start-timeout-ms = 30000 19 | peer-urls = "http://127.0.0.1:2380" 20 | advertise-client-urls = "http://127.0.0.1:2379" 21 | advertise-peer-urls = "http://127.0.0.1:2380" 22 | client-urls = "http://127.0.0.1:2379" 23 | data-dir = "/tmp/meta0" 24 | node-name = "meta0" 25 | initial-cluster = "meta0=http://127.0.0.1:2380" 26 | default-cluster-node-count = 1 27 | 28 | [log] 29 | level = "info" 30 | 31 | [etcd-log] 32 | level = "info" 33 | -------------------------------------------------------------------------------- /server/etcdutil/error.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package etcdutil 21 | 22 | import "github.com/apache/incubator-horaedb-meta/pkg/coderr" 23 | 24 | var ( 25 | ErrEtcdKVGet = coderr.NewCodeError(coderr.Internal, "etcd KV get failed") 26 | ErrEtcdKVGetResponse = coderr.NewCodeError(coderr.Internal, "etcd invalid get value response must only one") 27 | ErrEtcdKVGetNotFound = coderr.NewCodeError(coderr.Internal, "etcd KV get value not found") 28 | ) 29 | -------------------------------------------------------------------------------- /pkg/assert/assert.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package assert 21 | 22 | import "fmt" 23 | 24 | // Assertf panics and prints the appended message if the cond is false. 25 | func Assertf(cond bool, format string, a ...any) { 26 | if !cond { 27 | msg := fmt.Sprintf(format, a...) 28 | panic(msg) 29 | } 30 | } 31 | 32 | // Assertf panics and prints the appended message if the cond is false. 33 | func Assert(cond bool) { 34 | Assertf(cond, "unexpected case") 35 | } 36 | -------------------------------------------------------------------------------- /server/id/error.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package id 21 | 22 | import "github.com/apache/incubator-horaedb-meta/pkg/coderr" 23 | 24 | var ( 25 | ErrTxnPutEndID = coderr.NewCodeError(coderr.Internal, "put end id in txn") 26 | ErrAllocID = coderr.NewCodeError(coderr.Internal, "alloc id") 27 | ErrCollectID = coderr.NewCodeError(coderr.Internal, "collect invalid id") 28 | ErrCollectNotSupported = coderr.NewCodeError(coderr.Internal, "collect is not supported") 29 | ) 30 | -------------------------------------------------------------------------------- /server/config/error.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package config 21 | 22 | import ( 23 | "github.com/apache/incubator-horaedb-meta/pkg/coderr" 24 | ) 25 | 26 | var ( 27 | ErrHelpRequested = coderr.NewCodeError(coderr.PrintHelpUsage, "help requested") 28 | ErrInvalidPeerURL = coderr.NewCodeError(coderr.InvalidParams, "invalid peers url") 29 | ErrInvalidCommandArgs = coderr.NewCodeError(coderr.InvalidParams, "invalid command arguments") 30 | ErrRetrieveHostname = coderr.NewCodeError(coderr.Internal, "retrieve local hostname") 31 | ) 32 | -------------------------------------------------------------------------------- /config/example-cluster0.toml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | etcd-start-timeout-ms = 30000 19 | peer-urls = "http://127.0.0.1:2380" 20 | advertise-client-urls = "http://127.0.0.1:2379" 21 | advertise-peer-urls = "http://127.0.0.1:2380" 22 | client-urls = "http://127.0.0.1:2379" 23 | data-dir = "/tmp/meta0" 24 | node-name = "meta0" 25 | initial-cluster = "meta0=http://127.0.0.1:2380,meta1=http://127.0.0.1:12380,meta2=http://127.0.0.1:22380" 26 | default-cluster-node-count = 2 27 | http-port = 8080 28 | grpc-port = 2379 29 | 30 | [log] 31 | level = "info" 32 | 33 | [etcd-log] 34 | level = "info" 35 | -------------------------------------------------------------------------------- /config/example-cluster1.toml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | etcd-start-timeout-ms = 30000 19 | peer-urls = "http://127.0.0.1:12380" 20 | advertise-client-urls = "http://127.0.0.1:12379" 21 | advertise-peer-urls = "http://127.0.0.1:12380" 22 | client-urls = "http://127.0.0.1:12379" 23 | data-dir = "/tmp/meta1" 24 | node-name = "meta1" 25 | initial-cluster = "meta0=http://127.0.0.1:2380,meta1=http://127.0.0.1:12380,meta2=http://127.0.0.1:22380" 26 | default-cluster-node-count = 2 27 | http-port = 8081 28 | grpc-port = 12379 29 | 30 | [log] 31 | level = "info" 32 | 33 | [etcd-log] 34 | level = "info" 35 | -------------------------------------------------------------------------------- /config/example-cluster2.toml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | etcd-start-timeout-ms = 30000 19 | peer-urls = "http://127.0.0.1:22380" 20 | advertise-client-urls = "http://127.0.0.1:22379" 21 | advertise-peer-urls = "http://127.0.0.1:22380" 22 | client-urls = "http://127.0.0.1:22379" 23 | data-dir = "/tmp/meta2" 24 | node-name = "meta2" 25 | initial-cluster = "meta0=http://127.0.0.1:2380,meta1=http://127.0.0.1:12380,meta2=http://127.0.0.1:22380" 26 | default-cluster-node-count = 2 27 | http-port = 8082 28 | grpc-port = 22379 29 | 30 | [log] 31 | level = "info" 32 | 33 | [etcd-log] 34 | level = "info" 35 | -------------------------------------------------------------------------------- /pkg/coderr/error_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package coderr 21 | 22 | import ( 23 | "fmt" 24 | "strings" 25 | "testing" 26 | 27 | "github.com/stretchr/testify/require" 28 | ) 29 | 30 | func TestErrorStack(t *testing.T) { 31 | r := require.New(t) 32 | cerr := NewCodeError(Internal, "test internal error") 33 | err := cerr.WithCausef("failed reason:%s", "for test") 34 | errDesc := fmt.Sprintf("%s", err) 35 | expectErrDesc := "horaedb-meta/pkg/coderr/error_test.go:" 36 | 37 | r.True(strings.Contains(errDesc, expectErrDesc), "actual errDesc:%s", errDesc) 38 | } 39 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | GO_TOOLS_BIN_PATH := $(shell pwd)/.tools/bin 2 | PATH := $(GO_TOOLS_BIN_PATH):$(PATH) 3 | SHELL := env PATH='$(PATH)' GOBIN='$(GO_TOOLS_BIN_PATH)' $(shell which bash) 4 | 5 | COMMIT_ID := $(shell git rev-parse HEAD) 6 | BRANCH_NAME := $(shell git rev-parse --abbrev-ref HEAD) 7 | BUILD_DATE := $(shell date +'%Y/%m/%dT%H:%M:%S') 8 | 9 | default: build 10 | 11 | install-tools: 12 | @mkdir -p $(GO_TOOLS_BIN_PATH) 13 | @(which golangci-lint && golangci-lint version | grep '1.54') >/dev/null 2>&1 || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GO_TOOLS_BIN_PATH) v1.54.2 14 | 15 | META_PKG := github.com/apache/incubator-horaedb-meta 16 | PACKAGES := $(shell go list ./... | tail -n +2) 17 | PACKAGE_DIRECTORIES := $(subst $(META_PKG)/,,$(PACKAGES)) 18 | 19 | check: 20 | @ echo "gofmt ..." 21 | @ gofmt -s -l -d $(PACKAGE_DIRECTORIES) 2>&1 | awk '{ print } END { if (NR > 0) { exit 1 } }' 22 | @ echo "golangci-lint ..." 23 | @ golangci-lint run $(PACKAGE_DIRECTORIES) --config .golangci.yml 24 | 25 | test: 26 | @ echo "go test ..." 27 | @ go test -timeout 5m -coverprofile=coverage.txt -covermode=atomic $(PACKAGES) 28 | 29 | build: 30 | @ go build -ldflags="-X main.commitID=$(COMMIT_ID) -X main.branchName=$(BRANCH_NAME) -X main.buildDate=$(BUILD_DATE)" -o bin/horaemeta-server ./cmd/horaemeta-server 31 | 32 | integration-test: build 33 | @ bash ./scripts/run-integration-test.sh 34 | -------------------------------------------------------------------------------- /server/config/util.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package config 21 | 22 | import ( 23 | "net/url" 24 | "strings" 25 | ) 26 | 27 | // parseUrls parse a string into multiple urls. 28 | func parseUrls(s string) ([]url.URL, error) { 29 | items := strings.Split(s, ",") 30 | urls := make([]url.URL, 0, len(items)) 31 | for _, item := range items { 32 | u, err := url.Parse(item) 33 | if err != nil { 34 | return nil, ErrInvalidPeerURL.WithCausef("original url:%s, parsed item:%v, parse err:%v", s, item, err) 35 | } 36 | 37 | urls = append(urls, *u) 38 | } 39 | 40 | return urls, nil 41 | } 42 | -------------------------------------------------------------------------------- /server/error.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package server 21 | 22 | import "github.com/apache/incubator-horaedb-meta/pkg/coderr" 23 | 24 | var ( 25 | ErrCreateEtcdClient = coderr.NewCodeError(coderr.Internal, "create etcd etcdCli") 26 | ErrStartEtcd = coderr.NewCodeError(coderr.Internal, "start embed etcd") 27 | ErrStartEtcdTimeout = coderr.NewCodeError(coderr.Internal, "start etcd server timeout") 28 | ErrStartServer = coderr.NewCodeError(coderr.Internal, "start server") 29 | ErrFlowLimiterNotFound = coderr.NewCodeError(coderr.Internal, "flow limiter not found") 30 | ) 31 | -------------------------------------------------------------------------------- /server/service/grpc/error.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package grpc 21 | 22 | import ( 23 | "github.com/apache/incubator-horaedb-meta/pkg/coderr" 24 | ) 25 | 26 | var ( 27 | ErrRecvHeartbeat = coderr.NewCodeError(coderr.Internal, "receive heartbeat") 28 | ErrBindHeartbeatStream = coderr.NewCodeError(coderr.Internal, "bind heartbeat sender") 29 | ErrUnbindHeartbeatStream = coderr.NewCodeError(coderr.Internal, "unbind heartbeat sender") 30 | ErrForward = coderr.NewCodeError(coderr.Internal, "grpc forward") 31 | ErrFlowLimit = coderr.NewCodeError(coderr.TooManyRequests, "flow limit") 32 | ) 33 | -------------------------------------------------------------------------------- /server/coordinator/procedure/manager.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package procedure 21 | 22 | import ( 23 | "context" 24 | ) 25 | 26 | type Manager interface { 27 | // Start must be called before manager is used. 28 | Start(ctx context.Context) error 29 | // Stop must be called before manager is dropped. 30 | Stop(ctx context.Context) error 31 | 32 | // Submit procedure to be executed asynchronously. 33 | // TODO: change result type, add channel to get whether the procedure executed successfully 34 | Submit(ctx context.Context, procedure Procedure) error 35 | // ListRunningProcedure return immutable procedures info. 36 | ListRunningProcedure(ctx context.Context) ([]*Info, error) 37 | } 38 | -------------------------------------------------------------------------------- /server/coordinator/procedure/storage.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package procedure 21 | 22 | import ( 23 | "context" 24 | ) 25 | 26 | type Write interface { 27 | CreateOrUpdate(ctx context.Context, meta Meta) error 28 | CreateOrUpdateWithTTL(ctx context.Context, meta Meta, ttlSec int64) error 29 | } 30 | 31 | type Meta struct { 32 | ID uint64 33 | Kind Kind 34 | State State 35 | RawData []byte 36 | } 37 | 38 | type Storage interface { 39 | Write 40 | List(ctx context.Context, procedureType Kind, batchSize int) ([]*Meta, error) 41 | Delete(ctx context.Context, procedureType Kind, id uint64) error 42 | MarkDeleted(ctx context.Context, procedureType Kind, id uint64) error 43 | } 44 | -------------------------------------------------------------------------------- /server/status/status.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package status 21 | 22 | import "sync/atomic" 23 | 24 | type Status int32 25 | 26 | const ( 27 | StatusWaiting Status = iota 28 | StatusRunning 29 | Terminated 30 | ) 31 | 32 | type ServerStatus struct { 33 | status Status 34 | } 35 | 36 | func NewServerStatus() *ServerStatus { 37 | return &ServerStatus{ 38 | status: StatusWaiting, 39 | } 40 | } 41 | 42 | func (s *ServerStatus) Set(status Status) { 43 | atomic.StoreInt32((*int32)(&s.status), int32(status)) 44 | } 45 | 46 | func (s *ServerStatus) Get() Status { 47 | return Status(atomic.LoadInt32((*int32)(&s.status))) 48 | } 49 | 50 | func (s *ServerStatus) IsHealthy() bool { 51 | return s.Get() == StatusRunning 52 | } 53 | -------------------------------------------------------------------------------- /.github/workflows/dockerhub-publish.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | name: Publish Docker image 19 | 20 | on: 21 | workflow_dispatch: 22 | push: 23 | tags: 24 | - 'v*' 25 | 26 | jobs: 27 | docker: 28 | if: github.repository_owner == 'FIXME' 29 | runs-on: ubuntu-latest 30 | steps: 31 | - name: Checkout 32 | uses: actions/checkout@v3 33 | - name: Set up Docker Buildx 34 | uses: docker/setup-buildx-action@v2 35 | - name: Login to DockerHub 36 | uses: docker/login-action@v2 37 | with: 38 | username: ${{ secrets.DOCKERHUB_USERNAME }} 39 | password: ${{ secrets.DOCKERHUB_TOKEN }} 40 | - name: Build and Push HoraeDB Server Docker Image 41 | uses: docker/build-push-action@v3 42 | with: 43 | context: . 44 | push: true 45 | tags: apache/horaemeta-server:latest,horaedb/horaemeta-server:${{github.ref_name}} 46 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | ## Builder 19 | ARG GOLANG_VERSION=1.21.3 20 | FROM golang:${GOLANG_VERSION}-bullseye as build 21 | 22 | # cache mounts below may already exist and owned by root 23 | USER root 24 | 25 | RUN apt update && apt install --yes gcc g++ libssl-dev pkg-config cmake && rm -rf /var/lib/apt/lists/* 26 | 27 | COPY . /horaemeta 28 | WORKDIR /horaemeta 29 | 30 | RUN make build 31 | 32 | ## HoraeMeta 33 | FROM ubuntu:20.04 34 | 35 | RUN useradd -m -s /bin/bash horae 36 | 37 | RUN apt update && \ 38 | apt install --yes curl gdb iotop cron vim less net-tools && \ 39 | apt clean 40 | 41 | COPY --from=build /horaemeta/bin/horaemeta-server /usr/bin/horaemeta-server 42 | RUN chmod +x /usr/bin/horaemeta-server 43 | 44 | COPY ./docker/entrypoint.sh /entrypoint.sh 45 | COPY ./config/example-standalone.toml /etc/horaemeta/horaemeta.toml 46 | 47 | COPY ./docker/tini /tini 48 | RUN chmod +x /tini 49 | 50 | ARG USER horae 51 | 52 | ENTRYPOINT ["/tini", "--", "/entrypoint.sh"] 53 | -------------------------------------------------------------------------------- /server/member/error.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package member 21 | 22 | import "github.com/apache/incubator-horaedb-meta/pkg/coderr" 23 | 24 | var ( 25 | ErrResetLeader = coderr.NewCodeError(coderr.Internal, "reset leader by deleting leader key") 26 | ErrGetLeader = coderr.NewCodeError(coderr.Internal, "get leader by querying leader key") 27 | ErrTxnPutLeader = coderr.NewCodeError(coderr.Internal, "put leader key in txn") 28 | ErrMultipleLeader = coderr.NewCodeError(coderr.Internal, "multiple leaders found") 29 | ErrInvalidLeaderValue = coderr.NewCodeError(coderr.Internal, "invalid leader value") 30 | ErrMarshalMember = coderr.NewCodeError(coderr.Internal, "marshal member information") 31 | ErrGrantLease = coderr.NewCodeError(coderr.Internal, "grant lease") 32 | ErrRevokeLease = coderr.NewCodeError(coderr.Internal, "revoke lease") 33 | ErrCloseLease = coderr.NewCodeError(coderr.Internal, "close lease") 34 | ) 35 | -------------------------------------------------------------------------------- /.asf.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # For more information, see https://cwiki.apache.org/confluence/display/INFRA/Git+-+.asf.yaml+features. 19 | 20 | github: 21 | description: >- 22 | Meta service of HoraeDB cluster. 23 | homepage: https://horaedb.apache.org 24 | labels: 25 | - golang 26 | - sql 27 | - database 28 | - distributed-database 29 | - cloud-native 30 | - tsdb 31 | - timeseries-database 32 | - timeseries-analysis 33 | - iot-database 34 | - horaedb 35 | enabled_merge_buttons: 36 | squash: true 37 | merge: false 38 | rebase: true 39 | protected_branches: 40 | main: 41 | required_pull_request_reviews: 42 | dismiss_stale_reviews: true 43 | required_approving_review_count: 1 44 | 45 | notifications: 46 | commits: commits@horaedb.apache.org 47 | issues: commits@horaedb.apache.org 48 | pullrequests: commits@horaedb.apache.org 49 | jobs: commits@horaedb.apache.org 50 | discussions: commits@horaedb.apache.org 51 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | run: 19 | timeout: '5m' 20 | 21 | output: 22 | sort-results: true 23 | 24 | linters: 25 | disable-all: true 26 | enable: 27 | - bodyclose 28 | - dogsled 29 | - errcheck 30 | - goconst 31 | - gocritic 32 | - goimports 33 | - goprintffuncname 34 | - gosec 35 | - gosimple 36 | - govet 37 | - ineffassign 38 | - misspell 39 | - nakedret 40 | - exportloopref 41 | - staticcheck 42 | - stylecheck 43 | - typecheck 44 | - unconvert 45 | - unused 46 | - whitespace 47 | - gocyclo 48 | - exhaustive 49 | - typecheck 50 | - asciicheck 51 | #- errorlint" # disbale it for now, because the error style seems inconsistent with the one required by the linter 52 | - revive 53 | - exhaustruct 54 | 55 | linters-settings: 56 | revive: 57 | ignore-generated-header: false 58 | severity: warning 59 | confidence: 3 60 | exhaustruct: 61 | include: 62 | - 'github.com.apache.incubator-horaedb-meta.*' 63 | -------------------------------------------------------------------------------- /server/service/util.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package service 21 | 22 | import ( 23 | "context" 24 | "net/url" 25 | "strings" 26 | 27 | "github.com/apache/incubator-horaedb-meta/pkg/coderr" 28 | "google.golang.org/grpc" 29 | "google.golang.org/grpc/credentials/insecure" 30 | ) 31 | 32 | var ( 33 | ErrParseURL = coderr.NewCodeError(coderr.Internal, "parse url") 34 | ErrGRPCDial = coderr.NewCodeError(coderr.Internal, "grpc dial") 35 | ) 36 | 37 | // GetClientConn returns a gRPC client connection. 38 | func GetClientConn(ctx context.Context, addr string) (*grpc.ClientConn, error) { 39 | opt := grpc.WithTransportCredentials(insecure.NewCredentials()) 40 | 41 | host := addr 42 | if strings.HasPrefix(addr, "http") { 43 | u, err := url.Parse(addr) 44 | if err != nil { 45 | return nil, ErrParseURL.WithCause(err) 46 | } 47 | host = u.Host 48 | } 49 | 50 | cc, err := grpc.DialContext(ctx, host, opt) 51 | if err != nil { 52 | return nil, ErrGRPCDial.WithCause(err) 53 | } 54 | return cc, nil 55 | } 56 | -------------------------------------------------------------------------------- /server/coordinator/lock/entry_lock_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package lock 21 | 22 | import ( 23 | "testing" 24 | 25 | "github.com/stretchr/testify/require" 26 | ) 27 | 28 | func TestEntryLock(t *testing.T) { 29 | re := require.New(t) 30 | 31 | lock := NewEntryLock(3) 32 | 33 | lock1 := []uint64{1} 34 | result := lock.TryLock(lock1) 35 | re.Equal(true, result) 36 | result = lock.TryLock(lock1) 37 | re.Equal(false, result) 38 | lock.UnLock(lock1) 39 | result = lock.TryLock(lock1) 40 | re.Equal(true, result) 41 | lock.UnLock(lock1) 42 | 43 | lock2 := []uint64{2, 3, 4} 44 | lock3 := []uint64{3, 4, 5} 45 | result = lock.TryLock(lock2) 46 | re.Equal(true, result) 47 | result = lock.TryLock(lock2) 48 | re.Equal(false, result) 49 | result = lock.TryLock(lock3) 50 | re.Equal(false, result) 51 | lock.UnLock(lock2) 52 | result = lock.TryLock(lock2) 53 | re.Equal(true, result) 54 | lock.UnLock(lock2) 55 | 56 | re.Panics(func() { 57 | lock.UnLock(lock2) 58 | }, "this function did not panic") 59 | } 60 | -------------------------------------------------------------------------------- /server/coordinator/procedure/util.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package procedure 21 | 22 | import ( 23 | "github.com/apache/incubator-horaedb-meta/pkg/log" 24 | "github.com/looplab/fsm" 25 | "github.com/pkg/errors" 26 | "go.uber.org/zap" 27 | ) 28 | 29 | // CancelEventWithLog Cancel event when error is not nil. If error is nil, do nothing. 30 | func CancelEventWithLog(event *fsm.Event, err error, msg string, fields ...zap.Field) { 31 | if err == nil { 32 | return 33 | } 34 | fields = append(fields, zap.Error(err)) 35 | log.Error(msg, fields...) 36 | event.Cancel(errors.WithMessage(err, msg)) 37 | } 38 | 39 | // nolint 40 | func GetRequestFromEvent[T any](event *fsm.Event) (T, error) { 41 | if len(event.Args) != 1 { 42 | return *new(T), ErrGetRequest.WithCausef("event args length must be 1, actual length:%v", len(event.Args)) 43 | } 44 | 45 | switch request := event.Args[0].(type) { 46 | case T: 47 | return request, nil 48 | default: 49 | return *new(T), ErrGetRequest.WithCausef("event arg type must be same as return type") 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /server/service/http/service.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package http 21 | 22 | import ( 23 | "fmt" 24 | "net/http" 25 | "time" 26 | ) 27 | 28 | const defaultReadHeaderTimeout time.Duration = time.Duration(5) * time.Second 29 | 30 | // Service is wrapper for http.Server 31 | type Service struct { 32 | port int 33 | readTimeout time.Duration 34 | writeTimeout time.Duration 35 | 36 | router *Router 37 | server http.Server 38 | } 39 | 40 | func NewHTTPService(port int, readTimeout time.Duration, writeTimeout time.Duration, router *Router) *Service { 41 | return &Service{ 42 | port: port, 43 | readTimeout: readTimeout, 44 | writeTimeout: writeTimeout, 45 | router: router, 46 | server: http.Server{ 47 | ReadHeaderTimeout: defaultReadHeaderTimeout, 48 | }, 49 | } 50 | } 51 | 52 | func (s *Service) Start() error { 53 | s.server.ReadTimeout = s.readTimeout 54 | s.server.WriteTimeout = s.writeTimeout 55 | s.server.Addr = fmt.Sprintf(":%d", s.port) 56 | s.server.Handler = s.router 57 | 58 | return s.server.ListenAndServe() 59 | } 60 | 61 | func (s *Service) Stop() error { 62 | return s.server.Close() 63 | } 64 | -------------------------------------------------------------------------------- /pkg/coderr/code.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package coderr 21 | 22 | import "net/http" 23 | 24 | type Code int 25 | 26 | const ( 27 | Invalid Code = -1 28 | Ok = 0 29 | InvalidParams = http.StatusBadRequest 30 | BadRequest = http.StatusBadRequest 31 | NotFound = http.StatusNotFound 32 | TooManyRequests = http.StatusTooManyRequests 33 | Internal = http.StatusInternalServerError 34 | ErrNotImplemented = http.StatusNotImplemented 35 | 36 | // HTTPCodeUpperBound is a bound under which any Code should have the same meaning with the http status code. 37 | HTTPCodeUpperBound = Code(1000) 38 | PrintHelpUsage = 1001 39 | ClusterAlreadyExists = 1002 40 | ) 41 | 42 | // ToHTTPCode converts the Code to http code. 43 | // The Code below the HTTPCodeUpperBound has the same meaning as the http status code. However, for the other codes, we 44 | // should define the conversion rules by ourselves. 45 | func (c Code) ToHTTPCode() int { 46 | if c < HTTPCodeUpperBound { 47 | return int(c) 48 | } 49 | 50 | // TODO: use switch to convert the code to http code. 51 | return int(c) 52 | } 53 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | > :warning: This repository has been deprecated at 2024-01-25, further development will move to [here](https://github.com/apache/incubator-horaedb/tree/main/horaemeta). 2 | 3 | --- 4 | 5 | # HoraeMeta 6 | 7 | [![codecov](https://codecov.io/gh/apache/incubator-horaedb-meta/branch/main/graph/badge.svg?token=VTYXEAB2WU)](https://codecov.io/gh/apache/incubator-horaedb-meta) 8 | ![License](https://img.shields.io/badge/license-Apache--2.0-green.svg) 9 | 10 | HoraeMeta is the meta service for managing the HoraeDB cluster. 11 | 12 | ## Status 13 | The project is in a very early stage. 14 | 15 | ## Quick Start 16 | ### Build HoraeMeta binary 17 | ```bash 18 | make build 19 | ``` 20 | 21 | ### Standalone Mode 22 | Although HoraeMeta is designed to deployed as a cluster with three or more instances, it can also be started standalone: 23 | ```bash 24 | # HoraeMeta0 25 | mkdir /tmp/meta0 26 | ./bin/horaemeta-server --config ./config/example-standalone.toml 27 | ``` 28 | 29 | ### Cluster mode 30 | Here is an example for starting HoraeMeta in cluster mode (three instances) on single machine by using different ports: 31 | ```bash 32 | # Create directories. 33 | mkdir /tmp/meta0 34 | mkdir /tmp/meta1 35 | mkdir /tmp/meta2 36 | 37 | # horaemeta0 38 | ./bin/horaemeta-server --config ./config/exampl-cluster0.toml 39 | 40 | # horaemeta1 41 | ./bin/horaemeta-server --config ./config/exampl-cluster1.toml 42 | 43 | # horaemeta2 44 | ./bin/horaemeta-server --config ./config/exampl-cluster2.toml 45 | ``` 46 | 47 | ## Acknowledgment 48 | HoraeMeta refers to the excellent project [pd](https://github.com/tikv/pd) in design and some module and codes are forked from [pd](https://github.com/tikv/pd), thanks to the TiKV team. 49 | 50 | ## Contributing 51 | The project is under rapid development so that any contribution is welcome. 52 | Check our [Contributing Guide](https://github.com/apache/incubator-horaedb-meta/blob/main/CONTRIBUTING.md) and make your first contribution! 53 | 54 | ## License 55 | HoraeMeta is under [Apache License 2.0](./LICENSE). 56 | -------------------------------------------------------------------------------- /server/coordinator/lock/entry_lock.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package lock 21 | 22 | import ( 23 | "fmt" 24 | "sync" 25 | ) 26 | 27 | type EntryLock struct { 28 | lock sync.Mutex 29 | entryLocks map[uint64]struct{} 30 | } 31 | 32 | func NewEntryLock(initCapacity int) EntryLock { 33 | return EntryLock{ 34 | lock: sync.Mutex{}, 35 | entryLocks: make(map[uint64]struct{}, initCapacity), 36 | } 37 | } 38 | 39 | func (l *EntryLock) TryLock(locks []uint64) bool { 40 | l.lock.Lock() 41 | defer l.lock.Unlock() 42 | 43 | for _, lock := range locks { 44 | _, exists := l.entryLocks[lock] 45 | if exists { 46 | return false 47 | } 48 | } 49 | 50 | for _, lock := range locks { 51 | l.entryLocks[lock] = struct{}{} 52 | } 53 | 54 | return true 55 | } 56 | 57 | func (l *EntryLock) UnLock(locks []uint64) { 58 | l.lock.Lock() 59 | defer l.lock.Unlock() 60 | 61 | for _, lock := range locks { 62 | _, exists := l.entryLocks[lock] 63 | if !exists { 64 | panic(fmt.Sprintf("try to unlock nonexistent lock, exists locks:%v, unlock locks:%v", l.entryLocks, locks)) 65 | } 66 | } 67 | 68 | for _, lock := range locks { 69 | delete(l.entryLocks, lock) 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /server/storage/error.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package storage 21 | 22 | import "github.com/apache/incubator-horaedb-meta/pkg/coderr" 23 | 24 | var ( 25 | ErrEncode = coderr.NewCodeError(coderr.Internal, "storage encode") 26 | ErrDecode = coderr.NewCodeError(coderr.Internal, "storage decode") 27 | 28 | ErrCreateSchemaAgain = coderr.NewCodeError(coderr.Internal, "storage create schemas") 29 | ErrCreateClusterAgain = coderr.NewCodeError(coderr.Internal, "storage create cluster") 30 | ErrUpdateCluster = coderr.NewCodeError(coderr.Internal, "storage update cluster") 31 | ErrCreateClusterViewAgain = coderr.NewCodeError(coderr.Internal, "storage create cluster view") 32 | ErrUpdateClusterViewConflict = coderr.NewCodeError(coderr.Internal, "storage update cluster view") 33 | ErrCreateTableAgain = coderr.NewCodeError(coderr.Internal, "storage create tables") 34 | ErrDeleteTableAgain = coderr.NewCodeError(coderr.Internal, "storage delete table") 35 | ErrCreateShardViewAgain = coderr.NewCodeError(coderr.Internal, "storage create shard view") 36 | ErrUpdateShardViewConflict = coderr.NewCodeError(coderr.Internal, "storage update shard view") 37 | ) 38 | -------------------------------------------------------------------------------- /.github/workflows/check.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | name: Check 19 | on: 20 | pull_request: 21 | paths-ignore: 22 | - 'docs/**' 23 | - '**.md' 24 | workflow_dispatch: 25 | 26 | jobs: 27 | style-check: 28 | runs-on: ubuntu-latest 29 | timeout-minutes: 5 30 | steps: 31 | - uses: actions/checkout@v3 32 | - name: Check License Header 33 | uses: korandoru/hawkeye@v3 34 | - uses: actions/setup-go@v3 35 | with: 36 | go-version: 1.21.3 37 | - run: | 38 | make install-tools 39 | make check 40 | 41 | unit-test: 42 | runs-on: ubuntu-latest 43 | timeout-minutes: 10 44 | steps: 45 | - uses: actions/checkout@v3 46 | - uses: actions/setup-go@v3 47 | with: 48 | go-version: 1.21.3 49 | - run: | 50 | make install-tools 51 | make test 52 | 53 | integration-test: 54 | runs-on: ubuntu-latest 55 | timeout-minutes: 60 56 | steps: 57 | - uses: actions/checkout@v3 58 | - uses: actions/setup-go@v3 59 | with: 60 | go-version: 1.21.3 61 | - run: | 62 | make install-tools 63 | sudo apt install -y protobuf-compiler 64 | make integration-test 65 | -------------------------------------------------------------------------------- /pkg/log/config.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package log 21 | 22 | import ( 23 | "go.uber.org/zap" 24 | "go.uber.org/zap/zapcore" 25 | ) 26 | 27 | const ( 28 | DefaultLogLevel = "info" 29 | DefaultLogFile = "stdout" 30 | ) 31 | 32 | type Config struct { 33 | Level string `toml:"level" env:"LEVEL"` 34 | File string 35 | } 36 | 37 | // DefaultZapLoggerConfig defines default zap logger configuration. 38 | var DefaultZapLoggerConfig = zap.Config{ 39 | Level: zap.NewAtomicLevelAt(zapcore.InfoLevel), 40 | Development: false, 41 | Sampling: &zap.SamplingConfig{ 42 | Initial: 100, 43 | Thereafter: 100, 44 | }, 45 | Encoding: "console", 46 | EncoderConfig: zapcore.EncoderConfig{ 47 | TimeKey: "ts", 48 | LevelKey: "level", 49 | NameKey: "logger", 50 | CallerKey: "caller", 51 | MessageKey: "msg", 52 | StacktraceKey: "stacktrace", 53 | LineEnding: zapcore.DefaultLineEnding, 54 | EncodeLevel: zapcore.LowercaseLevelEncoder, 55 | EncodeTime: zapcore.ISO8601TimeEncoder, 56 | EncodeDuration: zapcore.StringDurationEncoder, 57 | EncodeCaller: zapcore.ShortCallerEncoder, 58 | }, 59 | OutputPaths: []string{"stdout"}, 60 | ErrorOutputPaths: []string{"stdout"}, 61 | } 62 | -------------------------------------------------------------------------------- /server/id/id_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package id 21 | 22 | import ( 23 | "context" 24 | "testing" 25 | "time" 26 | 27 | "github.com/apache/incubator-horaedb-meta/server/etcdutil" 28 | "github.com/stretchr/testify/require" 29 | clientv3 "go.etcd.io/etcd/client/v3" 30 | "go.uber.org/zap" 31 | ) 32 | 33 | const ( 34 | defaultRequestTimeout = time.Second * 30 35 | defaultStep = 100 36 | defaultRootPath = "/meta" 37 | defaultAllocIDKey = "/id" 38 | ) 39 | 40 | func TestMultipleAllocBasedOnKV(t *testing.T) { 41 | start := 0 42 | size := 201 43 | _, kv, closeSrv := etcdutil.PrepareEtcdServerAndClient(t) 44 | defer closeSrv() 45 | 46 | testAllocIDValue(t, kv, start, size) 47 | testAllocIDValue(t, kv, ((start+size)/defaultStep+1)*defaultStep, size) 48 | } 49 | 50 | func testAllocIDValue(t *testing.T, kv clientv3.KV, start, size int) { 51 | re := require.New(t) 52 | alloc := NewAllocatorImpl(zap.NewNop(), kv, defaultRootPath+defaultAllocIDKey, defaultStep) 53 | ctx, cancel := context.WithTimeout(context.Background(), defaultRequestTimeout) 54 | defer cancel() 55 | 56 | for i := start; i < start+size; i++ { 57 | value, err := alloc.Alloc(ctx) 58 | re.NoError(err) 59 | re.Equal(uint64(i), value) 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /pkg/log/log.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package log 21 | 22 | import ( 23 | "fmt" 24 | 25 | "go.uber.org/zap" 26 | "go.uber.org/zap/zapcore" 27 | ) 28 | 29 | func init() { 30 | defaultConfig := &Config{ 31 | Level: "info", 32 | File: "stdout", 33 | } 34 | _, err := InitGlobalLogger(defaultConfig) 35 | if err != nil { 36 | fmt.Println("fail to init global logger, err:", err) 37 | } 38 | } 39 | 40 | var ( 41 | globalLogger *zap.Logger 42 | globalLoggerCfg *zap.Config 43 | ) 44 | 45 | // InitGlobalLogger initializes the global logger with Config. 46 | func InitGlobalLogger(cfg *Config) (*zap.Logger, error) { 47 | zapCfg := DefaultZapLoggerConfig 48 | 49 | level, err := zapcore.ParseLevel(cfg.Level) 50 | if err != nil { 51 | return nil, err 52 | } 53 | zapCfg.Level.SetLevel(level) 54 | 55 | if len(cfg.File) > 0 { 56 | zapCfg.OutputPaths = []string{cfg.File} 57 | zapCfg.ErrorOutputPaths = []string{cfg.File} 58 | } 59 | 60 | logger, err := zapCfg.Build() 61 | if err != nil { 62 | return nil, err 63 | } 64 | 65 | globalLogger = logger 66 | globalLoggerCfg = &zapCfg 67 | return logger, nil 68 | } 69 | 70 | func GetLogger() *zap.Logger { 71 | return globalLogger 72 | } 73 | 74 | func GetLoggerConfig() *zap.Config { 75 | return globalLoggerCfg 76 | } 77 | -------------------------------------------------------------------------------- /pkg/log/global.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package log 21 | 22 | import ( 23 | "go.uber.org/zap" 24 | "go.uber.org/zap/zapcore" 25 | ) 26 | 27 | func Debug(msg string, fields ...zap.Field) { 28 | globalLogger.WithOptions(zap.AddCallerSkip(1)).Debug(msg, fields...) 29 | } 30 | 31 | func Info(msg string, fields ...zap.Field) { 32 | globalLogger.WithOptions(zap.AddCallerSkip(1)).Info(msg, fields...) 33 | } 34 | 35 | func Warn(msg string, fields ...zap.Field) { 36 | globalLogger.WithOptions(zap.AddCallerSkip(1)).Warn(msg, fields...) 37 | } 38 | 39 | func Error(msg string, fields ...zap.Field) { 40 | globalLogger.WithOptions(zap.AddCallerSkip(1)).Error(msg, fields...) 41 | } 42 | 43 | func Panic(msg string, fields ...zap.Field) { 44 | globalLogger.WithOptions(zap.AddCallerSkip(1)).Panic(msg, fields...) 45 | } 46 | 47 | func Fatal(msg string, fields ...zap.Field) { 48 | globalLogger.Fatal(msg, fields...) 49 | } 50 | 51 | func With(fields ...zap.Field) *zap.Logger { 52 | return globalLogger.With(fields...) 53 | } 54 | 55 | func SetLevel(lvl string) error { 56 | level, err := zapcore.ParseLevel(lvl) 57 | if err != nil { 58 | return err 59 | } 60 | globalLoggerCfg.Level.SetLevel(level) 61 | return nil 62 | } 63 | 64 | func GetLevel() zapcore.Level { 65 | return globalLoggerCfg.Level.Level() 66 | } 67 | -------------------------------------------------------------------------------- /.github/workflows/publish-nightly-image.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | name: Publish Nightly Docker image 19 | 20 | on: 21 | workflow_dispatch: 22 | schedule: 23 | - cron: '10 20 * * *' 24 | 25 | env: 26 | REGISTRY: ghcr.io 27 | IMAGE_NAME: apache/horaemeta-server 28 | 29 | jobs: 30 | docker: 31 | if: github.repository_owner == 'apache' 32 | runs-on: ubuntu-latest 33 | permissions: 34 | contents: read 35 | packages: write 36 | steps: 37 | - name: Checkout 38 | uses: actions/checkout@v3 39 | - name: Set up Docker Buildx 40 | uses: docker/setup-buildx-action@v2 41 | - name: Login to Container Registry 42 | uses: docker/login-action@v2 43 | with: 44 | registry: ${{ env.REGISTRY }} 45 | username: ${{ github.actor }} 46 | password: ${{ secrets.GITHUB_TOKEN }} 47 | - name: Set Environment Variables 48 | run: | 49 | echo "BUILD_DATE=$(TZ=':Asia/Shanghai' date '+%Y%m%d')" >> $GITHUB_ENV 50 | echo "SHORT_SHA=`echo ${GITHUB_SHA} | cut -c1-8`" >> $GITHUB_ENV 51 | - name: Build and Push Docker Image 52 | uses: docker/build-push-action@v3 53 | with: 54 | context: . 55 | push: true 56 | tags: | 57 | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:nightly-${{ env.BUILD_DATE }}-${{ env.SHORT_SHA }} 58 | -------------------------------------------------------------------------------- /server/coordinator/procedure/operation/transferleader/trasnfer_leader_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package transferleader_test 21 | 22 | import ( 23 | "context" 24 | "testing" 25 | 26 | "github.com/apache/incubator-horaedb-meta/server/coordinator/procedure/operation/transferleader" 27 | "github.com/apache/incubator-horaedb-meta/server/coordinator/procedure/test" 28 | "github.com/apache/incubator-horaedb-meta/server/storage" 29 | "github.com/stretchr/testify/require" 30 | ) 31 | 32 | func TestTransferLeader(t *testing.T) { 33 | re := require.New(t) 34 | ctx := context.Background() 35 | dispatch := test.MockDispatch{} 36 | c := test.InitEmptyCluster(ctx, t) 37 | s := test.NewTestStorage(t) 38 | 39 | snapshot := c.GetMetadata().GetClusterSnapshot() 40 | 41 | var targetShardID storage.ShardID 42 | for shardID := range snapshot.Topology.ShardViewsMapping { 43 | targetShardID = shardID 44 | break 45 | } 46 | newLeaderNodeName := snapshot.RegisteredNodes[0].Node.Name 47 | 48 | p, err := transferleader.NewProcedure(transferleader.ProcedureParams{ 49 | ID: 0, 50 | Dispatch: dispatch, 51 | Storage: s, 52 | ClusterSnapshot: snapshot, 53 | ShardID: targetShardID, 54 | OldLeaderNodeName: "", 55 | NewLeaderNodeName: newLeaderNodeName, 56 | }) 57 | re.NoError(err) 58 | 59 | err = p.Start(ctx) 60 | re.NoError(err) 61 | } 62 | -------------------------------------------------------------------------------- /server/cluster/metadata/error.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package metadata 21 | 22 | import "github.com/apache/incubator-horaedb-meta/pkg/coderr" 23 | 24 | var ( 25 | ErrCreateCluster = coderr.NewCodeError(coderr.BadRequest, "create cluster") 26 | ErrUpdateCluster = coderr.NewCodeError(coderr.Internal, "update cluster") 27 | ErrStartCluster = coderr.NewCodeError(coderr.Internal, "start cluster") 28 | ErrClusterAlreadyExists = coderr.NewCodeError(coderr.ClusterAlreadyExists, "cluster already exists") 29 | ErrClusterNotFound = coderr.NewCodeError(coderr.NotFound, "cluster not found") 30 | ErrClusterStateInvalid = coderr.NewCodeError(coderr.Internal, "cluster state invalid") 31 | ErrSchemaNotFound = coderr.NewCodeError(coderr.NotFound, "schema not found") 32 | ErrTableNotFound = coderr.NewCodeError(coderr.NotFound, "table not found") 33 | ErrShardNotFound = coderr.NewCodeError(coderr.NotFound, "shard not found") 34 | ErrVersionNotFound = coderr.NewCodeError(coderr.NotFound, "version not found") 35 | ErrNodeNotFound = coderr.NewCodeError(coderr.NotFound, "NodeName not found") 36 | ErrTableAlreadyExists = coderr.NewCodeError(coderr.Internal, "table already exists") 37 | ErrOpenTable = coderr.NewCodeError(coderr.Internal, "open table") 38 | ErrParseTopologyType = coderr.NewCodeError(coderr.Internal, "parse topology type") 39 | ) 40 | -------------------------------------------------------------------------------- /server/limiter/limiter.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package limiter 21 | 22 | import ( 23 | "sync" 24 | 25 | "github.com/apache/incubator-horaedb-meta/server/config" 26 | "golang.org/x/time/rate" 27 | ) 28 | 29 | type FlowLimiter struct { 30 | // enable is used to control the switch of the limiter. 31 | enable bool 32 | l *rate.Limiter 33 | // RWMutex is used to protect following fields. 34 | lock sync.RWMutex 35 | // limit is the updated rate of tokens. 36 | limit int 37 | // burst is the maximum number of tokens. 38 | burst int 39 | } 40 | 41 | func NewFlowLimiter(config config.LimiterConfig) *FlowLimiter { 42 | newLimiter := rate.NewLimiter(rate.Limit(config.Limit), config.Burst) 43 | 44 | return &FlowLimiter{ 45 | enable: config.Enable, 46 | l: newLimiter, 47 | lock: sync.RWMutex{}, 48 | limit: config.Limit, 49 | burst: config.Burst, 50 | } 51 | } 52 | 53 | func (f *FlowLimiter) Allow() bool { 54 | if !f.enable { 55 | return true 56 | } 57 | return f.l.Allow() 58 | } 59 | 60 | func (f *FlowLimiter) UpdateLimiter(config config.LimiterConfig) error { 61 | f.lock.Lock() 62 | defer f.lock.Unlock() 63 | 64 | f.enable = config.Enable 65 | f.l.SetLimit(rate.Limit(config.Limit)) 66 | f.l.SetBurst(config.Burst) 67 | f.limit = config.Limit 68 | f.burst = config.Burst 69 | return nil 70 | } 71 | 72 | func (f *FlowLimiter) GetConfig() *config.LimiterConfig { 73 | return &config.LimiterConfig{ 74 | Enable: f.enable, 75 | Limit: f.limit, 76 | Burst: f.burst, 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /server/coordinator/scheduler/scheduler.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package scheduler 21 | 22 | import ( 23 | "context" 24 | 25 | "github.com/apache/incubator-horaedb-meta/server/cluster/metadata" 26 | "github.com/apache/incubator-horaedb-meta/server/coordinator/procedure" 27 | "github.com/apache/incubator-horaedb-meta/server/storage" 28 | ) 29 | 30 | type ScheduleResult struct { 31 | Procedure procedure.Procedure 32 | // The reason that the procedure is generated for. 33 | Reason string 34 | } 35 | 36 | type ShardAffinity struct { 37 | ShardID storage.ShardID `json:"shardID"` 38 | NumAllowedOtherShards uint `json:"numAllowedOtherShards"` 39 | } 40 | 41 | type ShardAffinityRule struct { 42 | Affinities []ShardAffinity 43 | } 44 | 45 | type Scheduler interface { 46 | Name() string 47 | // Schedule will generate procedure based on current cluster snapshot, which will be submitted to ProcedureManager, and whether it is actually executed depends on the current state of ProcedureManager. 48 | Schedule(ctx context.Context, clusterSnapshot metadata.Snapshot) (ScheduleResult, error) 49 | // UpdateEnableSchedule is used to update enableSchedule for scheduler, 50 | // EnableSchedule means that the cluster topology is locked and the mapping between shards and nodes cannot be changed. 51 | UpdateEnableSchedule(ctx context.Context, enable bool) 52 | AddShardAffinityRule(ctx context.Context, rule ShardAffinityRule) error 53 | RemoveShardAffinityRule(ctx context.Context, shardID storage.ShardID) error 54 | ListShardAffinityRule(ctx context.Context) (ShardAffinityRule, error) 55 | } 56 | -------------------------------------------------------------------------------- /server/id/reusable_id_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package id 21 | 22 | import ( 23 | "context" 24 | "testing" 25 | 26 | "github.com/stretchr/testify/require" 27 | ) 28 | 29 | func TestAlloc(t *testing.T) { 30 | re := require.New(t) 31 | ctx := context.Background() 32 | 33 | allocor := NewReusableAllocatorImpl([]uint64{}, uint64(1)) 34 | // IDs: [] 35 | // Alloc: 1 36 | // IDs: [1] 37 | id, err := allocor.Alloc(ctx) 38 | re.NoError(err) 39 | re.Equal(uint64(1), id) 40 | 41 | // IDs: [1] 42 | // Alloc: 2 43 | // IDs: [1,2] 44 | id, err = allocor.Alloc(ctx) 45 | re.NoError(err) 46 | re.Equal(uint64(2), id) 47 | 48 | // IDs: [1,2] 49 | // Collect: 2 50 | // IDs: [1] 51 | err = allocor.Collect(ctx, uint64(2)) 52 | re.NoError(err) 53 | 54 | // IDs: [1] 55 | // Alloc: 2 56 | // IDs: [1,2] 57 | id, err = allocor.Alloc(ctx) 58 | re.NoError(err) 59 | re.Equal(uint64(2), id) 60 | 61 | // IDs: [1,2,3,5,6] 62 | allocor = NewReusableAllocatorImpl([]uint64{1, 2, 3, 5, 6}, uint64(1)) 63 | 64 | // IDs: [1,2,3,5,6] 65 | // Alloc: 4 66 | // IDs: [1,2,3,4,5,6] 67 | id, err = allocor.Alloc(ctx) 68 | re.NoError(err) 69 | re.Equal(uint64(4), id) 70 | 71 | // IDs: [1,2,3,4,5,6] 72 | // Alloc: 7 73 | // IDs: [1,2,3,4,5,6,7] 74 | id, err = allocor.Alloc(ctx) 75 | re.NoError(err) 76 | re.Equal(uint64(7), id) 77 | 78 | // IDs: [1,2,3,4,5,6,7] 79 | // Collect: 1 80 | // IDs: [2,3,4,5,6,7] 81 | err = allocor.Collect(ctx, uint64(1)) 82 | re.NoError(err) 83 | 84 | // IDs: [2,3,4,5,6,7] 85 | // Alloc: 1 86 | // IDs: [1,2,3,4,5,6,7] 87 | id, err = allocor.Alloc(ctx) 88 | re.NoError(err) 89 | re.Equal(uint64(1), id) 90 | } 91 | -------------------------------------------------------------------------------- /server/limiter/limiter_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package limiter 21 | 22 | import ( 23 | "testing" 24 | "time" 25 | 26 | "github.com/apache/incubator-horaedb-meta/server/config" 27 | "github.com/stretchr/testify/require" 28 | ) 29 | 30 | const ( 31 | defaultInitialLimiterRate = 10 * 1000 32 | defaultInitialLimiterCapacity = 1000 33 | defaultEnableLimiter = true 34 | defaultUpdateLimiterRate = 100 * 1000 35 | defaultUpdateLimiterCapacity = 100 * 1000 36 | ) 37 | 38 | func TestFlowLimiter(t *testing.T) { 39 | re := require.New(t) 40 | flowLimiter := NewFlowLimiter(config.LimiterConfig{ 41 | Limit: defaultInitialLimiterRate, 42 | Burst: defaultInitialLimiterCapacity, 43 | Enable: defaultEnableLimiter, 44 | }) 45 | 46 | for i := 0; i < defaultInitialLimiterCapacity; i++ { 47 | flag := flowLimiter.Allow() 48 | re.Equal(true, flag) 49 | } 50 | 51 | time.Sleep(time.Millisecond) 52 | for i := 0; i < defaultInitialLimiterRate/1000; i++ { 53 | flag := flowLimiter.Allow() 54 | re.Equal(true, flag) 55 | } 56 | 57 | err := flowLimiter.UpdateLimiter(config.LimiterConfig{ 58 | Limit: defaultUpdateLimiterRate, 59 | Burst: defaultUpdateLimiterCapacity, 60 | Enable: defaultEnableLimiter, 61 | }) 62 | re.NoError(err) 63 | 64 | limiter := flowLimiter.GetConfig() 65 | re.Equal(defaultUpdateLimiterRate, limiter.Limit) 66 | re.Equal(defaultUpdateLimiterCapacity, limiter.Burst) 67 | re.Equal(defaultEnableLimiter, limiter.Enable) 68 | 69 | time.Sleep(time.Millisecond) 70 | for i := 0; i < defaultUpdateLimiterRate/1000; i++ { 71 | flag := flowLimiter.Allow() 72 | re.Equal(true, flag) 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /server/coordinator/eventdispatch/dispatch.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package eventdispatch 21 | 22 | import ( 23 | "context" 24 | 25 | "github.com/apache/incubator-horaedb-meta/server/cluster/metadata" 26 | ) 27 | 28 | type Dispatch interface { 29 | OpenShard(context context.Context, address string, request OpenShardRequest) error 30 | CloseShard(context context.Context, address string, request CloseShardRequest) error 31 | CreateTableOnShard(context context.Context, address string, request CreateTableOnShardRequest) (uint64, error) 32 | DropTableOnShard(context context.Context, address string, request DropTableOnShardRequest) (uint64, error) 33 | OpenTableOnShard(ctx context.Context, address string, request OpenTableOnShardRequest) error 34 | CloseTableOnShard(context context.Context, address string, request CloseTableOnShardRequest) error 35 | } 36 | 37 | type OpenShardRequest struct { 38 | Shard metadata.ShardInfo 39 | } 40 | 41 | type CloseShardRequest struct { 42 | ShardID uint32 43 | } 44 | 45 | type UpdateShardInfo struct { 46 | CurrShardInfo metadata.ShardInfo 47 | } 48 | 49 | type CreateTableOnShardRequest struct { 50 | UpdateShardInfo UpdateShardInfo 51 | TableInfo metadata.TableInfo 52 | EncodedSchema []byte 53 | Engine string 54 | CreateIfNotExist bool 55 | Options map[string]string 56 | } 57 | 58 | type DropTableOnShardRequest struct { 59 | UpdateShardInfo UpdateShardInfo 60 | TableInfo metadata.TableInfo 61 | } 62 | 63 | type OpenTableOnShardRequest struct { 64 | UpdateShardInfo UpdateShardInfo 65 | TableInfo metadata.TableInfo 66 | } 67 | 68 | type CloseTableOnShardRequest struct { 69 | UpdateShardInfo UpdateShardInfo 70 | TableInfo metadata.TableInfo 71 | } 72 | -------------------------------------------------------------------------------- /server/coordinator/procedure/ddl/createtable/create_table_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package createtable_test 21 | 22 | import ( 23 | "context" 24 | "fmt" 25 | "testing" 26 | 27 | "github.com/apache/incubator-horaedb-meta/server/cluster/metadata" 28 | "github.com/apache/incubator-horaedb-meta/server/coordinator/procedure/ddl/createtable" 29 | "github.com/apache/incubator-horaedb-meta/server/coordinator/procedure/test" 30 | "github.com/apache/incubator-horaedb-proto/golang/pkg/metaservicepb" 31 | "github.com/stretchr/testify/require" 32 | ) 33 | 34 | func TestCreateTable(t *testing.T) { 35 | re := require.New(t) 36 | ctx := context.Background() 37 | dispatch := test.MockDispatch{} 38 | c := test.InitStableCluster(ctx, t) 39 | 40 | // Select a shard to create table. 41 | snapshot := c.GetMetadata().GetClusterSnapshot() 42 | shardNode := snapshot.Topology.ClusterView.ShardNodes[0] 43 | 44 | // New CreateTableProcedure to create a new table. 45 | p, err := createtable.NewProcedure(createtable.ProcedureParams{ 46 | Dispatch: dispatch, 47 | ClusterMetadata: c.GetMetadata(), 48 | ClusterSnapshot: snapshot, 49 | ID: uint64(1), 50 | ShardID: shardNode.ID, 51 | SourceReq: &metaservicepb.CreateTableRequest{ 52 | Header: &metaservicepb.RequestHeader{ 53 | Node: shardNode.NodeName, 54 | ClusterName: test.ClusterName, 55 | }, 56 | SchemaName: test.TestSchemaName, 57 | Name: test.TestTableName0, 58 | }, 59 | OnSucceeded: func(_ metadata.CreateTableResult) error { 60 | return nil 61 | }, 62 | OnFailed: func(err error) error { 63 | panic(fmt.Sprintf("create table failed, err:%v", err)) 64 | }, 65 | }) 66 | re.NoError(err) 67 | err = p.Start(context.Background()) 68 | re.NoError(err) 69 | } 70 | -------------------------------------------------------------------------------- /server/etcdutil/config.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package etcdutil 21 | 22 | import ( 23 | "fmt" 24 | "net/url" 25 | "os" 26 | "testing" 27 | 28 | "github.com/stretchr/testify/assert" 29 | "github.com/tikv/pd/pkg/tempurl" 30 | clientv3 "go.etcd.io/etcd/client/v3" 31 | "go.etcd.io/etcd/server/v3/embed" 32 | ) 33 | 34 | type CloseFn = func() 35 | 36 | // NewTestSingleConfig is used to create an etcd config for the unit test purpose. 37 | func NewTestSingleConfig() *embed.Config { 38 | cfg := embed.NewConfig() 39 | cfg.Name = "test_etcd" 40 | cfg.Dir, _ = os.MkdirTemp("/tmp", "test_etcd") 41 | cfg.WalDir = "" 42 | cfg.Logger = "zap" 43 | cfg.LogOutputs = []string{"stdout"} 44 | 45 | pu, _ := url.Parse(tempurl.Alloc()) 46 | cfg.LPUrls = []url.URL{*pu} 47 | cfg.APUrls = cfg.LPUrls 48 | cu, _ := url.Parse(tempurl.Alloc()) 49 | cfg.LCUrls = []url.URL{*cu} 50 | cfg.ACUrls = cfg.LCUrls 51 | 52 | cfg.StrictReconfigCheck = false 53 | cfg.InitialCluster = fmt.Sprintf("%s=%s", cfg.Name, &cfg.LPUrls[0]) 54 | cfg.ClusterState = embed.ClusterStateFlagNew 55 | return cfg 56 | } 57 | 58 | // CleanConfig is used to clean the etcd data for the unit test purpose. 59 | func CleanConfig(cfg *embed.Config) { 60 | // Clean data directory 61 | os.RemoveAll(cfg.Dir) 62 | } 63 | 64 | // PrepareEtcdServerAndClient makes the server and client for testing. 65 | // 66 | // Caller should take responsibilities to close the server and client. 67 | func PrepareEtcdServerAndClient(t *testing.T) (*embed.Etcd, *clientv3.Client, CloseFn) { 68 | cfg := NewTestSingleConfig() 69 | etcd, err := embed.StartEtcd(cfg) 70 | assert.NoError(t, err) 71 | 72 | <-etcd.Server.ReadyNotify() 73 | 74 | endpoint := cfg.LCUrls[0].String() 75 | client, err := clientv3.New(clientv3.Config{ 76 | Endpoints: []string{endpoint}, 77 | }) 78 | assert.NoError(t, err) 79 | 80 | closeSrv := func() { 81 | etcd.Close() 82 | CleanConfig(cfg) 83 | } 84 | return etcd, client, closeSrv 85 | } 86 | -------------------------------------------------------------------------------- /server/coordinator/scheduler/reopen/scheduler_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package reopen_test 21 | 22 | import ( 23 | "context" 24 | "testing" 25 | 26 | "github.com/apache/incubator-horaedb-meta/server/cluster/metadata" 27 | "github.com/apache/incubator-horaedb-meta/server/coordinator" 28 | "github.com/apache/incubator-horaedb-meta/server/coordinator/procedure/test" 29 | "github.com/apache/incubator-horaedb-meta/server/coordinator/scheduler/reopen" 30 | "github.com/apache/incubator-horaedb-meta/server/storage" 31 | "github.com/stretchr/testify/require" 32 | "go.uber.org/zap" 33 | ) 34 | 35 | func TestReopenShardScheduler(t *testing.T) { 36 | re := require.New(t) 37 | ctx := context.Background() 38 | emptyCluster := test.InitEmptyCluster(ctx, t) 39 | 40 | procedureFactory := coordinator.NewFactory(zap.NewNop(), test.MockIDAllocator{}, test.MockDispatch{}, test.NewTestStorage(t), emptyCluster.GetMetadata()) 41 | 42 | s := reopen.NewShardScheduler(procedureFactory, 1) 43 | 44 | // ReopenShardScheduler should not schedule when cluster is not stable. 45 | result, err := s.Schedule(ctx, emptyCluster.GetMetadata().GetClusterSnapshot()) 46 | re.NoError(err) 47 | re.Nil(result.Procedure) 48 | 49 | stableCluster := test.InitStableCluster(ctx, t) 50 | snapshot := stableCluster.GetMetadata().GetClusterSnapshot() 51 | 52 | // Add shard with ready status. 53 | snapshot.RegisteredNodes[0].ShardInfos = append(snapshot.RegisteredNodes[0].ShardInfos, metadata.ShardInfo{ 54 | ID: 0, 55 | Role: storage.ShardRoleLeader, 56 | Version: 0, 57 | Status: storage.ShardStatusReady, 58 | }) 59 | re.NoError(err) 60 | re.Nil(result.Procedure) 61 | 62 | // Add shard with partitionOpen status. 63 | snapshot.RegisteredNodes[0].ShardInfos = append(snapshot.RegisteredNodes[0].ShardInfos, metadata.ShardInfo{ 64 | ID: 1, 65 | Role: storage.ShardRoleLeader, 66 | Version: 0, 67 | Status: storage.ShardStatusPartialOpen, 68 | }) 69 | result, err = s.Schedule(ctx, snapshot) 70 | re.NoError(err) 71 | re.NotNil(result.Procedure) 72 | } 73 | -------------------------------------------------------------------------------- /server/coordinator/persist_shard_picker.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package coordinator 21 | 22 | import ( 23 | "context" 24 | 25 | "github.com/apache/incubator-horaedb-meta/server/cluster/metadata" 26 | "github.com/apache/incubator-horaedb-meta/server/storage" 27 | ) 28 | 29 | type PersistShardPicker struct { 30 | cluster *metadata.ClusterMetadata 31 | internal ShardPicker 32 | } 33 | 34 | func NewPersistShardPicker(cluster *metadata.ClusterMetadata, internal ShardPicker) *PersistShardPicker { 35 | return &PersistShardPicker{cluster: cluster, internal: internal} 36 | } 37 | 38 | func (p *PersistShardPicker) PickShards(ctx context.Context, snapshot metadata.Snapshot, schemaName string, tableNames []string) (map[string]storage.ShardNode, error) { 39 | result := map[string]storage.ShardNode{} 40 | 41 | shardNodeMap := make(map[storage.ShardID]storage.ShardNode, len(tableNames)) 42 | for _, shardNode := range snapshot.Topology.ClusterView.ShardNodes { 43 | shardNodeMap[shardNode.ID] = shardNode 44 | } 45 | 46 | var missingTables []string 47 | // If table assign has been created, just reuse it. 48 | for i := 0; i < len(tableNames); i++ { 49 | shardID, exists, err := p.cluster.GetTableAssignedShard(ctx, schemaName, tableNames[i]) 50 | if err != nil { 51 | return map[string]storage.ShardNode{}, err 52 | } 53 | if exists { 54 | result[tableNames[i]] = shardNodeMap[shardID] 55 | } else { 56 | missingTables = append(missingTables, tableNames[i]) 57 | } 58 | } 59 | 60 | // All table has been assigned to shard. 61 | if len(missingTables) == 0 { 62 | return result, nil 63 | } 64 | 65 | // No table assign has been created, try to pick shard and save table assigns. 66 | shardNodes, err := p.internal.PickShards(ctx, snapshot, len(missingTables)) 67 | if err != nil { 68 | return map[string]storage.ShardNode{}, err 69 | } 70 | 71 | for i, shardNode := range shardNodes { 72 | result[missingTables[i]] = shardNode 73 | err = p.cluster.AssignTableToShard(ctx, schemaName, missingTables[i], shardNode.ID) 74 | if err != nil { 75 | return map[string]storage.ShardNode{}, err 76 | } 77 | } 78 | 79 | return result, nil 80 | } 81 | -------------------------------------------------------------------------------- /server/coordinator/procedure/error.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package procedure 21 | 22 | import "github.com/apache/incubator-horaedb-meta/pkg/coderr" 23 | 24 | var ( 25 | ErrShardLeaderNotFound = coderr.NewCodeError(coderr.Internal, "shard leader not found") 26 | ErrShardNotMatch = coderr.NewCodeError(coderr.Internal, "target shard not match to persis data") 27 | ErrProcedureNotFound = coderr.NewCodeError(coderr.Internal, "procedure not found") 28 | ErrClusterConfigChanged = coderr.NewCodeError(coderr.Internal, "cluster config changed") 29 | ErrTableNotExists = coderr.NewCodeError(coderr.Internal, "table not exists") 30 | ErrTableAlreadyExists = coderr.NewCodeError(coderr.Internal, "table already exists") 31 | ErrListRunningProcedure = coderr.NewCodeError(coderr.Internal, "procedure type not match") 32 | ErrListProcedure = coderr.NewCodeError(coderr.Internal, "list running procedure") 33 | ErrDecodeRawData = coderr.NewCodeError(coderr.Internal, "decode raw data") 34 | ErrEncodeRawData = coderr.NewCodeError(coderr.Internal, "encode raw data") 35 | ErrGetRequest = coderr.NewCodeError(coderr.Internal, "get request from event") 36 | ErrNodeNumberNotEnough = coderr.NewCodeError(coderr.Internal, "node number not enough") 37 | ErrEmptyPartitionNames = coderr.NewCodeError(coderr.Internal, "partition names is empty") 38 | ErrDropTableResult = coderr.NewCodeError(coderr.Internal, "length of shard not correct") 39 | ErrPickShard = coderr.NewCodeError(coderr.Internal, "pick shard failed") 40 | ErrSubmitProcedure = coderr.NewCodeError(coderr.Internal, "submit new procedure") 41 | ErrQueueFull = coderr.NewCodeError(coderr.Internal, "queue is full, unable to offer more data") 42 | ErrPushDuplicatedProcedure = coderr.NewCodeError(coderr.Internal, "try to push duplicated procedure") 43 | ErrShardNumberNotEnough = coderr.NewCodeError(coderr.Internal, "shard number not enough") 44 | ErrEmptyBatchProcedure = coderr.NewCodeError(coderr.Internal, "procedure batch is empty") 45 | ErrMergeBatchProcedure = coderr.NewCodeError(coderr.Internal, "failed to merge procedures batch") 46 | ) 47 | -------------------------------------------------------------------------------- /server/cluster/metadata/compare_benchmark_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package metadata 21 | 22 | import ( 23 | "testing" 24 | 25 | "github.com/apache/incubator-horaedb-meta/server/storage" 26 | ) 27 | 28 | func buildRegisterNode(shardNumber int) RegisteredNode { 29 | shardInfos := make([]ShardInfo, 0, shardNumber) 30 | for i := shardNumber; i > 0; i-- { 31 | shardInfos = append(shardInfos, ShardInfo{ 32 | ID: storage.ShardID(i), 33 | Role: 0, 34 | Version: 0, 35 | Status: storage.ShardStatusUnknown, 36 | }) 37 | } 38 | return RegisteredNode{ 39 | Node: storage.Node{ 40 | Name: "", 41 | NodeStats: storage.NewEmptyNodeStats(), 42 | LastTouchTime: 0, 43 | State: storage.NodeStateUnknown, 44 | }, 45 | ShardInfos: shardInfos, 46 | } 47 | } 48 | 49 | func BenchmarkSortWith10Shards(b *testing.B) { 50 | registerNode := buildRegisterNode(10) 51 | oldCache := registerNode 52 | for i := 0; i < b.N; i++ { 53 | sortCompare(oldCache.ShardInfos, registerNode.ShardInfos) 54 | } 55 | } 56 | 57 | func BenchmarkCompareWith10Shards(b *testing.B) { 58 | registerNode := buildRegisterNode(10) 59 | oldCache := registerNode 60 | for i := 0; i < b.N; i++ { 61 | simpleCompare(oldCache.ShardInfos, registerNode.ShardInfos) 62 | } 63 | } 64 | 65 | func BenchmarkSortWith50Shards(b *testing.B) { 66 | registerNode := buildRegisterNode(50) 67 | oldCache := registerNode 68 | for i := 0; i < b.N; i++ { 69 | sortCompare(oldCache.ShardInfos, registerNode.ShardInfos) 70 | } 71 | } 72 | 73 | func BenchmarkCompareWith50Shards(b *testing.B) { 74 | registerNode := buildRegisterNode(50) 75 | oldCache := registerNode 76 | for i := 0; i < b.N; i++ { 77 | simpleCompare(oldCache.ShardInfos, registerNode.ShardInfos) 78 | } 79 | } 80 | 81 | func BenchmarkSortWith100Shards(b *testing.B) { 82 | registerNode := buildRegisterNode(100) 83 | oldCache := registerNode 84 | for i := 0; i < b.N; i++ { 85 | sortCompare(oldCache.ShardInfos, registerNode.ShardInfos) 86 | } 87 | } 88 | 89 | func BenchmarkCompareWith100Shards(b *testing.B) { 90 | registerNode := buildRegisterNode(100) 91 | oldCache := registerNode 92 | for i := 0; i < b.N; i++ { 93 | simpleCompare(oldCache.ShardInfos, registerNode.ShardInfos) 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /server/coordinator/scheduler/rebalanced/scheduler_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package rebalanced_test 21 | 22 | import ( 23 | "context" 24 | "testing" 25 | 26 | "github.com/apache/incubator-horaedb-meta/server/coordinator" 27 | "github.com/apache/incubator-horaedb-meta/server/coordinator/procedure/test" 28 | "github.com/apache/incubator-horaedb-meta/server/coordinator/scheduler/nodepicker" 29 | "github.com/apache/incubator-horaedb-meta/server/coordinator/scheduler/rebalanced" 30 | "github.com/stretchr/testify/require" 31 | "go.uber.org/zap" 32 | ) 33 | 34 | func TestRebalancedScheduler(t *testing.T) { 35 | re := require.New(t) 36 | ctx := context.Background() 37 | 38 | // EmptyCluster would be scheduled an empty procedure. 39 | emptyCluster := test.InitEmptyCluster(ctx, t) 40 | procedureFactory := coordinator.NewFactory(zap.NewNop(), test.MockIDAllocator{}, test.MockDispatch{}, test.NewTestStorage(t), emptyCluster.GetMetadata()) 41 | s := rebalanced.NewShardScheduler(zap.NewNop(), procedureFactory, nodepicker.NewConsistentUniformHashNodePicker(zap.NewNop()), 1) 42 | result, err := s.Schedule(ctx, emptyCluster.GetMetadata().GetClusterSnapshot()) 43 | re.NoError(err) 44 | re.Empty(result) 45 | 46 | // PrepareCluster would be scheduled an empty procedure. 47 | prepareCluster := test.InitPrepareCluster(ctx, t) 48 | procedureFactory = coordinator.NewFactory(zap.NewNop(), test.MockIDAllocator{}, test.MockDispatch{}, test.NewTestStorage(t), prepareCluster.GetMetadata()) 49 | s = rebalanced.NewShardScheduler(zap.NewNop(), procedureFactory, nodepicker.NewConsistentUniformHashNodePicker(zap.NewNop()), 1) 50 | _, err = s.Schedule(ctx, prepareCluster.GetMetadata().GetClusterSnapshot()) 51 | re.NoError(err) 52 | 53 | // StableCluster with all shards assigned would be scheduled a load balance procedure. 54 | stableCluster := test.InitStableCluster(ctx, t) 55 | procedureFactory = coordinator.NewFactory(zap.NewNop(), test.MockIDAllocator{}, test.MockDispatch{}, test.NewTestStorage(t), stableCluster.GetMetadata()) 56 | s = rebalanced.NewShardScheduler(zap.NewNop(), procedureFactory, nodepicker.NewConsistentUniformHashNodePicker(zap.NewNop()), 1) 57 | _, err = s.Schedule(ctx, stableCluster.GetMetadata().GetClusterSnapshot()) 58 | re.NoError(err) 59 | } 60 | -------------------------------------------------------------------------------- /server/service/grpc/forward.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package grpc 21 | 22 | import ( 23 | "context" 24 | 25 | "github.com/apache/incubator-horaedb-meta/pkg/log" 26 | "github.com/apache/incubator-horaedb-meta/server/service" 27 | "github.com/apache/incubator-horaedb-proto/golang/pkg/metaservicepb" 28 | "github.com/pkg/errors" 29 | "go.uber.org/zap" 30 | "google.golang.org/grpc" 31 | ) 32 | 33 | // getForwardedMetaClient get forwarded horaemeta client. When current node is the leader, this func will return (nil,nil). 34 | func (s *Service) getForwardedMetaClient(ctx context.Context) (metaservicepb.MetaRpcServiceClient, error) { 35 | forwardedAddr, _, err := s.getForwardedAddr(ctx) 36 | if err != nil { 37 | return nil, errors.WithMessage(err, "get forwarded horaemeta client") 38 | } 39 | 40 | if forwardedAddr != "" { 41 | horaeClient, err := s.getMetaClient(ctx, forwardedAddr) 42 | if err != nil { 43 | return nil, errors.WithMessagef(err, "get forwarded horaemeta client, addr:%s", forwardedAddr) 44 | } 45 | return horaeClient, nil 46 | } 47 | return nil, nil 48 | } 49 | 50 | func (s *Service) getMetaClient(ctx context.Context, addr string) (metaservicepb.MetaRpcServiceClient, error) { 51 | client, err := s.getForwardedGrpcClient(ctx, addr) 52 | if err != nil { 53 | return nil, errors.WithMessagef(err, "get horaemeta client, addr:%s", addr) 54 | } 55 | return metaservicepb.NewMetaRpcServiceClient(client), nil 56 | } 57 | 58 | func (s *Service) getForwardedGrpcClient(ctx context.Context, forwardedAddr string) (*grpc.ClientConn, error) { 59 | client, ok := s.conns.Load(forwardedAddr) 60 | if !ok { 61 | log.Info("try to create horaemeta client", zap.String("addr", forwardedAddr)) 62 | cc, err := service.GetClientConn(ctx, forwardedAddr) 63 | if err != nil { 64 | return nil, err 65 | } 66 | client = cc 67 | s.conns.Store(forwardedAddr, cc) 68 | } 69 | return client.(*grpc.ClientConn), nil 70 | } 71 | 72 | func (s *Service) getForwardedAddr(ctx context.Context) (string, bool, error) { 73 | resp, err := s.h.GetLeader(ctx) 74 | if err != nil { 75 | return "", false, errors.WithMessage(err, "get forwarded addr") 76 | } 77 | if resp.IsLocal { 78 | return "", true, nil 79 | } 80 | return resp.LeaderEndpoint, false, nil 81 | } 82 | -------------------------------------------------------------------------------- /server/coordinator/scheduler/static/scheduler_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package static_test 21 | 22 | import ( 23 | "context" 24 | "testing" 25 | 26 | "github.com/apache/incubator-horaedb-meta/server/coordinator" 27 | "github.com/apache/incubator-horaedb-meta/server/coordinator/procedure/test" 28 | "github.com/apache/incubator-horaedb-meta/server/coordinator/scheduler/nodepicker" 29 | "github.com/apache/incubator-horaedb-meta/server/coordinator/scheduler/static" 30 | "github.com/stretchr/testify/require" 31 | "go.uber.org/zap" 32 | ) 33 | 34 | func TestStaticTopologyScheduler(t *testing.T) { 35 | re := require.New(t) 36 | ctx := context.Background() 37 | 38 | // EmptyCluster would be scheduled an empty procedure. 39 | emptyCluster := test.InitEmptyCluster(ctx, t) 40 | procedureFactory := coordinator.NewFactory(zap.NewNop(), test.MockIDAllocator{}, test.MockDispatch{}, test.NewTestStorage(t), emptyCluster.GetMetadata()) 41 | s := static.NewShardScheduler(procedureFactory, nodepicker.NewConsistentUniformHashNodePicker(zap.NewNop()), 1) 42 | result, err := s.Schedule(ctx, emptyCluster.GetMetadata().GetClusterSnapshot()) 43 | re.NoError(err) 44 | re.Empty(result) 45 | 46 | // PrepareCluster would be scheduled a transfer leader procedure. 47 | prepareCluster := test.InitPrepareCluster(ctx, t) 48 | procedureFactory = coordinator.NewFactory(zap.NewNop(), test.MockIDAllocator{}, test.MockDispatch{}, test.NewTestStorage(t), prepareCluster.GetMetadata()) 49 | s = static.NewShardScheduler(procedureFactory, nodepicker.NewConsistentUniformHashNodePicker(zap.NewNop()), 1) 50 | result, err = s.Schedule(ctx, prepareCluster.GetMetadata().GetClusterSnapshot()) 51 | re.NoError(err) 52 | re.NotEmpty(result) 53 | 54 | // StableCluster with all shards assigned would be scheduled a transfer leader procedure by hash rule. 55 | stableCluster := test.InitStableCluster(ctx, t) 56 | procedureFactory = coordinator.NewFactory(zap.NewNop(), test.MockIDAllocator{}, test.MockDispatch{}, test.NewTestStorage(t), stableCluster.GetMetadata()) 57 | s = static.NewShardScheduler(procedureFactory, nodepicker.NewConsistentUniformHashNodePicker(zap.NewNop()), 1) 58 | result, err = s.Schedule(ctx, stableCluster.GetMetadata().GetClusterSnapshot()) 59 | re.NoError(err) 60 | re.NotEmpty(result) 61 | } 62 | -------------------------------------------------------------------------------- /server/member/watch_leader_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package member 21 | 22 | import ( 23 | "context" 24 | "testing" 25 | "time" 26 | 27 | "github.com/apache/incubator-horaedb-meta/server/etcdutil" 28 | "github.com/stretchr/testify/assert" 29 | clientv3 "go.etcd.io/etcd/client/v3" 30 | "go.etcd.io/etcd/server/v3/etcdserver" 31 | ) 32 | 33 | type mockWatchCtx struct { 34 | stopped bool 35 | client *clientv3.Client 36 | srv *etcdserver.EtcdServer 37 | } 38 | 39 | func (ctx *mockWatchCtx) ShouldStop() bool { 40 | return ctx.stopped 41 | } 42 | 43 | func (ctx *mockWatchCtx) EtcdLeaderID() (uint64, error) { 44 | return ctx.srv.Lead(), nil 45 | } 46 | 47 | func TestWatchLeaderSingle(t *testing.T) { 48 | etcd, client, closeSrv := etcdutil.PrepareEtcdServerAndClient(t) 49 | defer closeSrv() 50 | 51 | watchCtx := &mockWatchCtx{ 52 | stopped: false, 53 | client: client, 54 | srv: etcd.Server, 55 | } 56 | leaderGetter := &etcdutil.LeaderGetterWrapper{Server: etcd.Server} 57 | rpcTimeout := time.Duration(10) * time.Second 58 | leaseTTLSec := int64(1) 59 | mem := NewMember("", uint64(etcd.Server.ID()), "mem0", "", client, leaderGetter, rpcTimeout) 60 | leaderWatcher := NewLeaderWatcher(watchCtx, mem, leaseTTLSec, true) 61 | 62 | ctx, cancelWatch := context.WithCancel(context.Background()) 63 | watchedDone := make(chan struct{}, 1) 64 | go func() { 65 | leaderWatcher.Watch(ctx, nil) 66 | watchedDone <- struct{}{} 67 | }() 68 | 69 | // Wait for watcher starting 70 | // TODO: This unit test may fail. Currently, it is solved by increasing the sleep time, and the code needs to be optimized in the future. 71 | time.Sleep(time.Duration(2000) * time.Millisecond) 72 | 73 | // check the member has been the leader 74 | ctx, cancel := context.WithTimeout(context.Background(), rpcTimeout) 75 | defer cancel() 76 | resp, err := mem.getLeader(ctx) 77 | assert.NoError(t, err) 78 | assert.NotNil(t, resp) 79 | assert.Equal(t, resp.Leader.Id, mem.ID) 80 | 81 | // cancel the watch 82 | cancelWatch() 83 | <-watchedDone 84 | 85 | // check again whether the leader should be reset 86 | ctx, cancel = context.WithTimeout(context.Background(), rpcTimeout) 87 | defer cancel() 88 | resp, err = mem.getLeader(ctx) 89 | assert.NoError(t, err) 90 | assert.NotNil(t, resp) 91 | assert.Nil(t, resp.Leader) 92 | } 93 | -------------------------------------------------------------------------------- /server/coordinator/watch/watch_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package watch 21 | 22 | import ( 23 | "context" 24 | "testing" 25 | "time" 26 | 27 | "github.com/apache/incubator-horaedb-meta/server/etcdutil" 28 | "github.com/apache/incubator-horaedb-meta/server/storage" 29 | "github.com/apache/incubator-horaedb-proto/golang/pkg/metaeventpb" 30 | "github.com/stretchr/testify/require" 31 | clientv3 "go.etcd.io/etcd/client/v3" 32 | "go.uber.org/zap" 33 | "google.golang.org/protobuf/proto" 34 | ) 35 | 36 | const ( 37 | TestClusterName = "defaultCluster" 38 | TestRootPath = "/rootPath" 39 | TestShardPath = "shards" 40 | TestShardID = 1 41 | TestNodeName = "testNode" 42 | ) 43 | 44 | func TestWatch(t *testing.T) { 45 | re := require.New(t) 46 | ctx := context.Background() 47 | 48 | _, client, _ := etcdutil.PrepareEtcdServerAndClient(t) 49 | watch := NewEtcdShardWatch(zap.NewNop(), TestClusterName, TestRootPath, client) 50 | err := watch.Start(ctx) 51 | re.NoError(err) 52 | 53 | testCallback := testShardEventCallback{ 54 | result: 0, 55 | re: re, 56 | } 57 | 58 | watch.RegisteringEventCallback(&testCallback) 59 | 60 | // Valid that callback function is executed and the params are as expected. 61 | b, err := proto.Marshal(&metaeventpb.ShardLockValue{NodeName: TestNodeName}) 62 | re.NoError(err) 63 | 64 | keyPath := encodeShardKey(TestRootPath, TestClusterName, TestShardPath, TestShardID) 65 | _, err = client.Put(ctx, keyPath, string(b)) 66 | re.NoError(err) 67 | time.Sleep(time.Millisecond * 10) 68 | re.Equal(2, testCallback.result) 69 | 70 | _, err = client.Delete(ctx, keyPath, clientv3.WithPrevKV()) 71 | re.NoError(err) 72 | time.Sleep(time.Millisecond * 10) 73 | re.Equal(1, testCallback.result) 74 | } 75 | 76 | type testShardEventCallback struct { 77 | result int 78 | re *require.Assertions 79 | } 80 | 81 | func (c *testShardEventCallback) OnShardRegistered(_ context.Context, event ShardRegisterEvent) error { 82 | c.result = 2 83 | c.re.Equal(storage.ShardID(TestShardID), event.ShardID) 84 | c.re.Equal(TestNodeName, event.NewLeaderNode) 85 | return nil 86 | } 87 | 88 | func (c *testShardEventCallback) OnShardExpired(_ context.Context, event ShardExpireEvent) error { 89 | c.result = 1 90 | c.re.Equal(storage.ShardID(TestShardID), event.ShardID) 91 | c.re.Equal(TestNodeName, event.OldLeaderNode) 92 | return nil 93 | } 94 | -------------------------------------------------------------------------------- /server/coordinator/scheduler/manager/scheduler_manager_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package manager_test 21 | 22 | import ( 23 | "context" 24 | "testing" 25 | 26 | "github.com/apache/incubator-horaedb-meta/server/coordinator" 27 | "github.com/apache/incubator-horaedb-meta/server/coordinator/procedure" 28 | "github.com/apache/incubator-horaedb-meta/server/coordinator/procedure/test" 29 | "github.com/apache/incubator-horaedb-meta/server/coordinator/scheduler/manager" 30 | "github.com/apache/incubator-horaedb-meta/server/etcdutil" 31 | "github.com/apache/incubator-horaedb-meta/server/storage" 32 | "github.com/stretchr/testify/require" 33 | "go.uber.org/zap" 34 | ) 35 | 36 | func TestSchedulerManager(t *testing.T) { 37 | ctx := context.Background() 38 | re := require.New(t) 39 | 40 | // Init dependencies for scheduler manager. 41 | c := test.InitStableCluster(ctx, t) 42 | procedureManager, err := procedure.NewManagerImpl(zap.NewNop(), c.GetMetadata()) 43 | re.NoError(err) 44 | dispatch := test.MockDispatch{} 45 | allocator := test.MockIDAllocator{} 46 | s := test.NewTestStorage(t) 47 | f := coordinator.NewFactory(zap.NewNop(), allocator, dispatch, s, c.GetMetadata()) 48 | _, client, _ := etcdutil.PrepareEtcdServerAndClient(t) 49 | 50 | // Create scheduler manager with enableScheduler equal to false. 51 | schedulerManager := manager.NewManager(zap.NewNop(), procedureManager, f, c.GetMetadata(), client, "/rootPath", storage.TopologyTypeStatic, 1) 52 | err = schedulerManager.Start(ctx) 53 | re.NoError(err) 54 | err = schedulerManager.Stop(ctx) 55 | re.NoError(err) 56 | 57 | // Create scheduler manager with static topology. 58 | schedulerManager = manager.NewManager(zap.NewNop(), procedureManager, f, c.GetMetadata(), client, "/rootPath", storage.TopologyTypeStatic, 1) 59 | err = schedulerManager.Start(ctx) 60 | re.NoError(err) 61 | schedulers := schedulerManager.ListScheduler() 62 | re.Equal(2, len(schedulers)) 63 | err = schedulerManager.Stop(ctx) 64 | re.NoError(err) 65 | 66 | // Create scheduler manager with dynamic topology. 67 | schedulerManager = manager.NewManager(zap.NewNop(), procedureManager, f, c.GetMetadata(), client, "/rootPath", storage.TopologyTypeDynamic, 1) 68 | err = schedulerManager.Start(ctx) 69 | re.NoError(err) 70 | schedulers = schedulerManager.ListScheduler() 71 | re.Equal(2, len(schedulers)) 72 | err = schedulerManager.Stop(ctx) 73 | re.NoError(err) 74 | } 75 | -------------------------------------------------------------------------------- /pkg/coderr/error.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package coderr 21 | 22 | import ( 23 | "fmt" 24 | 25 | "github.com/pkg/errors" 26 | ) 27 | 28 | var _ CodeError = &codeError{code: 0, desc: "", cause: nil} 29 | 30 | // CodeError is an error with code. 31 | type CodeError interface { 32 | error 33 | Code() Code 34 | // WithCausef should generate a new CodeError instance with the provided cause details. 35 | WithCausef(format string, a ...any) CodeError 36 | // WithCause should generate a new CodeError instance with the provided cause details. 37 | WithCause(cause error) CodeError 38 | } 39 | 40 | // Is checks whether the cause of `err` is the kind of error specified by the `expectCode`. 41 | // Returns false if the cause of `err` is not CodeError. 42 | func Is(err error, expectCode Code) bool { 43 | code, b := GetCauseCode(err) 44 | if b && code == expectCode { 45 | return true 46 | } 47 | 48 | return false 49 | } 50 | 51 | func GetCauseCode(err error) (Code, bool) { 52 | if err == nil { 53 | return Invalid, false 54 | } 55 | 56 | cause := errors.Cause(err) 57 | cerr, ok := cause.(CodeError) 58 | if !ok { 59 | return Invalid, false 60 | } 61 | return cerr.Code(), true 62 | } 63 | 64 | // NewCodeError creates a base CodeError definition. 65 | // The provided code should be defined in the code.go in this package. 66 | func NewCodeError(code Code, desc string) CodeError { 67 | return &codeError{ 68 | code: code, 69 | desc: desc, 70 | cause: nil, 71 | } 72 | } 73 | 74 | // codeError is the default implementation of CodeError. 75 | type codeError struct { 76 | code Code 77 | desc string 78 | cause error 79 | } 80 | 81 | func (e *codeError) Error() string { 82 | return fmt.Sprintf("(#%d)%s, cause:%+v", e.code, e.desc, e.cause) 83 | } 84 | 85 | func (e *codeError) Code() Code { 86 | return e.code 87 | } 88 | 89 | func (e *codeError) WithCausef(format string, a ...any) CodeError { 90 | errMsg := fmt.Sprintf(format, a...) 91 | causeWithStack := errors.WithStack(errors.New(errMsg)) 92 | return &codeError{ 93 | code: e.code, 94 | desc: e.desc, 95 | cause: causeWithStack, 96 | } 97 | } 98 | 99 | func (e *codeError) WithCause(cause error) CodeError { 100 | causeWithStack := errors.WithStack(cause) 101 | return &codeError{ 102 | code: e.code, 103 | desc: e.desc, 104 | cause: causeWithStack, 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /server/coordinator/procedure/procedure.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package procedure 21 | 22 | import ( 23 | "context" 24 | 25 | "github.com/apache/incubator-horaedb-meta/server/storage" 26 | ) 27 | 28 | type State string 29 | 30 | const ( 31 | StateInit = "init" 32 | StateRunning = "running" 33 | StateFinished = "finished" 34 | StateFailed = "failed" 35 | StateCancelled = "cancelled" 36 | ) 37 | 38 | type Kind uint 39 | 40 | const ( 41 | // Cluster Operation 42 | Create Kind = iota 43 | Delete 44 | TransferLeader 45 | Migrate 46 | Split 47 | Merge 48 | Scatter 49 | 50 | // DDL 51 | CreateTable 52 | DropTable 53 | CreatePartitionTable 54 | DropPartitionTable 55 | ) 56 | 57 | type Priority uint32 58 | 59 | // Lower value means higher priority. 60 | const ( 61 | PriorityHigh Priority = 3 62 | PriorityMed Priority = 5 63 | PriorityLow Priority = 10 64 | ) 65 | 66 | // Procedure is used to describe how to execute a set of operations from the scheduler, e.g. SwitchLeaderProcedure, MergeShardProcedure. 67 | type Procedure interface { 68 | // ID of the procedure. 69 | ID() uint64 70 | 71 | // Kind of the procedure. 72 | Kind() Kind 73 | 74 | // Start the procedure. 75 | Start(ctx context.Context) error 76 | 77 | // Cancel the procedure. 78 | Cancel(ctx context.Context) error 79 | 80 | // State of the procedure. Retrieve the state of this procedure. 81 | State() State 82 | 83 | // RelatedVersionInfo return the related shard and version information corresponding to this procedure for verifying whether the procedure can be executed. 84 | RelatedVersionInfo() RelatedVersionInfo 85 | 86 | // Priority present the priority of this procedure, the procedure with high level priority will be executed first. 87 | Priority() Priority 88 | } 89 | 90 | // Info is used to provide immutable description procedure information. 91 | type Info struct { 92 | ID uint64 93 | Kind Kind 94 | State State 95 | } 96 | 97 | type RelatedVersionInfo struct { 98 | ClusterID storage.ClusterID 99 | // shardWithVersion return the shardID associated with this procedure. 100 | ShardWithVersion map[storage.ShardID]uint64 101 | // clusterVersion return the cluster version when the procedure is created. 102 | // When performing cluster operation, it is necessary to ensure cluster version consistency. 103 | ClusterVersion uint64 104 | } 105 | -------------------------------------------------------------------------------- /server/service/http/error.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package http 21 | 22 | import "github.com/apache/incubator-horaedb-meta/pkg/coderr" 23 | 24 | var ( 25 | ErrParseRequest = coderr.NewCodeError(coderr.BadRequest, "parse request params") 26 | ErrInvalidParamsForCreateCluster = coderr.NewCodeError(coderr.BadRequest, "invalid params to create cluster") 27 | ErrTable = coderr.NewCodeError(coderr.Internal, "table") 28 | ErrRoute = coderr.NewCodeError(coderr.Internal, "route table") 29 | ErrGetNodeShards = coderr.NewCodeError(coderr.Internal, "get node shards") 30 | ErrDropNodeShards = coderr.NewCodeError(coderr.Internal, "drop node shards") 31 | ErrCreateProcedure = coderr.NewCodeError(coderr.Internal, "create procedure") 32 | ErrSubmitProcedure = coderr.NewCodeError(coderr.Internal, "submit procedure") 33 | ErrGetCluster = coderr.NewCodeError(coderr.Internal, "get cluster") 34 | ErrAllocShardID = coderr.NewCodeError(coderr.Internal, "alloc shard id") 35 | ErrForwardToLeader = coderr.NewCodeError(coderr.Internal, "forward to leader") 36 | ErrParseLeaderAddr = coderr.NewCodeError(coderr.Internal, "parse leader addr") 37 | ErrHealthCheck = coderr.NewCodeError(coderr.Internal, "server health check") 38 | ErrParseTopology = coderr.NewCodeError(coderr.Internal, "parse topology type") 39 | ErrUpdateFlowLimiter = coderr.NewCodeError(coderr.Internal, "update flow limiter") 40 | ErrGetEnableSchedule = coderr.NewCodeError(coderr.Internal, "get enableSchedule") 41 | ErrUpdateEnableSchedule = coderr.NewCodeError(coderr.Internal, "update enableSchedule") 42 | ErrAddLearner = coderr.NewCodeError(coderr.Internal, "add member as learner") 43 | ErrListMembers = coderr.NewCodeError(coderr.Internal, "get member list") 44 | ErrRemoveMembers = coderr.NewCodeError(coderr.Internal, "remove member") 45 | ErrGetMember = coderr.NewCodeError(coderr.Internal, "get member") 46 | ErrListAffinityRules = coderr.NewCodeError(coderr.Internal, "list affinity rules") 47 | ErrAddAffinityRule = coderr.NewCodeError(coderr.Internal, "add affinity rule") 48 | ErrRemoveAffinityRule = coderr.NewCodeError(coderr.Internal, "remove affinity rule") 49 | ) 50 | -------------------------------------------------------------------------------- /server/coordinator/procedure/delay_queue_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package procedure 21 | 22 | import ( 23 | "context" 24 | "testing" 25 | "time" 26 | 27 | "github.com/apache/incubator-horaedb-meta/server/storage" 28 | "github.com/stretchr/testify/require" 29 | ) 30 | 31 | type TestProcedure struct{ ProcedureID uint64 } 32 | 33 | func (t TestProcedure) RelatedVersionInfo() RelatedVersionInfo { 34 | return RelatedVersionInfo{ 35 | ClusterID: 0, 36 | ShardWithVersion: map[storage.ShardID]uint64{}, 37 | ClusterVersion: 0, 38 | } 39 | } 40 | 41 | func (t TestProcedure) Priority() Priority { 42 | return PriorityLow 43 | } 44 | 45 | func (t TestProcedure) ID() uint64 { 46 | return t.ProcedureID 47 | } 48 | 49 | func (t TestProcedure) Kind() Kind { 50 | return CreateTable 51 | } 52 | 53 | func (t TestProcedure) Start(_ context.Context) error { 54 | return nil 55 | } 56 | 57 | func (t TestProcedure) Cancel(_ context.Context) error { 58 | return nil 59 | } 60 | 61 | func (t TestProcedure) State() State { 62 | return StateInit 63 | } 64 | 65 | func TestDelayQueue(t *testing.T) { 66 | re := require.New(t) 67 | 68 | testProcedure0 := TestProcedure{ProcedureID: 0} 69 | testProcedure1 := TestProcedure{ProcedureID: 1} 70 | testProcedure2 := TestProcedure{ProcedureID: 2} 71 | testProcedure3 := TestProcedure{ProcedureID: 3} 72 | 73 | queue := NewProcedureDelayQueue(3) 74 | err := queue.Push(testProcedure0, time.Millisecond*40) 75 | re.NoError(err) 76 | err = queue.Push(testProcedure0, time.Millisecond*30) 77 | re.Error(err) 78 | err = queue.Push(testProcedure1, time.Millisecond*10) 79 | re.NoError(err) 80 | err = queue.Push(testProcedure2, time.Millisecond*20) 81 | re.NoError(err) 82 | err = queue.Push(testProcedure3, time.Millisecond*20) 83 | re.Error(err) 84 | re.Equal(3, queue.Len()) 85 | 86 | po := queue.Pop() 87 | re.Nil(po) 88 | 89 | time.Sleep(time.Millisecond * 100) 90 | 91 | p0 := queue.Pop() 92 | re.Equal(uint64(1), p0.ID()) 93 | p1 := queue.Pop() 94 | re.Equal(uint64(2), p1.ID()) 95 | p2 := queue.Pop() 96 | re.Equal(uint64(0), p2.ID()) 97 | p := queue.Pop() 98 | re.Nil(p) 99 | 100 | err = queue.Push(testProcedure0, time.Millisecond*20) 101 | re.NoError(err) 102 | 103 | time.Sleep(time.Millisecond * 10) 104 | p0 = queue.Pop() 105 | re.Nil(p0) 106 | 107 | time.Sleep(time.Millisecond * 10) 108 | p0 = queue.Pop() 109 | re.Equal(uint64(0), p0.ID()) 110 | } 111 | -------------------------------------------------------------------------------- /server/id/reusable_id_impl.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package id 21 | 22 | import ( 23 | "context" 24 | "sort" 25 | "sync" 26 | ) 27 | 28 | type ReusableAllocatorImpl struct { 29 | // Mutex is used to protect following fields. 30 | lock sync.Mutex 31 | 32 | minID uint64 33 | existIDs *OrderedList 34 | } 35 | 36 | type OrderedList struct { 37 | sorted []uint64 38 | } 39 | 40 | // FindMinHoleValueAndIndex Find the minimum hole value and its index. 41 | // If the list is empty, then return min value and 0 as index; 42 | // If no hole is found, then return the `last_value + 1` in the list and l.Len() as the index; 43 | func (l *OrderedList) FindMinHoleValueAndIndex(min uint64) (uint64, int) { 44 | if len(l.sorted) == 0 { 45 | return min, 0 46 | } 47 | if l.sorted[0] > min { 48 | return min, 0 49 | } 50 | if len(l.sorted) == 1 { 51 | return l.sorted[0] + 1, 1 52 | } 53 | 54 | s := l.sorted 55 | for i := 0; i < len(l.sorted)-1; i++ { 56 | if s[i]+1 != s[i+1] { 57 | return s[i] + 1, i + 1 58 | } 59 | } 60 | 61 | return s[len(s)-1] + 1, len(s) 62 | } 63 | 64 | // Insert the value at the idx whose correctness should be ensured by the caller. 65 | func (l *OrderedList) Insert(v uint64, i int) { 66 | if len(l.sorted) == i { 67 | l.sorted = append(l.sorted, v) 68 | } else { 69 | l.sorted = append(l.sorted[:i+1], l.sorted[i:]...) 70 | l.sorted[i] = v 71 | } 72 | } 73 | 74 | func (l *OrderedList) Remove(v uint64) int { 75 | removeIndex := -1 76 | for i, value := range l.sorted { 77 | if value == v { 78 | removeIndex = i 79 | } 80 | } 81 | l.sorted = append(l.sorted[:removeIndex], l.sorted[removeIndex+1:]...) 82 | return removeIndex 83 | } 84 | 85 | func NewReusableAllocatorImpl(existIDs []uint64, minID uint64) Allocator { 86 | sort.Slice(existIDs, func(i, j int) bool { 87 | return existIDs[i] < existIDs[j] 88 | }) 89 | return &ReusableAllocatorImpl{ 90 | lock: sync.Mutex{}, 91 | 92 | minID: minID, 93 | existIDs: &OrderedList{sorted: existIDs}, 94 | } 95 | } 96 | 97 | func (a *ReusableAllocatorImpl) Alloc(_ context.Context) (uint64, error) { 98 | a.lock.Lock() 99 | defer a.lock.Unlock() 100 | // Find minimum unused ID bigger than minID 101 | v, i := a.existIDs.FindMinHoleValueAndIndex(a.minID) 102 | a.existIDs.Insert(v, i) 103 | return v, nil 104 | } 105 | 106 | func (a *ReusableAllocatorImpl) Collect(_ context.Context, id uint64) error { 107 | a.lock.Lock() 108 | defer a.lock.Unlock() 109 | a.existIDs.Remove(id) 110 | return nil 111 | } 112 | -------------------------------------------------------------------------------- /docs/style_guide.md: -------------------------------------------------------------------------------- 1 | # Style Guide 2 | HoraeMeta is written in Golang so the basic code style we adhere to is the [CodeReviewComments](https://github.com/golang/go/wiki/CodeReviewComments). 3 | 4 | Besides the [CodeReviewComments](https://github.com/golang/go/wiki/CodeReviewComments), there are also some custom rules for the project: 5 | - Error Handling 6 | - Logging 7 | 8 | ## Error Handling 9 | ### Principles 10 | - Global error code: 11 | - Any error defined in the repo should be assigned an error code, 12 | - An error code can be used by multiple different errors, 13 | - The error codes are defined in the single global package [coderr](https://github.com/apache/incubator-horaedb-meta/tree/main/pkg/coderr). 14 | - Construct: define leaf errors on package level (often in a separate `error.go` file) by package [coderr](https://github.com/apache/incubator-horaedb-meta/tree/main/pkg/coderr). 15 | - Wrap: wrap errors by `errors.WithMessage` or `errors.WithMessagef`. 16 | - Check: test the error identity by calling `coderr.Is`. 17 | - Log: only log the error on the top level package. 18 | - Respond: respond the `CodeError`(defined in package [coderr](https://github.com/apache/incubator-horaedb-meta/tree/main/pkg/coderr)) unwrapped by `errors.Cause` to client on service level. 19 | 20 | ### Example 21 | `errors.go` in the package `server`: 22 | ```go 23 | var ErrStartEtcd = coderr.NewCodeError(coderr.Internal, "start embed etcd") 24 | var ErrStartEtcdTimeout = coderr.NewCodeError(coderr.Internal, "start etcd server timeout") 25 | ``` 26 | 27 | `server.go` in the package `server`: 28 | ```go 29 | func (srv *Server) startEtcd() error { 30 | etcdSrv, err := embed.StartEtcd(srv.etcdCfg) 31 | if err != nil { 32 | return ErrStartEtcd.WithCause(err) 33 | } 34 | 35 | newCtx, cancel := context.WithTimeout(srv.ctx, srv.cfg.EtcdStartTimeout()) 36 | defer cancel() 37 | 38 | select { 39 | case <-etcdSrv.Server.ReadyNotify(): 40 | case <-newCtx.Done(): 41 | return ErrStartEtcdTimeout.WithCausef("timeout is:%v", srv.cfg.EtcdStartTimeout()) 42 | } 43 | 44 | return nil 45 | } 46 | ``` 47 | 48 | `main.go` in the package `main`: 49 | ```go 50 | func main() { 51 | err := srv.startEtcd() 52 | if err != nil { 53 | return 54 | } 55 | if coderr.Is(err, coderr.Internal) { 56 | log.Error("internal error") 57 | } 58 | 59 | cerr, ok := err.(coderr.CodeError) 60 | if ok { 61 | log.Error("found a CodeError") 62 | } else { 63 | log.Error("not a CodeError) 64 | } 65 | 66 | return 67 | } 68 | ``` 69 | 70 | ## Logging 71 | ### Principles 72 | - Structured log by [zap](https://github.com/uber-go/zap). 73 | - Use the package `github.com/horaemeta/pkg/log` which is based on [zap](https://github.com/uber-go/zap). 74 | - Create local logger with common fields if necessary. 75 | 76 | ### Example 77 | Normal usage: 78 | ```go 79 | import "github.com/horaemeta/pkg/log" 80 | 81 | func main() { 82 | if err := srv.Run(); err != nil { 83 | log.Error("fail to run server", zap.Error(err)) 84 | return 85 | } 86 | } 87 | ``` 88 | 89 | Local logger: 90 | ```go 91 | import "github.com/horaemeta/pkg/log" 92 | 93 | type lease struct { 94 | ID int64 95 | logger *zap.Logger 96 | } 97 | 98 | func NewLease(ID int64) *lease { 99 | logger := log.With(zap.Int64("lease-id", ID)) 100 | 101 | return &lease { 102 | ID, 103 | logger, 104 | } 105 | } 106 | 107 | func (l *lease) Close() { 108 | l.logger.Info("lease is closed") 109 | l.ID = 0 110 | } 111 | ``` 112 | -------------------------------------------------------------------------------- /server/cluster/metadata/table_manager_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package metadata_test 21 | 22 | import ( 23 | "context" 24 | "path" 25 | "testing" 26 | 27 | "github.com/apache/incubator-horaedb-meta/server/cluster/metadata" 28 | "github.com/apache/incubator-horaedb-meta/server/etcdutil" 29 | "github.com/apache/incubator-horaedb-meta/server/id" 30 | "github.com/apache/incubator-horaedb-meta/server/storage" 31 | "github.com/stretchr/testify/require" 32 | "go.uber.org/zap" 33 | ) 34 | 35 | const ( 36 | TestRootPath = "/testRootPath" 37 | TestClusterID = 0 38 | TestClusterName = "TestClusterName" 39 | TestSchemaIDPrefix = "TestSchemaIDPrefix" 40 | TestTableIDPrefix = "TestTableIDPrefix" 41 | TestIDAllocatorStep = 5 42 | TestSchemaName = "TestSchemaName" 43 | TestTableName = "TestTableName" 44 | ) 45 | 46 | func TestTableManager(t *testing.T) { 47 | ctx := context.Background() 48 | re := require.New(t) 49 | 50 | _, client, _ := etcdutil.PrepareEtcdServerAndClient(t) 51 | clusterStorage := storage.NewStorageWithEtcdBackend(client, TestRootPath, storage.Options{ 52 | MaxScanLimit: 100, MinScanLimit: 10, MaxOpsPerTxn: 10, 53 | }) 54 | 55 | schemaIDAlloc := id.NewAllocatorImpl(zap.NewNop(), client, path.Join(TestRootPath, TestClusterName, TestSchemaIDPrefix), TestIDAllocatorStep) 56 | tableIDAlloc := id.NewAllocatorImpl(zap.NewNop(), client, path.Join(TestRootPath, TestClusterName, TestTableIDPrefix), TestIDAllocatorStep) 57 | tableManager := metadata.NewTableManagerImpl(zap.NewNop(), clusterStorage, storage.ClusterID(TestClusterID), schemaIDAlloc, tableIDAlloc) 58 | err := tableManager.Load(ctx) 59 | re.NoError(err) 60 | 61 | testSchema(ctx, re, tableManager) 62 | testCreateAndDropTable(ctx, re, tableManager) 63 | } 64 | 65 | func testSchema(ctx context.Context, re *require.Assertions, manager metadata.TableManager) { 66 | _, exists := manager.GetSchema(TestSchemaName) 67 | re.False(exists) 68 | schema, exists, err := manager.GetOrCreateSchema(ctx, TestSchemaName) 69 | re.NoError(err) 70 | re.False(exists) 71 | re.Equal(TestSchemaName, schema.Name) 72 | } 73 | 74 | func testCreateAndDropTable(ctx context.Context, re *require.Assertions, manager metadata.TableManager) { 75 | _, exists, err := manager.GetTable(TestSchemaName, TestTableName) 76 | re.NoError(err) 77 | re.False(exists) 78 | 79 | t, err := manager.CreateTable(ctx, TestSchemaName, TestTableName, storage.PartitionInfo{Info: nil}) 80 | re.NoError(err) 81 | re.Equal(TestTableName, t.Name) 82 | 83 | t, exists, err = manager.GetTable(TestSchemaName, TestTableName) 84 | re.NoError(err) 85 | re.True(exists) 86 | re.Equal(TestTableName, t.Name) 87 | 88 | err = manager.DropTable(ctx, TestSchemaName, TestTableName) 89 | re.NoError(err) 90 | 91 | _, exists, err = manager.GetTable(TestSchemaName, TestTableName) 92 | re.NoError(err) 93 | re.False(exists) 94 | } 95 | -------------------------------------------------------------------------------- /server/coordinator/procedure/ddl/createpartitiontable/create_partition_table_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package createpartitiontable_test 21 | 22 | import ( 23 | "context" 24 | "testing" 25 | 26 | "github.com/apache/incubator-horaedb-meta/server/cluster/metadata" 27 | "github.com/apache/incubator-horaedb-meta/server/coordinator" 28 | "github.com/apache/incubator-horaedb-meta/server/coordinator/procedure/ddl/createpartitiontable" 29 | "github.com/apache/incubator-horaedb-meta/server/coordinator/procedure/test" 30 | "github.com/apache/incubator-horaedb-meta/server/storage" 31 | "github.com/apache/incubator-horaedb-proto/golang/pkg/metaservicepb" 32 | "github.com/stretchr/testify/require" 33 | ) 34 | 35 | func TestCreatePartitionTable(t *testing.T) { 36 | re := require.New(t) 37 | ctx := context.Background() 38 | dispatch := test.MockDispatch{} 39 | s := test.NewTestStorage(t) 40 | c := test.InitStableCluster(ctx, t) 41 | 42 | shardNode := c.GetMetadata().GetClusterSnapshot().Topology.ClusterView.ShardNodes[0] 43 | 44 | request := &metaservicepb.CreateTableRequest{ 45 | Header: &metaservicepb.RequestHeader{ 46 | Node: shardNode.NodeName, 47 | ClusterName: test.ClusterName, 48 | }, 49 | PartitionTableInfo: &metaservicepb.PartitionTableInfo{ 50 | SubTableNames: []string{"p1", "p2"}, 51 | }, 52 | SchemaName: test.TestSchemaName, 53 | Name: test.TestTableName0, 54 | } 55 | 56 | shardPicker := coordinator.NewLeastTableShardPicker() 57 | subTableShards, err := shardPicker.PickShards(ctx, c.GetMetadata().GetClusterSnapshot(), len(request.GetPartitionTableInfo().SubTableNames)) 58 | 59 | shardNodesWithVersion := make([]metadata.ShardNodeWithVersion, 0, len(subTableShards)) 60 | for _, subTableShard := range subTableShards { 61 | shardView, exists := c.GetMetadata().GetClusterSnapshot().Topology.ShardViewsMapping[subTableShard.ID] 62 | re.True(exists) 63 | shardNodesWithVersion = append(shardNodesWithVersion, metadata.ShardNodeWithVersion{ 64 | ShardInfo: metadata.ShardInfo{ 65 | ID: shardView.ShardID, 66 | Role: subTableShard.ShardRole, 67 | Version: shardView.Version, 68 | Status: storage.ShardStatusUnknown, 69 | }, 70 | ShardNode: subTableShard, 71 | }) 72 | } 73 | 74 | re.NoError(err) 75 | procedure, err := createpartitiontable.NewProcedure(createpartitiontable.ProcedureParams{ 76 | ID: 0, 77 | ClusterMetadata: c.GetMetadata(), 78 | ClusterSnapshot: c.GetMetadata().GetClusterSnapshot(), 79 | Dispatch: dispatch, 80 | Storage: s, 81 | SourceReq: request, 82 | SubTablesShards: shardNodesWithVersion, 83 | OnSucceeded: func(result metadata.CreateTableResult) error { 84 | return nil 85 | }, 86 | OnFailed: func(err error) error { 87 | return nil 88 | }, 89 | }) 90 | re.NoError(err) 91 | 92 | err = procedure.Start(ctx) 93 | re.NoError(err) 94 | } 95 | -------------------------------------------------------------------------------- /server/coordinator/shard_picker.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package coordinator 21 | 22 | import ( 23 | "context" 24 | "sort" 25 | 26 | "github.com/apache/incubator-horaedb-meta/pkg/assert" 27 | "github.com/apache/incubator-horaedb-meta/server/cluster/metadata" 28 | "github.com/apache/incubator-horaedb-meta/server/storage" 29 | "github.com/pkg/errors" 30 | ) 31 | 32 | // ShardPicker is used to pick up the shards suitable for scheduling in the cluster. 33 | // If expectShardNum bigger than cluster node number, the result depends on enableDuplicateNode: 34 | // TODO: Consider refactor this interface, abstracts the parameters of PickShards as PickStrategy. 35 | type ShardPicker interface { 36 | PickShards(ctx context.Context, snapshot metadata.Snapshot, expectShardNum int) ([]storage.ShardNode, error) 37 | } 38 | 39 | // LeastTableShardPicker selects shards based on the number of tables on the current shard, 40 | // and always selects the shard with the smallest number of current tables. 41 | type leastTableShardPicker struct{} 42 | 43 | func NewLeastTableShardPicker() ShardPicker { 44 | return &leastTableShardPicker{} 45 | } 46 | 47 | func (l leastTableShardPicker) PickShards(_ context.Context, snapshot metadata.Snapshot, expectShardNum int) ([]storage.ShardNode, error) { 48 | if len(snapshot.Topology.ClusterView.ShardNodes) == 0 { 49 | return nil, errors.WithMessage(ErrNodeNumberNotEnough, "no shard is assigned") 50 | } 51 | 52 | shardNodeMapping := make(map[storage.ShardID]storage.ShardNode, len(snapshot.Topology.ShardViewsMapping)) 53 | sortedShardsByTableCount := make([]storage.ShardID, 0, len(snapshot.Topology.ShardViewsMapping)) 54 | for _, shardNode := range snapshot.Topology.ClusterView.ShardNodes { 55 | shardNodeMapping[shardNode.ID] = shardNode 56 | // Only collect the shards witch has been allocated to a node. 57 | sortedShardsByTableCount = append(sortedShardsByTableCount, shardNode.ID) 58 | } 59 | 60 | // Sort shard by table number, 61 | // the shard with the smallest number of tables is at the front of the array. 62 | sort.SliceStable(sortedShardsByTableCount, func(i, j int) bool { 63 | shardView1 := snapshot.Topology.ShardViewsMapping[sortedShardsByTableCount[i]] 64 | shardView2 := snapshot.Topology.ShardViewsMapping[sortedShardsByTableCount[j]] 65 | // When the number of tables is the same, sort according to the size of ShardID. 66 | if len(shardView1.TableIDs) == len(shardView2.TableIDs) { 67 | return shardView1.ShardID < shardView2.ShardID 68 | } 69 | return len(shardView1.TableIDs) < len(shardView2.TableIDs) 70 | }) 71 | 72 | result := make([]storage.ShardNode, 0, expectShardNum) 73 | 74 | for i := 0; i < expectShardNum; i++ { 75 | selectShardID := sortedShardsByTableCount[i%len(sortedShardsByTableCount)] 76 | shardNode, ok := shardNodeMapping[selectShardID] 77 | assert.Assert(ok) 78 | result = append(result, shardNode) 79 | } 80 | 81 | return result, nil 82 | } 83 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Thank you for thinking of contributing! We very much welcome contributions from the community. 4 | To make the process easier and more valuable for everyone involved we have a few rules and guidelines to follow. 5 | 6 | ## Submitting Issues and Feature Requests 7 | 8 | Before you file an [issue](https://github.com/apache/incubator-horaedb-meta/issues/new), please search existing issues in case the same or similar issues have already been filed. 9 | If you find an existing open ticket covering your issue then please avoid adding "👍" or "me too" comments; Github notifications can cause a lot of noise for the project maintainers who triage the back-log. 10 | However, if you have a new piece of information for an existing ticket and you think it may help the investigation or resolution, then please do add it as a comment! 11 | You can signal to the team that you're experiencing an existing issue with one of Github's emoji reactions (these are a good way to add "weight" to an issue from a prioritisation perspective). 12 | 13 | ### Submitting an Issue 14 | 15 | The [New Issue]((https://github.com/apache/incubator-horaedb-meta/issues/new)) page has templates for both bug reports and feature requests. 16 | Please fill one of them out! 17 | The issue templates provide details on what information we will find useful to help us fix an issue. 18 | In short though, the more information you can provide us about your environment and what behaviour you're seeing, the easier we can fix the issue. 19 | If you can push a PR with test cases that trigger a defect or bug, even better! 20 | 21 | As well as bug reports we also welcome feature requests (there is a dedicated issue template for these). 22 | Typically, the maintainers will periodically review community feature requests and make decisions about if we want to add them. 23 | For features we don't plan to support we will close the feature request ticket (so, again, please check closed tickets for feature requests before submitting them). 24 | 25 | ## Contributing Changes 26 | 27 | Please see the [Style Guide](docs/style_guide.md) for more details. 28 | 29 | ### Making a PR 30 | 31 | To open a PR you will need to have a Github account. 32 | Fork the `horaemeta` repo and work on a branch on your fork. 33 | When you have completed your changes, or you want some incremental feedback make a Pull Request to HoraeDB [here](https://github.com/apache/incubator-horaedb-meta/compare). 34 | 35 | If you want to discuss some work in progress then please prefix `[WIP]` to the 36 | PR title. 37 | 38 | For PRs that you consider ready for review, verify the following locally before you submit it: 39 | 40 | * you have a coherent set of logical commits, with messages conforming to the [Conventional Commits](https://github.com/apache/incubator-horaedb-docs/blob/main/docs/src/en/dev/conventional_commit.md) specification; 41 | * all the tests and/or benchmarks pass, including documentation tests; 42 | * the code is correctly formatted and all linter checks pass; and 43 | * you haven't left any "code cruft" (commented out code blocks etc). 44 | 45 | There are some tips on verifying the above in the [next section](#running-tests). 46 | 47 | **After** submitting a PR, you should: 48 | 49 | * verify that all CI status checks pass and the PR is 💚; 50 | * ask for help on the PR if any of the status checks are 🔴, and you don't know why; 51 | * wait patiently for one of the team to review your PR, which could take a few days. 52 | 53 | ## Running Tests 54 | 55 | The rule for running test has been configured in the [Makefile](./Makefile) so just run: 56 | 57 | ```shell 58 | make test 59 | ``` 60 | 61 | ## Running linter check 62 | 63 | CI will check the code formatting and best practices by some specific linters. 64 | 65 | And you can run the check locally by the command: 66 | 67 | ```shell 68 | make check 69 | ``` 70 | -------------------------------------------------------------------------------- /server/coordinator/persist_shard_picker_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package coordinator_test 21 | 22 | import ( 23 | "context" 24 | "testing" 25 | 26 | "github.com/apache/incubator-horaedb-meta/server/cluster/metadata" 27 | "github.com/apache/incubator-horaedb-meta/server/coordinator" 28 | "github.com/apache/incubator-horaedb-meta/server/coordinator/procedure/test" 29 | "github.com/apache/incubator-horaedb-meta/server/storage" 30 | "github.com/stretchr/testify/require" 31 | ) 32 | 33 | func TestPersistShardPicker(t *testing.T) { 34 | re := require.New(t) 35 | ctx := context.Background() 36 | 37 | c := test.InitStableCluster(ctx, t) 38 | 39 | persistShardPicker := coordinator.NewPersistShardPicker(c.GetMetadata(), coordinator.NewLeastTableShardPicker()) 40 | pickResult, err := persistShardPicker.PickShards(ctx, c.GetMetadata().GetClusterSnapshot(), test.TestSchemaName, []string{test.TestTableName0}) 41 | re.NoError(err) 42 | re.Equal(len(pickResult), 1) 43 | 44 | createResult, err := c.GetMetadata().CreateTable(ctx, metadata.CreateTableRequest{ 45 | ShardID: pickResult[test.TestTableName0].ID, 46 | LatestVersion: 0, 47 | SchemaName: test.TestSchemaName, 48 | TableName: test.TestTableName0, 49 | PartitionInfo: storage.PartitionInfo{Info: nil}, 50 | }) 51 | re.NoError(err) 52 | re.Equal(test.TestTableName0, createResult.Table.Name) 53 | 54 | // Try to pick shard for same table after the table is created. 55 | newPickResult, err := persistShardPicker.PickShards(ctx, c.GetMetadata().GetClusterSnapshot(), test.TestSchemaName, []string{test.TestTableName0}) 56 | re.NoError(err) 57 | re.Equal(len(newPickResult), 1) 58 | re.Equal(newPickResult[test.TestTableName0], pickResult[test.TestTableName0]) 59 | 60 | // Try to pick shard for another table. 61 | pickResult, err = persistShardPicker.PickShards(ctx, c.GetMetadata().GetClusterSnapshot(), test.TestSchemaName, []string{test.TestTableName1}) 62 | re.NoError(err) 63 | re.Equal(len(pickResult), 1) 64 | 65 | err = c.GetMetadata().DropTable(ctx, metadata.DropTableRequest{ 66 | SchemaName: test.TestSchemaName, 67 | TableName: test.TestTableName0, 68 | ShardID: pickResult[test.TestTableName0].ID, 69 | LatestVersion: 0, 70 | }) 71 | re.NoError(err) 72 | 73 | // Try to pick shard for table1 after drop table0. 74 | newPickResult, err = persistShardPicker.PickShards(ctx, c.GetMetadata().GetClusterSnapshot(), test.TestSchemaName, []string{test.TestTableName1}) 75 | re.NoError(err) 76 | re.Equal(len(pickResult), 1) 77 | re.Equal(newPickResult[test.TestTableName1], pickResult[test.TestTableName1]) 78 | 79 | err = c.GetMetadata().DeleteTableAssignedShard(ctx, test.TestSchemaName, test.TestTableName1) 80 | re.NoError(err) 81 | 82 | // Try to pick another for table1 after drop table1 assign result. 83 | newPickResult, err = persistShardPicker.PickShards(ctx, c.GetMetadata().GetClusterSnapshot(), test.TestSchemaName, []string{test.TestTableName1}) 84 | re.NoError(err) 85 | re.Equal(len(pickResult), 1) 86 | re.NotEqual(newPickResult[test.TestTableName1], pickResult[test.TestTableName1]) 87 | } 88 | -------------------------------------------------------------------------------- /server/coordinator/procedure/storage_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package procedure 21 | 22 | import ( 23 | "context" 24 | "testing" 25 | "time" 26 | 27 | "github.com/apache/incubator-horaedb-meta/server/etcdutil" 28 | "github.com/stretchr/testify/require" 29 | ) 30 | 31 | const ( 32 | TestClusterID = 1 33 | DefaultTimeout = time.Second * 10 34 | DefaultScanBatchSie = 100 35 | TestRootPath = "/rootPath" 36 | ) 37 | 38 | func testWrite(t *testing.T, storage Storage) { 39 | re := require.New(t) 40 | ctx, cancel := context.WithTimeout(context.Background(), DefaultTimeout) 41 | defer cancel() 42 | 43 | testMeta1 := Meta{ 44 | ID: uint64(1), 45 | Kind: TransferLeader, 46 | State: StateInit, 47 | RawData: []byte("test"), 48 | } 49 | 50 | // Test create new procedure 51 | err := storage.CreateOrUpdate(ctx, testMeta1) 52 | re.NoError(err) 53 | 54 | testMeta2 := Meta{ 55 | ID: uint64(2), 56 | Kind: TransferLeader, 57 | State: StateInit, 58 | RawData: []byte("test"), 59 | } 60 | err = storage.CreateOrUpdate(ctx, testMeta2) 61 | re.NoError(err) 62 | 63 | // Test update procedure 64 | testMeta2.RawData = []byte("test update") 65 | err = storage.CreateOrUpdate(ctx, testMeta2) 66 | re.NoError(err) 67 | } 68 | 69 | func testScan(t *testing.T, storage Storage) { 70 | re := require.New(t) 71 | ctx, cancel := context.WithTimeout(context.Background(), DefaultTimeout) 72 | defer cancel() 73 | 74 | metas, err := storage.List(ctx, TransferLeader, DefaultScanBatchSie) 75 | re.NoError(err) 76 | re.Equal(2, len(metas)) 77 | re.Equal("test", string(metas[0].RawData)) 78 | re.Equal("test update", string(metas[1].RawData)) 79 | } 80 | 81 | func testDelete(t *testing.T, storage Storage) { 82 | re := require.New(t) 83 | ctx, cancel := context.WithTimeout(context.Background(), DefaultTimeout) 84 | defer cancel() 85 | 86 | testMeta1 := &Meta{ 87 | ID: uint64(1), 88 | Kind: TransferLeader, 89 | State: StateInit, 90 | RawData: []byte("test"), 91 | } 92 | err := storage.MarkDeleted(ctx, TransferLeader, testMeta1.ID) 93 | re.NoError(err) 94 | 95 | metas, err := storage.List(ctx, TransferLeader, DefaultScanBatchSie) 96 | re.NoError(err) 97 | re.Equal(1, len(metas)) 98 | 99 | testMeta2 := Meta{ 100 | ID: uint64(2), 101 | Kind: TransferLeader, 102 | State: StateInit, 103 | RawData: []byte("test"), 104 | } 105 | err = storage.Delete(ctx, TransferLeader, testMeta2.ID) 106 | re.NoError(err) 107 | 108 | metas, err = storage.List(ctx, TransferLeader, DefaultScanBatchSie) 109 | re.NoError(err) 110 | re.Equal(0, len(metas)) 111 | } 112 | 113 | func NewTestStorage(t *testing.T) Storage { 114 | _, client, _ := etcdutil.PrepareEtcdServerAndClient(t) 115 | storage := NewEtcdStorageImpl(client, TestRootPath, TestClusterID) 116 | return storage 117 | } 118 | 119 | func TestStorage(t *testing.T) { 120 | storage := NewTestStorage(t) 121 | testWrite(t, storage) 122 | testScan(t, storage) 123 | testDelete(t, storage) 124 | } 125 | -------------------------------------------------------------------------------- /server/coordinator/procedure/operation/split/split_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package split_test 21 | 22 | import ( 23 | "context" 24 | "testing" 25 | 26 | "github.com/apache/incubator-horaedb-meta/server/cluster/metadata" 27 | "github.com/apache/incubator-horaedb-meta/server/coordinator/procedure/operation/split" 28 | "github.com/apache/incubator-horaedb-meta/server/coordinator/procedure/test" 29 | "github.com/apache/incubator-horaedb-meta/server/storage" 30 | "github.com/stretchr/testify/require" 31 | ) 32 | 33 | func TestSplit(t *testing.T) { 34 | re := require.New(t) 35 | ctx := context.Background() 36 | dispatch := test.MockDispatch{} 37 | c := test.InitStableCluster(ctx, t) 38 | s := test.NewTestStorage(t) 39 | 40 | snapshot := c.GetMetadata().GetClusterSnapshot() 41 | 42 | // Randomly select a shardNode to split. 43 | createTableNodeShard := snapshot.Topology.ClusterView.ShardNodes[0] 44 | 45 | // Create some tables in this shard. 46 | _, err := c.GetMetadata().CreateTable(ctx, metadata.CreateTableRequest{ 47 | ShardID: createTableNodeShard.ID, 48 | LatestVersion: 0, 49 | SchemaName: test.TestSchemaName, 50 | TableName: test.TestTableName0, 51 | PartitionInfo: storage.PartitionInfo{Info: nil}, 52 | }) 53 | re.NoError(err) 54 | _, err = c.GetMetadata().CreateTable(ctx, metadata.CreateTableRequest{ 55 | ShardID: createTableNodeShard.ID, 56 | LatestVersion: 0, 57 | SchemaName: test.TestSchemaName, 58 | TableName: test.TestTableName1, 59 | PartitionInfo: storage.PartitionInfo{Info: nil}, 60 | }) 61 | re.NoError(err) 62 | 63 | // Split one table from this shard. 64 | targetShardNode := c.GetMetadata().GetClusterSnapshot().Topology.ClusterView.ShardNodes[0] 65 | newShardID, err := c.GetMetadata().AllocShardID(ctx) 66 | re.NoError(err) 67 | p, err := split.NewProcedure(split.ProcedureParams{ 68 | ID: 0, 69 | Dispatch: dispatch, 70 | Storage: s, 71 | ClusterMetadata: c.GetMetadata(), 72 | ClusterSnapshot: c.GetMetadata().GetClusterSnapshot(), 73 | ShardID: createTableNodeShard.ID, 74 | NewShardID: storage.ShardID(newShardID), 75 | SchemaName: test.TestSchemaName, 76 | TableNames: []string{test.TestTableName0}, 77 | TargetNodeName: createTableNodeShard.NodeName, 78 | }) 79 | re.NoError(err) 80 | err = p.Start(ctx) 81 | re.NoError(err) 82 | 83 | // Validate split result: 84 | // 1. Shards on node, split shard and new shard must be all exists. 85 | // 2. Tables mapping of split shard and new shard must be all exists. 86 | // 3. Tables in table mapping must be correct, the split table only exists on the new shard. 87 | snapshot = c.GetMetadata().GetClusterSnapshot() 88 | 89 | splitShard, exists := snapshot.Topology.ShardViewsMapping[targetShardNode.ID] 90 | re.True(exists) 91 | newShard, exists := snapshot.Topology.ShardViewsMapping[storage.ShardID(newShardID)] 92 | re.True(exists) 93 | re.NotNil(splitShard) 94 | re.NotNil(newShard) 95 | 96 | splitShardTables := splitShard.TableIDs 97 | newShardTables := newShard.TableIDs 98 | re.NotNil(splitShardTables) 99 | re.NotNil(newShardTables) 100 | } 101 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/apache/incubator-horaedb-meta 2 | 3 | go 1.21 4 | 5 | require ( 6 | github.com/apache/incubator-horaedb-proto/golang v0.0.0-20231228071726-92152841fc8a 7 | github.com/caarlos0/env/v6 v6.10.1 8 | github.com/julienschmidt/httprouter v1.3.0 9 | github.com/looplab/fsm v0.3.0 10 | github.com/pelletier/go-toml/v2 v2.0.6 11 | github.com/pkg/errors v0.9.1 12 | github.com/spaolacci/murmur3 v1.1.0 13 | github.com/stretchr/testify v1.8.1 14 | github.com/tikv/pd v2.1.19+incompatible 15 | go.etcd.io/etcd/api/v3 v3.5.4 16 | go.etcd.io/etcd/client/pkg/v3 v3.5.4 17 | go.etcd.io/etcd/client/v3 v3.5.4 18 | go.etcd.io/etcd/server/v3 v3.5.4 19 | go.uber.org/zap v1.21.0 20 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c 21 | golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba 22 | google.golang.org/grpc v1.47.0 23 | google.golang.org/protobuf v1.28.0 24 | ) 25 | 26 | require ( 27 | github.com/BurntSushi/toml v1.1.0 // indirect 28 | github.com/benbjohnson/clock v1.1.0 // indirect 29 | github.com/beorn7/perks v1.0.1 // indirect 30 | github.com/cespare/xxhash/v2 v2.1.1 // indirect 31 | github.com/coreos/go-semver v0.3.0 // indirect 32 | github.com/coreos/go-systemd/v22 v22.3.2 // indirect 33 | github.com/davecgh/go-spew v1.1.1 // indirect 34 | github.com/dustin/go-humanize v1.0.0 // indirect 35 | github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect 36 | github.com/gogo/protobuf v1.3.2 // indirect 37 | github.com/golang/protobuf v1.5.2 // indirect 38 | github.com/google/btree v1.0.1 // indirect 39 | github.com/google/go-cmp v0.5.8 // indirect 40 | github.com/gorilla/websocket v1.4.2 // indirect 41 | github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect 42 | github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect 43 | github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect 44 | github.com/jonboulle/clockwork v0.2.2 // indirect 45 | github.com/json-iterator/go v1.1.11 // indirect 46 | github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect 47 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 48 | github.com/modern-go/reflect2 v1.0.1 // indirect 49 | github.com/pingcap/log v1.1.0 // indirect 50 | github.com/pmezard/go-difflib v1.0.0 // indirect 51 | github.com/prometheus/client_golang v1.11.1 // indirect 52 | github.com/prometheus/client_model v0.2.0 // indirect 53 | github.com/prometheus/common v0.26.0 // indirect 54 | github.com/prometheus/procfs v0.6.0 // indirect 55 | github.com/sirupsen/logrus v1.7.0 // indirect 56 | github.com/soheilhy/cmux v0.1.5 // indirect 57 | github.com/spf13/pflag v1.0.5 // indirect 58 | github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect 59 | github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect 60 | go.etcd.io/bbolt v1.3.6 // indirect 61 | go.etcd.io/etcd/client/v2 v2.305.4 // indirect 62 | go.etcd.io/etcd/pkg/v3 v3.5.4 // indirect 63 | go.etcd.io/etcd/raft/v3 v3.5.4 // indirect 64 | go.opentelemetry.io/contrib v0.20.0 // indirect 65 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 // indirect 66 | go.opentelemetry.io/otel v0.20.0 // indirect 67 | go.opentelemetry.io/otel/exporters/otlp v0.20.0 // indirect 68 | go.opentelemetry.io/otel/metric v0.20.0 // indirect 69 | go.opentelemetry.io/otel/sdk v0.20.0 // indirect 70 | go.opentelemetry.io/otel/sdk/export/metric v0.20.0 // indirect 71 | go.opentelemetry.io/otel/sdk/metric v0.20.0 // indirect 72 | go.opentelemetry.io/otel/trace v0.20.0 // indirect 73 | go.opentelemetry.io/proto/otlp v0.7.0 // indirect 74 | go.uber.org/atomic v1.9.0 // indirect 75 | go.uber.org/multierr v1.7.0 // indirect 76 | golang.org/x/crypto v0.14.0 // indirect 77 | golang.org/x/net v0.16.0 // indirect 78 | golang.org/x/sys v0.13.0 // indirect 79 | golang.org/x/text v0.13.0 // indirect 80 | google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect 81 | gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect 82 | gopkg.in/yaml.v2 v2.4.0 // indirect 83 | gopkg.in/yaml.v3 v3.0.1 // indirect 84 | sigs.k8s.io/yaml v1.2.0 // indirect 85 | ) 86 | -------------------------------------------------------------------------------- /cmd/horaemeta-server/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package main 21 | 22 | import ( 23 | "context" 24 | "fmt" 25 | "os" 26 | "os/signal" 27 | "syscall" 28 | 29 | "github.com/apache/incubator-horaedb-meta/pkg/coderr" 30 | "github.com/apache/incubator-horaedb-meta/pkg/log" 31 | "github.com/apache/incubator-horaedb-meta/server" 32 | "github.com/apache/incubator-horaedb-meta/server/config" 33 | "github.com/pelletier/go-toml/v2" 34 | "go.uber.org/zap" 35 | ) 36 | 37 | var ( 38 | buildDate string 39 | branchName string 40 | commitID string 41 | ) 42 | 43 | func buildVersion() string { 44 | return fmt.Sprintf("HoraeMeta Server\nGit commit:%s\nGit branch:%s\nBuild date:%s", commitID, branchName, buildDate) 45 | } 46 | 47 | func panicf(format string, args ...any) { 48 | msg := fmt.Sprintf(format, args...) 49 | panic(msg) 50 | } 51 | 52 | func main() { 53 | cfgParser, err := config.MakeConfigParser() 54 | if err != nil { 55 | panicf("fail to generate config builder, err:%v", err) 56 | } 57 | 58 | cfg, err := cfgParser.Parse(os.Args[1:]) 59 | if coderr.Is(err, coderr.PrintHelpUsage) { 60 | return 61 | } 62 | 63 | if err != nil { 64 | panicf("fail to parse config from command line params, err:%v", err) 65 | } 66 | 67 | if cfgParser.NeedPrintVersion() { 68 | println(buildVersion()) 69 | return 70 | } 71 | 72 | if err := cfg.ValidateAndAdjust(); err != nil { 73 | panicf("invalid config, err:%v", err) 74 | } 75 | 76 | if err := cfgParser.ParseConfigFromToml(); err != nil { 77 | panicf("fail to parse config from toml file, err:%v", err) 78 | } 79 | 80 | if err := cfgParser.ParseConfigFromEnv(); err != nil { 81 | panicf("fail to parse config from environment variable, err:%v", err) 82 | } 83 | 84 | cfgByte, err := toml.Marshal(cfg) 85 | if err != nil { 86 | panicf("fail to marshal server config, err:%v", err) 87 | } 88 | 89 | if err = os.MkdirAll(cfg.DataDir, os.ModePerm); err != nil { 90 | panicf("fail to create data dir, data_dir:%v, err:%v", cfg.DataDir, err) 91 | } 92 | 93 | logger, err := log.InitGlobalLogger(&cfg.Log) 94 | if err != nil { 95 | panicf("fail to init global logger, err:%v", err) 96 | } 97 | defer logger.Sync() //nolint:errcheck 98 | log.Info(fmt.Sprintf("server start with version: %s", buildVersion())) 99 | // TODO: Do adjustment to config for preparing joining existing cluster. 100 | log.Info("server start with config", zap.String("config", string(cfgByte))) 101 | 102 | srv, err := server.CreateServer(cfg) 103 | if err != nil { 104 | log.Error("fail to create server", zap.Error(err)) 105 | return 106 | } 107 | 108 | ctx, cancel := context.WithCancel(context.Background()) 109 | defer cancel() 110 | sc := make(chan os.Signal, 1) 111 | signal.Notify(sc, 112 | syscall.SIGHUP, 113 | syscall.SIGINT, 114 | syscall.SIGTERM, 115 | syscall.SIGQUIT) 116 | 117 | var sig os.Signal 118 | go func() { 119 | sig = <-sc 120 | cancel() 121 | }() 122 | 123 | if err := srv.Run(ctx); err != nil { 124 | log.Error("fail to run server", zap.Error(err)) 125 | return 126 | } 127 | 128 | <-ctx.Done() 129 | log.Info("got signal to exit", zap.Any("signal", sig)) 130 | 131 | srv.Close() 132 | } 133 | -------------------------------------------------------------------------------- /server/storage/meta.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package storage 21 | 22 | import ( 23 | "context" 24 | 25 | clientv3 "go.etcd.io/etcd/client/v3" 26 | ) 27 | 28 | // Storage defines the storage operations on the HoraeDB cluster meta info. 29 | type Storage interface { 30 | // GetCluster get cluster metadata by clusterID. 31 | GetCluster(ctx context.Context, clusterID ClusterID) (Cluster, error) 32 | // ListClusters list all clusters. 33 | ListClusters(ctx context.Context) (ListClustersResult, error) 34 | // CreateCluster create new cluster, return error if cluster already exists. 35 | CreateCluster(ctx context.Context, req CreateClusterRequest) error 36 | // UpdateCluster update cluster metadata. 37 | UpdateCluster(ctx context.Context, req UpdateClusterRequest) error 38 | 39 | // CreateClusterView create cluster view. 40 | CreateClusterView(ctx context.Context, req CreateClusterViewRequest) error 41 | // GetClusterView get cluster view by cluster id. 42 | GetClusterView(ctx context.Context, req GetClusterViewRequest) (GetClusterViewResult, error) 43 | // UpdateClusterView update cluster view. 44 | UpdateClusterView(ctx context.Context, req UpdateClusterViewRequest) error 45 | 46 | // ListSchemas list all schemas in specified cluster. 47 | ListSchemas(ctx context.Context, req ListSchemasRequest) (ListSchemasResult, error) 48 | // CreateSchema create schema in specified cluster. 49 | CreateSchema(ctx context.Context, req CreateSchemaRequest) error 50 | 51 | // CreateTable create new table in specified cluster and schema, return error if table already exists. 52 | CreateTable(ctx context.Context, req CreateTableRequest) error 53 | // GetTable get table by table name in specified cluster and schema. 54 | GetTable(ctx context.Context, req GetTableRequest) (GetTableResult, error) 55 | // ListTables list all tables in specified cluster and schema. 56 | ListTables(ctx context.Context, req ListTableRequest) (ListTablesResult, error) 57 | // DeleteTable delete table by table name in specified cluster and schema. 58 | DeleteTable(ctx context.Context, req DeleteTableRequest) error 59 | 60 | // AssignTableToShard save table assign result. 61 | AssignTableToShard(ctx context.Context, req AssignTableToShardRequest) error 62 | // DeleteTableAssignedShard delete table assign result. 63 | DeleteTableAssignedShard(ctx context.Context, req DeleteTableAssignedRequest) error 64 | // ListTableAssignedShard list table assign result. 65 | ListTableAssignedShard(ctx context.Context, req ListAssignTableRequest) (ListTableAssignedShardResult, error) 66 | 67 | // CreateShardViews create shard views in specified cluster. 68 | CreateShardViews(ctx context.Context, req CreateShardViewsRequest) error 69 | // ListShardViews list all shard views in specified cluster. 70 | ListShardViews(ctx context.Context, req ListShardViewsRequest) (ListShardViewsResult, error) 71 | // UpdateShardView update shard views in specified cluster. 72 | UpdateShardView(ctx context.Context, req UpdateShardViewRequest) error 73 | 74 | // ListNodes list all nodes in specified cluster. 75 | ListNodes(ctx context.Context, req ListNodesRequest) (ListNodesResult, error) 76 | // CreateOrUpdateNode create or update node in specified cluster. 77 | CreateOrUpdateNode(ctx context.Context, req CreateOrUpdateNodeRequest) error 78 | } 79 | 80 | // NewStorageWithEtcdBackend creates a new storage with etcd backend. 81 | func NewStorageWithEtcdBackend(client *clientv3.Client, rootPath string, opts Options) Storage { 82 | return newEtcdStorage(client, rootPath, opts) 83 | } 84 | -------------------------------------------------------------------------------- /server/etcdutil/util_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package etcdutil 21 | 22 | import ( 23 | "context" 24 | "fmt" 25 | "testing" 26 | 27 | "github.com/stretchr/testify/require" 28 | ) 29 | 30 | func makeTestKeys(num int) []string { 31 | keys := make([]string, 0, num) 32 | for idx := 0; idx < num; idx++ { 33 | keys = append(keys, fmt.Sprintf("%010d", idx)) 34 | } 35 | 36 | return keys 37 | } 38 | 39 | // Put some keys and scan all of them successfully. 40 | func TestScanNormal(t *testing.T) { 41 | r := require.New(t) 42 | 43 | _, client, closeSrv := PrepareEtcdServerAndClient(t) 44 | defer closeSrv() 45 | 46 | keys := makeTestKeys(51) 47 | lastKey := keys[len(keys)-1] 48 | keys = keys[0 : len(keys)-1] 49 | ctx := context.Background() 50 | 51 | // Put the keys. 52 | for _, key := range keys { 53 | // Let the value equal key for simplicity. 54 | val := key 55 | _, err := client.Put(ctx, key, val) 56 | r.NoError(err) 57 | } 58 | 59 | // Scan the keys with different batch size. 60 | batchSizes := []int{1, 10, 12, 30, 50, 90} 61 | startKey, endKey := keys[0], lastKey 62 | collectedKeys := make([]string, 0, len(keys)) 63 | for _, batchSz := range batchSizes { 64 | collectedKeys = collectedKeys[:0] 65 | 66 | do := func(key string, value []byte) error { 67 | r.Equal(key, string(value)) 68 | 69 | collectedKeys = append(collectedKeys, key) 70 | return nil 71 | } 72 | err := Scan(ctx, client, startKey, endKey, batchSz, do) 73 | r.NoError(err) 74 | 75 | r.Equal(collectedKeys, keys) 76 | } 77 | } 78 | 79 | // Test the cases where scan fails. 80 | func TestScanFailed(t *testing.T) { 81 | r := require.New(t) 82 | 83 | _, client, closeSrv := PrepareEtcdServerAndClient(t) 84 | defer closeSrv() 85 | 86 | keys := makeTestKeys(50) 87 | ctx := context.Background() 88 | 89 | // Put the keys. 90 | for _, key := range keys { 91 | // Let the value equal key for simplicity. 92 | val := key 93 | _, err := client.Put(ctx, key, val) 94 | r.NoError(err) 95 | } 96 | 97 | fakeErr := fmt.Errorf("fake error for mock failed scan") 98 | do := func(key string, value []byte) error { 99 | if key > keys[len(keys)/2] { 100 | return fakeErr 101 | } 102 | return nil 103 | } 104 | startKey, endKey := keys[0], keys[len(keys)-1] 105 | err := Scan(ctx, client, startKey, endKey, 10, do) 106 | r.Equal(fakeErr, err) 107 | } 108 | 109 | func TestScanWithPrefix(t *testing.T) { 110 | r := require.New(t) 111 | 112 | _, client, closeSrv := PrepareEtcdServerAndClient(t) 113 | defer closeSrv() 114 | ctx := context.Background() 115 | 116 | // Build keys with different prefix. 117 | keys := []string{} 118 | keys = append(keys, "/prefix/0") 119 | keys = append(keys, "/prefix/1") 120 | keys = append(keys, "/diff/0") 121 | 122 | // Put the keys. 123 | for _, key := range keys { 124 | // Let the value equal key for simplicity. 125 | val := key 126 | _, err := client.Put(ctx, key, val) 127 | r.NoError(err) 128 | } 129 | 130 | var scanResult []string 131 | do := func(key string, value []byte) error { 132 | scanResult = append(scanResult, key) 133 | return nil 134 | } 135 | err := ScanWithPrefix(ctx, client, "/prefix", do) 136 | r.NoError(err) 137 | r.Equal(len(scanResult), 2) 138 | } 139 | 140 | func TestGetLastPathSegment(t *testing.T) { 141 | r := require.New(t) 142 | 143 | path := "/prefix/a/b/c" 144 | lastPathSegment := GetLastPathSegment(path) 145 | r.Equal("c", lastPathSegment) 146 | } 147 | -------------------------------------------------------------------------------- /server/service/http/forward.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package http 21 | 22 | import ( 23 | "context" 24 | "net" 25 | "net/http" 26 | "net/url" 27 | "strconv" 28 | "strings" 29 | "time" 30 | 31 | "github.com/apache/incubator-horaedb-meta/pkg/log" 32 | "github.com/apache/incubator-horaedb-meta/server/member" 33 | "github.com/apache/incubator-horaedb-meta/server/service" 34 | "github.com/pkg/errors" 35 | "go.uber.org/zap" 36 | ) 37 | 38 | type ForwardClient struct { 39 | member *member.Member 40 | client *http.Client 41 | port int 42 | } 43 | 44 | func NewForwardClient(member *member.Member, port int) *ForwardClient { 45 | return &ForwardClient{ 46 | member: member, 47 | client: getForwardedHTTPClient(), 48 | port: port, 49 | } 50 | } 51 | 52 | func (s *ForwardClient) GetLeaderAddr(ctx context.Context) (string, error) { 53 | resp, err := s.member.GetLeaderAddr(ctx) 54 | if err != nil { 55 | return "", err 56 | } 57 | 58 | return resp.LeaderEndpoint, nil 59 | } 60 | 61 | func (s *ForwardClient) getForwardedAddr(ctx context.Context) (string, bool, error) { 62 | resp, err := s.member.GetLeaderAddr(ctx) 63 | if err != nil { 64 | return "", false, errors.WithMessage(err, "get forwarded addr") 65 | } 66 | if resp.IsLocal { 67 | return "", true, nil 68 | } 69 | // TODO: In the current implementation, if the HTTP port of each node of HoraeMeta is inconsistent, the forwarding address will be wrong 70 | httpAddr, err := formatHTTPAddr(resp.LeaderEndpoint, s.port) 71 | if err != nil { 72 | return "", false, errors.WithMessage(err, "format http addr") 73 | } 74 | log.Info("getForwardedAddr", zap.String("leaderAddr", httpAddr), zap.Int("port", s.port)) 75 | return httpAddr, false, nil 76 | } 77 | 78 | func (s *ForwardClient) forwardToLeader(req *http.Request) (*http.Response, bool, error) { 79 | addr, isLeader, err := s.getForwardedAddr(req.Context()) 80 | if err != nil { 81 | log.Error("get forward addr failed", zap.Error(err)) 82 | return nil, false, err 83 | } 84 | if isLeader { 85 | return nil, true, nil 86 | } 87 | 88 | // Update remote host 89 | req.RequestURI = "" 90 | if req.TLS == nil { 91 | req.URL.Scheme = "http" 92 | } else { 93 | req.URL.Scheme = "https" 94 | } 95 | req.URL.Host = addr 96 | 97 | resp, err := s.client.Do(req) 98 | if err != nil { 99 | log.Error("forward client send request failed", zap.Error(err)) 100 | return nil, false, err 101 | } 102 | 103 | return resp, false, nil 104 | } 105 | 106 | func getForwardedHTTPClient() *http.Client { 107 | return &http.Client{ 108 | Transport: &http.Transport{ 109 | Proxy: http.ProxyFromEnvironment, 110 | DialContext: (&net.Dialer{ 111 | Timeout: 30 * time.Second, 112 | Deadline: time.Time{}, 113 | KeepAlive: 30 * time.Second, 114 | }).DialContext, 115 | TLSHandshakeTimeout: 10 * time.Second, 116 | }, 117 | } 118 | } 119 | 120 | // formatHttpAddr convert grpcAddr(http://127.0.0.1:8831) httpPort(5000) to httpAddr(127.0.0.1:5000). 121 | func formatHTTPAddr(grpcAddr string, httpPort int) (string, error) { 122 | url, err := url.Parse(grpcAddr) 123 | if err != nil { 124 | return "", service.ErrParseURL.WithCause(err) 125 | } 126 | hostAndPort := strings.Split(url.Host, ":") 127 | if len(hostAndPort) != 2 { 128 | return "", errors.WithMessagef(ErrParseLeaderAddr, "parse leader addr, grpcAdd:%s", grpcAddr) 129 | } 130 | hostAndPort[1] = strconv.Itoa(httpPort) 131 | httpAddr := strings.Join(hostAndPort, ":") 132 | return httpAddr, nil 133 | } 134 | -------------------------------------------------------------------------------- /server/coordinator/procedure/delay_queue.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package procedure 21 | 22 | import ( 23 | "container/heap" 24 | "fmt" 25 | "sync" 26 | "time" 27 | 28 | "github.com/pkg/errors" 29 | ) 30 | 31 | type procedureScheduleEntry struct { 32 | procedure Procedure 33 | runAfter time.Time 34 | } 35 | 36 | type DelayQueue struct { 37 | maxLen int 38 | 39 | // This lock is used to protect the following fields. 40 | lock sync.RWMutex 41 | heapQueue *heapPriorityQueue 42 | // existingProcs is used to record procedures has been pushed into the queue, 43 | // and they will be used to verify the addition of duplicate elements. 44 | existingProcs map[uint64]struct{} 45 | } 46 | 47 | // heapPriorityQueue is no internal lock, 48 | // and its thread safety is guaranteed by the external caller. 49 | type heapPriorityQueue struct { 50 | procedures []*procedureScheduleEntry 51 | } 52 | 53 | func (q *heapPriorityQueue) Len() int { 54 | return len(q.procedures) 55 | } 56 | 57 | // The dequeue order of elements is determined by the less method. 58 | // When return procedures[i].runAfter < procedures[j].runAfter, the element with smallest will be pop first. 59 | func (q *heapPriorityQueue) Less(i, j int) bool { 60 | return q.procedures[i].runAfter.Before(q.procedures[j].runAfter) 61 | } 62 | 63 | func (q *heapPriorityQueue) Swap(i, j int) { 64 | q.procedures[i], q.procedures[j] = q.procedures[j], q.procedures[i] 65 | } 66 | 67 | func (q *heapPriorityQueue) Push(x any) { 68 | item := x.(*procedureScheduleEntry) 69 | q.procedures = append(q.procedures, item) 70 | } 71 | 72 | func (q *heapPriorityQueue) Pop() any { 73 | length := len(q.procedures) 74 | if length == 0 { 75 | return nil 76 | } 77 | item := q.procedures[length-1] 78 | q.procedures = q.procedures[:length-1] 79 | return item 80 | } 81 | 82 | func (q *heapPriorityQueue) Peek() any { 83 | length := len(q.procedures) 84 | if length == 0 { 85 | return nil 86 | } 87 | item := q.procedures[0] 88 | return item 89 | } 90 | 91 | func NewProcedureDelayQueue(maxLen int) *DelayQueue { 92 | return &DelayQueue{ 93 | maxLen: maxLen, 94 | 95 | lock: sync.RWMutex{}, 96 | heapQueue: &heapPriorityQueue{procedures: []*procedureScheduleEntry{}}, 97 | existingProcs: map[uint64]struct{}{}, 98 | } 99 | } 100 | 101 | func (q *DelayQueue) Len() int { 102 | q.lock.RLock() 103 | defer q.lock.RUnlock() 104 | 105 | return q.heapQueue.Len() 106 | } 107 | 108 | func (q *DelayQueue) Push(p Procedure, delay time.Duration) error { 109 | q.lock.Lock() 110 | defer q.lock.Unlock() 111 | 112 | if q.heapQueue.Len() >= q.maxLen { 113 | return errors.WithMessage(ErrQueueFull, fmt.Sprintf("queue max length is %d", q.maxLen)) 114 | } 115 | 116 | if _, exists := q.existingProcs[p.ID()]; exists { 117 | return errors.WithMessage(ErrPushDuplicatedProcedure, fmt.Sprintf("procedure has been pushed, %v", p)) 118 | } 119 | 120 | heap.Push(q.heapQueue, &procedureScheduleEntry{ 121 | procedure: p, 122 | runAfter: time.Now().Add(delay), 123 | }) 124 | q.existingProcs[p.ID()] = struct{}{} 125 | 126 | return nil 127 | } 128 | 129 | func (q *DelayQueue) Pop() Procedure { 130 | q.lock.Lock() 131 | defer q.lock.Unlock() 132 | 133 | if q.heapQueue.Len() == 0 { 134 | return nil 135 | } 136 | 137 | entry := q.heapQueue.Peek().(*procedureScheduleEntry) 138 | if time.Now().Before(entry.runAfter) { 139 | return nil 140 | } 141 | 142 | heap.Pop(q.heapQueue) 143 | delete(q.existingProcs, entry.procedure.ID()) 144 | 145 | return entry.procedure 146 | } 147 | -------------------------------------------------------------------------------- /server/cluster/cluster.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package cluster 21 | 22 | import ( 23 | "context" 24 | "strings" 25 | 26 | "github.com/apache/incubator-horaedb-meta/server/cluster/metadata" 27 | "github.com/apache/incubator-horaedb-meta/server/coordinator" 28 | "github.com/apache/incubator-horaedb-meta/server/coordinator/eventdispatch" 29 | "github.com/apache/incubator-horaedb-meta/server/coordinator/procedure" 30 | "github.com/apache/incubator-horaedb-meta/server/coordinator/scheduler/manager" 31 | "github.com/apache/incubator-horaedb-meta/server/id" 32 | "github.com/apache/incubator-horaedb-meta/server/storage" 33 | "github.com/pkg/errors" 34 | clientv3 "go.etcd.io/etcd/client/v3" 35 | "go.uber.org/zap" 36 | ) 37 | 38 | const ( 39 | defaultProcedurePrefixKey = "ProcedureID" 40 | defaultAllocStep = 50 41 | ) 42 | 43 | type Cluster struct { 44 | logger *zap.Logger 45 | metadata *metadata.ClusterMetadata 46 | 47 | procedureFactory *coordinator.Factory 48 | procedureManager procedure.Manager 49 | schedulerManager manager.SchedulerManager 50 | } 51 | 52 | func NewCluster(logger *zap.Logger, metadata *metadata.ClusterMetadata, client *clientv3.Client, rootPath string) (*Cluster, error) { 53 | procedureStorage := procedure.NewEtcdStorageImpl(client, rootPath, uint32(metadata.GetClusterID())) 54 | procedureManager, err := procedure.NewManagerImpl(logger, metadata) 55 | if err != nil { 56 | return nil, errors.WithMessage(err, "create procedure manager") 57 | } 58 | dispatch := eventdispatch.NewDispatchImpl() 59 | 60 | procedureIDRootPath := strings.Join([]string{rootPath, metadata.Name(), defaultProcedurePrefixKey}, "/") 61 | procedureFactory := coordinator.NewFactory(logger, id.NewAllocatorImpl(logger, client, procedureIDRootPath, defaultAllocStep), dispatch, procedureStorage, metadata) 62 | 63 | schedulerManager := manager.NewManager(logger, procedureManager, procedureFactory, metadata, client, rootPath, metadata.GetTopologyType(), metadata.GetProcedureExecutingBatchSize()) 64 | 65 | return &Cluster{ 66 | logger: logger, 67 | metadata: metadata, 68 | procedureFactory: procedureFactory, 69 | procedureManager: procedureManager, 70 | schedulerManager: schedulerManager, 71 | }, nil 72 | } 73 | 74 | func (c *Cluster) Start(ctx context.Context) error { 75 | if err := c.procedureManager.Start(ctx); err != nil { 76 | return errors.WithMessage(err, "start procedure manager") 77 | } 78 | if err := c.schedulerManager.Start(ctx); err != nil { 79 | return errors.WithMessage(err, "start scheduler manager") 80 | } 81 | return nil 82 | } 83 | 84 | func (c *Cluster) Stop(ctx context.Context) error { 85 | if err := c.procedureManager.Stop(ctx); err != nil { 86 | return errors.WithMessage(err, "stop procedure manager") 87 | } 88 | if err := c.schedulerManager.Stop(ctx); err != nil { 89 | return errors.WithMessage(err, "stop scheduler manager") 90 | } 91 | return nil 92 | } 93 | 94 | func (c *Cluster) GetMetadata() *metadata.ClusterMetadata { 95 | return c.metadata 96 | } 97 | 98 | func (c *Cluster) GetProcedureManager() procedure.Manager { 99 | return c.procedureManager 100 | } 101 | 102 | func (c *Cluster) GetProcedureFactory() *coordinator.Factory { 103 | return c.procedureFactory 104 | } 105 | 106 | func (c *Cluster) GetSchedulerManager() manager.SchedulerManager { 107 | return c.schedulerManager 108 | } 109 | 110 | func (c *Cluster) GetShards() []storage.ShardID { 111 | return c.metadata.GetShards() 112 | } 113 | 114 | func (c *Cluster) GetShardNodes() metadata.GetShardNodesResult { 115 | return c.metadata.GetShardNodes() 116 | } 117 | -------------------------------------------------------------------------------- /server/coordinator/procedure/operation/transferleader/batch_transfer_leader_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package transferleader_test 21 | 22 | import ( 23 | "context" 24 | "testing" 25 | 26 | "github.com/apache/incubator-horaedb-meta/server/coordinator/procedure" 27 | "github.com/apache/incubator-horaedb-meta/server/coordinator/procedure/operation/transferleader" 28 | "github.com/apache/incubator-horaedb-meta/server/storage" 29 | "github.com/stretchr/testify/require" 30 | ) 31 | 32 | type mockProcedure struct { 33 | ClusterID storage.ClusterID 34 | clusterVersion uint64 35 | kind procedure.Kind 36 | ShardWithVersion map[storage.ShardID]uint64 37 | } 38 | 39 | func (m mockProcedure) ID() uint64 { 40 | return 0 41 | } 42 | 43 | func (m mockProcedure) Kind() procedure.Kind { 44 | return m.kind 45 | } 46 | 47 | func (m mockProcedure) Start(_ context.Context) error { 48 | return nil 49 | } 50 | 51 | func (m mockProcedure) Cancel(_ context.Context) error { 52 | return nil 53 | } 54 | 55 | func (m mockProcedure) State() procedure.State { 56 | return procedure.StateInit 57 | } 58 | 59 | func (m mockProcedure) RelatedVersionInfo() procedure.RelatedVersionInfo { 60 | return procedure.RelatedVersionInfo{ 61 | ClusterID: m.ClusterID, 62 | ShardWithVersion: m.ShardWithVersion, 63 | ClusterVersion: m.clusterVersion, 64 | } 65 | } 66 | 67 | func (m mockProcedure) Priority() procedure.Priority { 68 | return procedure.PriorityLow 69 | } 70 | 71 | func TestBatchProcedure(t *testing.T) { 72 | re := require.New(t) 73 | var procedures []procedure.Procedure 74 | 75 | // Procedures with same type and version. 76 | for i := 0; i < 3; i++ { 77 | shardWithVersion := map[storage.ShardID]uint64{} 78 | shardWithVersion[storage.ShardID(i)] = 0 79 | p := CreateMockProcedure(storage.ClusterID(0), 0, 0, shardWithVersion) 80 | procedures = append(procedures, p) 81 | } 82 | _, err := transferleader.NewBatchTransferLeaderProcedure(0, procedures) 83 | re.NoError(err) 84 | 85 | // Procedure with different clusterID. 86 | for i := 0; i < 3; i++ { 87 | shardWithVersion := map[storage.ShardID]uint64{} 88 | shardWithVersion[storage.ShardID(i)] = 0 89 | p := CreateMockProcedure(storage.ClusterID(i), 0, procedure.TransferLeader, shardWithVersion) 90 | procedures = append(procedures, p) 91 | } 92 | _, err = transferleader.NewBatchTransferLeaderProcedure(0, procedures) 93 | re.Error(err) 94 | 95 | // Procedures with different type. 96 | for i := 0; i < 3; i++ { 97 | shardWithVersion := map[storage.ShardID]uint64{} 98 | shardWithVersion[storage.ShardID(i)] = 0 99 | p := CreateMockProcedure(0, 0, procedure.Kind(i), shardWithVersion) 100 | procedures = append(procedures, p) 101 | } 102 | _, err = transferleader.NewBatchTransferLeaderProcedure(0, procedures) 103 | re.Error(err) 104 | 105 | // Procedures with different version. 106 | for i := 0; i < 3; i++ { 107 | shardWithVersion := map[storage.ShardID]uint64{} 108 | shardWithVersion[storage.ShardID(0)] = uint64(i) 109 | p := CreateMockProcedure(0, 0, procedure.Kind(i), shardWithVersion) 110 | procedures = append(procedures, p) 111 | } 112 | _, err = transferleader.NewBatchTransferLeaderProcedure(0, procedures) 113 | re.Error(err) 114 | } 115 | 116 | func CreateMockProcedure(clusterID storage.ClusterID, clusterVersion uint64, typ procedure.Kind, shardWithVersion map[storage.ShardID]uint64) procedure.Procedure { 117 | return mockProcedure{ 118 | ClusterID: clusterID, 119 | clusterVersion: clusterVersion, 120 | kind: typ, 121 | ShardWithVersion: shardWithVersion, 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /server/coordinator/scheduler/reopen/scheduler.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package reopen 21 | 22 | import ( 23 | "context" 24 | "fmt" 25 | "strings" 26 | "time" 27 | 28 | "github.com/apache/incubator-horaedb-meta/server/cluster/metadata" 29 | "github.com/apache/incubator-horaedb-meta/server/coordinator" 30 | "github.com/apache/incubator-horaedb-meta/server/coordinator/procedure" 31 | "github.com/apache/incubator-horaedb-meta/server/coordinator/scheduler" 32 | "github.com/apache/incubator-horaedb-meta/server/storage" 33 | ) 34 | 35 | // schedulerImpl used to reopen shards in status PartitionOpen. 36 | type schedulerImpl struct { 37 | factory *coordinator.Factory 38 | procedureExecutingBatchSize uint32 39 | } 40 | 41 | func NewShardScheduler(factory *coordinator.Factory, procedureExecutingBatchSize uint32) scheduler.Scheduler { 42 | return schedulerImpl{ 43 | factory: factory, 44 | procedureExecutingBatchSize: procedureExecutingBatchSize, 45 | } 46 | } 47 | 48 | func (r schedulerImpl) Name() string { 49 | return "reopen_scheduler" 50 | } 51 | 52 | func (r schedulerImpl) UpdateEnableSchedule(_ context.Context, _ bool) { 53 | // ReopenShardScheduler do not need enableSchedule. 54 | } 55 | 56 | func (r schedulerImpl) AddShardAffinityRule(_ context.Context, _ scheduler.ShardAffinityRule) error { 57 | return nil 58 | } 59 | 60 | func (r schedulerImpl) RemoveShardAffinityRule(_ context.Context, _ storage.ShardID) error { 61 | return nil 62 | } 63 | 64 | func (r schedulerImpl) ListShardAffinityRule(_ context.Context) (scheduler.ShardAffinityRule, error) { 65 | return scheduler.ShardAffinityRule{Affinities: []scheduler.ShardAffinity{}}, nil 66 | } 67 | 68 | func (r schedulerImpl) Schedule(ctx context.Context, clusterSnapshot metadata.Snapshot) (scheduler.ScheduleResult, error) { 69 | var scheduleRes scheduler.ScheduleResult 70 | // ReopenShardScheduler can only be scheduled when the cluster is stable. 71 | if !clusterSnapshot.Topology.IsStable() { 72 | return scheduleRes, nil 73 | } 74 | now := time.Now() 75 | 76 | var procedures []procedure.Procedure 77 | var reasons strings.Builder 78 | 79 | for _, registeredNode := range clusterSnapshot.RegisteredNodes { 80 | if registeredNode.IsExpired(now) { 81 | continue 82 | } 83 | 84 | for _, shardInfo := range registeredNode.ShardInfos { 85 | if !needReopen(shardInfo) { 86 | continue 87 | } 88 | p, err := r.factory.CreateTransferLeaderProcedure(ctx, coordinator.TransferLeaderRequest{ 89 | Snapshot: clusterSnapshot, 90 | ShardID: shardInfo.ID, 91 | OldLeaderNodeName: "", 92 | NewLeaderNodeName: registeredNode.Node.Name, 93 | }) 94 | if err != nil { 95 | return scheduleRes, err 96 | } 97 | 98 | procedures = append(procedures, p) 99 | reasons.WriteString(fmt.Sprintf("the shard needs to be reopen , shardID:%d, shardStatus:%d, node:%s.", shardInfo.ID, shardInfo.Status, registeredNode.Node.Name)) 100 | if len(procedures) >= int(r.procedureExecutingBatchSize) { 101 | break 102 | } 103 | } 104 | } 105 | 106 | if len(procedures) == 0 { 107 | return scheduleRes, nil 108 | } 109 | 110 | batchProcedure, err := r.factory.CreateBatchTransferLeaderProcedure(ctx, coordinator.BatchRequest{ 111 | Batch: procedures, 112 | BatchType: procedure.TransferLeader, 113 | }) 114 | if err != nil { 115 | return scheduleRes, err 116 | } 117 | 118 | scheduleRes = scheduler.ScheduleResult{ 119 | Procedure: batchProcedure, 120 | Reason: reasons.String(), 121 | } 122 | return scheduleRes, nil 123 | } 124 | 125 | func needReopen(shardInfo metadata.ShardInfo) bool { 126 | return shardInfo.Status == storage.ShardStatusPartialOpen 127 | } 128 | -------------------------------------------------------------------------------- /server/service/http/route.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Prometheus Authors 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software 9 | // distributed under the License is distributed on an "AS IS" BASIS, 10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | // This file is copied from: 15 | // https://github.com/prometheus/common/blob/8c9cb3fa6d01832ea16937b20ea561eed81abd2f/route/route.go 16 | 17 | package http 18 | 19 | import ( 20 | "context" 21 | "net/http" 22 | 23 | "github.com/julienschmidt/httprouter" 24 | ) 25 | 26 | type param string 27 | 28 | const DebugPrefix = "/debug" 29 | 30 | // Router wraps httprouter.Router and adds support for prefixed sub-routers, 31 | // per-request context injections and instrumentation. 32 | type Router struct { 33 | rtr *httprouter.Router 34 | prefix string 35 | instrh func(handlerName string, handler http.HandlerFunc) http.HandlerFunc 36 | } 37 | 38 | func New() *Router { 39 | return &Router{ 40 | rtr: httprouter.New(), 41 | prefix: "", 42 | instrh: nil, 43 | } 44 | } 45 | 46 | // WithPrefix returns a router that prefixes all registered routes with prefix. 47 | func (r *Router) WithPrefix(prefix string) *Router { 48 | return &Router{rtr: r.rtr, prefix: r.prefix + prefix, instrh: r.instrh} 49 | } 50 | 51 | // WithInstrumentation returns a router with instrumentation support. 52 | func (r *Router) WithInstrumentation(instrh func(handlerName string, handler http.HandlerFunc) http.HandlerFunc) *Router { 53 | if r.instrh != nil { 54 | newInstrh := instrh 55 | instrh = func(handlerName string, handler http.HandlerFunc) http.HandlerFunc { 56 | return newInstrh(handlerName, r.instrh(handlerName, handler)) 57 | } 58 | } 59 | return &Router{rtr: r.rtr, prefix: r.prefix, instrh: instrh} 60 | } 61 | 62 | // ServeHTTP implements http.Handler. 63 | func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) { 64 | r.rtr.ServeHTTP(w, req) 65 | } 66 | 67 | // Get registers a new GET route. 68 | func (r *Router) Get(path string, h http.HandlerFunc) { 69 | r.rtr.GET(r.prefix+path, r.handle(path, h)) 70 | } 71 | 72 | // DebugGet registers a new GET route without prefix. 73 | func (r *Router) DebugGet(path string, h http.HandlerFunc) { 74 | r.rtr.GET(DebugPrefix+path, r.handle(path, h)) 75 | } 76 | 77 | // Options registers a new OPTIONS route. 78 | func (r *Router) Options(path string, h http.HandlerFunc) { 79 | r.rtr.OPTIONS(r.prefix+path, r.handle(path, h)) 80 | } 81 | 82 | // Del registers a new DELETE route. 83 | func (r *Router) Del(path string, h http.HandlerFunc) { 84 | r.rtr.DELETE(r.prefix+path, r.handle(path, h)) 85 | } 86 | 87 | // Put registers a new PUT route. 88 | func (r *Router) Put(path string, h http.HandlerFunc) { 89 | r.rtr.PUT(r.prefix+path, r.handle(path, h)) 90 | } 91 | 92 | // DebugPut registers a new PUT route without prefix. 93 | func (r *Router) DebugPut(path string, h http.HandlerFunc) { 94 | r.rtr.PUT(DebugPrefix+path, r.handle(path, h)) 95 | } 96 | 97 | // Post registers a new POST route. 98 | func (r *Router) Post(path string, h http.HandlerFunc) { 99 | r.rtr.POST(r.prefix+path, r.handle(path, h)) 100 | } 101 | 102 | // Head registers a new HEAD route. 103 | func (r *Router) Head(path string, h http.HandlerFunc) { 104 | r.rtr.HEAD(r.prefix+path, r.handle(path, h)) 105 | } 106 | 107 | // handle turns a HandlerFunc into a httprouter.Handle. 108 | func (r *Router) handle(handlerName string, h http.HandlerFunc) httprouter.Handle { 109 | if r.instrh != nil { 110 | // This needs to be outside the closure to avoid data race when reading and writing to 'h'. 111 | h = r.instrh(handlerName, h) 112 | } 113 | return func(w http.ResponseWriter, req *http.Request, params httprouter.Params) { 114 | ctx, cancel := context.WithCancel(req.Context()) 115 | defer cancel() 116 | 117 | for _, p := range params { 118 | ctx = context.WithValue(ctx, param(p.Key), p.Value) 119 | } 120 | h(w, req.WithContext(ctx)) 121 | } 122 | } 123 | 124 | // Param returns param p for the context, or the empty string when 125 | // param does not exist in context. 126 | func Param(ctx context.Context, p string) string { 127 | if v := ctx.Value(param(p)); v != nil { 128 | return v.(string) 129 | } 130 | return "" 131 | } 132 | -------------------------------------------------------------------------------- /server/coordinator/shard_picker_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package coordinator_test 21 | 22 | import ( 23 | "context" 24 | "sort" 25 | "testing" 26 | 27 | "github.com/apache/incubator-horaedb-meta/server/cluster/metadata" 28 | "github.com/apache/incubator-horaedb-meta/server/coordinator" 29 | "github.com/apache/incubator-horaedb-meta/server/coordinator/procedure/test" 30 | "github.com/apache/incubator-horaedb-meta/server/storage" 31 | "github.com/stretchr/testify/require" 32 | ) 33 | 34 | func TestLeastTableShardPicker(t *testing.T) { 35 | re := require.New(t) 36 | ctx := context.Background() 37 | 38 | c := test.InitStableCluster(ctx, t) 39 | snapshot := c.GetMetadata().GetClusterSnapshot() 40 | 41 | shardPicker := coordinator.NewLeastTableShardPicker() 42 | 43 | shardNodes, err := shardPicker.PickShards(ctx, snapshot, 4) 44 | re.NoError(err) 45 | re.Equal(len(shardNodes), 4) 46 | // Each shardNode should be different shard. 47 | shardIDs := map[storage.ShardID]struct{}{} 48 | for _, shardNode := range shardNodes { 49 | shardIDs[shardNode.ID] = struct{}{} 50 | } 51 | re.Equal(len(shardIDs), 4) 52 | 53 | shardNodes, err = shardPicker.PickShards(ctx, snapshot, 7) 54 | re.NoError(err) 55 | re.Equal(len(shardNodes), 7) 56 | // Each shardNode should be different shard. 57 | shardIDs = map[storage.ShardID]struct{}{} 58 | for _, shardNode := range shardNodes { 59 | shardIDs[shardNode.ID] = struct{}{} 60 | } 61 | re.Equal(len(shardIDs), 4) 62 | 63 | // Create table on shard 0. 64 | _, err = c.GetMetadata().CreateTable(ctx, metadata.CreateTableRequest{ 65 | ShardID: 0, 66 | LatestVersion: 0, 67 | SchemaName: test.TestSchemaName, 68 | TableName: "test", 69 | PartitionInfo: storage.PartitionInfo{ 70 | Info: nil, 71 | }, 72 | }) 73 | re.NoError(err) 74 | 75 | // shard 0 should not exist in pick result. 76 | shardNodes, err = shardPicker.PickShards(ctx, snapshot, 3) 77 | re.NoError(err) 78 | re.Equal(len(shardNodes), 3) 79 | for _, shardNode := range shardNodes { 80 | re.NotEqual(shardNode.ID, 0) 81 | } 82 | 83 | // drop shard node 1, shard 1 should not be picked. 84 | for _, shardNode := range snapshot.Topology.ClusterView.ShardNodes { 85 | if shardNode.ID == 1 { 86 | err = c.GetMetadata().DropShardNode(ctx, []storage.ShardNode{shardNode}) 87 | re.NoError(err) 88 | } 89 | } 90 | shardNodes, err = shardPicker.PickShards(ctx, snapshot, 8) 91 | re.NoError(err) 92 | for _, shardNode := range shardNodes { 93 | re.NotEqual(shardNode.ID, 1) 94 | } 95 | 96 | checkPartitionTable(ctx, shardPicker, t, 50, 256, 20, 2) 97 | checkPartitionTable(ctx, shardPicker, t, 50, 256, 30, 2) 98 | checkPartitionTable(ctx, shardPicker, t, 50, 256, 40, 2) 99 | checkPartitionTable(ctx, shardPicker, t, 50, 256, 50, 2) 100 | } 101 | 102 | func checkPartitionTable(ctx context.Context, shardPicker coordinator.ShardPicker, t *testing.T, nodeNumber int, shardNumber int, subTableNumber int, maxDifference int) { 103 | re := require.New(t) 104 | 105 | var shardNodes []storage.ShardNode 106 | 107 | c := test.InitStableClusterWithConfig(ctx, t, nodeNumber, shardNumber) 108 | shardNodes, err := shardPicker.PickShards(ctx, c.GetMetadata().GetClusterSnapshot(), subTableNumber) 109 | re.NoError(err) 110 | 111 | nodeTableCountMapping := make(map[string]int, 0) 112 | for _, shardNode := range shardNodes { 113 | nodeTableCountMapping[shardNode.NodeName]++ 114 | } 115 | 116 | // Ensure the difference in the number of tables is no greater than maxDifference 117 | var nodeTableNumberSlice []int 118 | for _, tableNumber := range nodeTableCountMapping { 119 | nodeTableNumberSlice = append(nodeTableNumberSlice, tableNumber) 120 | } 121 | sort.Ints(nodeTableNumberSlice) 122 | minTableNumber := nodeTableNumberSlice[0] 123 | maxTableNumber := nodeTableNumberSlice[len(nodeTableNumberSlice)-1] 124 | re.LessOrEqual(maxTableNumber-minTableNumber, maxDifference) 125 | } 126 | --------------------------------------------------------------------------------