├── docs ├── faq.md ├── assets │ ├── deployment.png │ ├── architecture.png │ └── gaea_dingtalk.png ├── architecture.md ├── concepts.md ├── quickstart.md ├── compatibility.md ├── grafana.md ├── multi-tenant.md ├── prepare.md ├── sequence-id.md ├── connection-pool.md └── config-reloading.md ├── backend ├── handshake.go ├── const.go ├── interface.go ├── direct_connection_test.go └── pooled_connection.go ├── .gitignore ├── misc ├── golint-all ├── gofmt-all └── git │ ├── pre-commit │ └── hooks │ ├── goimports │ ├── govet │ ├── gofmt │ └── golint ├── provider ├── provider.go ├── provider_type.go └── registry.go ├── logging ├── log_format.go ├── logger.go └── locked_multi_core.go ├── proxy ├── server │ ├── var.go │ └── executor_stmt_test.go ├── plan │ ├── plan_exec_test.go │ ├── plan_unshard_test.go │ ├── decorator_binary_operation_expr.go │ ├── decorator_limit.go │ ├── merge_result_test.go │ ├── route_result.go │ └── plan_explain.go ├── router │ ├── shard_test.go │ └── router.go └── sequence │ ├── sequence.go │ └── mysql.go ├── util ├── math │ └── compare.go ├── testleak │ ├── add-leaktest.sh │ ├── check-leaktest.sh │ └── fake.go ├── sync2 │ ├── doc.go │ ├── atomic_test.go │ ├── semaphore_flaky_test.go │ └── semaphore.go ├── string.go ├── request_context_test.go ├── cache │ └── perf_test.go ├── hack │ ├── hack_test.go │ └── hack.go ├── crypto │ └── xaes_test.go ├── padding.go ├── request_context.go ├── timer │ ├── randticker.go │ ├── randticker_flaky_test.go │ ├── timer_flaky_test.go │ └── timer.go ├── types.go ├── ip_test.go ├── requests │ └── api_test.go ├── bucketpool │ └── bucketpool.go ├── util.go ├── murmur.go ├── time_wheel_test.go └── ip.go ├── .travis.yml ├── genver.sh ├── config ├── configuration_source.go ├── source │ ├── etcd_source_test.go │ └── file_source.go └── boot_config.go ├── etc ├── gaea_cc.ini ├── file │ └── namespace │ │ ├── test_namespace_1.json │ │ └── test_namespace_2.json └── gaea.ini ├── cmd ├── gaea │ ├── main_test.go │ └── main.go └── gaea-cc │ └── main.go ├── go.mod ├── stats ├── util.go ├── variable_interface.go ├── ring.go ├── multidimensional_test.go ├── kebab_case_converter_test.go ├── multidimensional.go ├── snake_case_converter_test.go ├── counter_test.go ├── kebab_case_converter.go ├── snake_case_converter.go ├── histogram_test.go ├── timings_test.go ├── duration_test.go └── duration.go ├── parser ├── util.go └── sql_preview.go ├── Makefile ├── example └── namespace │ └── test_namespace_1.json ├── gen_version.sh ├── mysql ├── error_test.go ├── type_test.go ├── util_test.go ├── resultset_sort_test.go ├── charset_tidb_test.go ├── error.go └── resultset_sort.go ├── models ├── sequence.go ├── encode_test.go ├── proxy_monitor_metric.go ├── encode.go ├── store_test.go ├── slice.go ├── cc.go └── user.go ├── gen_ldflags.sh ├── core ├── string_builder.go └── version.go ├── README.md └── cc └── proxy └── proxy_api.go /docs/faq.md: -------------------------------------------------------------------------------- 1 | # FAQ -------------------------------------------------------------------------------- /backend/handshake.go: -------------------------------------------------------------------------------- 1 | package backend 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | vendor 2 | y.output 3 | bin 4 | logs 5 | .coverage.* 6 | .vscode 7 | .idea 8 | debug/ -------------------------------------------------------------------------------- /misc/golint-all: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | find . ! -path "./vendor/*" -name '*.go' -exec golint {} \; 4 | -------------------------------------------------------------------------------- /docs/assets/deployment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/endink/go-sharding/HEAD/docs/assets/deployment.png -------------------------------------------------------------------------------- /misc/gofmt-all: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | find . ! -path "./vendor/*" -name '*.go' -exec gofmt -s -w {} \; 4 | -------------------------------------------------------------------------------- /provider/provider.go: -------------------------------------------------------------------------------- 1 | package provider 2 | 3 | type Provider interface { 4 | GetName() string 5 | } 6 | -------------------------------------------------------------------------------- /docs/assets/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/endink/go-sharding/HEAD/docs/assets/architecture.png -------------------------------------------------------------------------------- /docs/assets/gaea_dingtalk.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/endink/go-sharding/HEAD/docs/assets/gaea_dingtalk.png -------------------------------------------------------------------------------- /provider/provider_type.go: -------------------------------------------------------------------------------- 1 | package provider 2 | 3 | type Type int 4 | 5 | const ( 6 | ConfigSource Type = iota 7 | ) 8 | -------------------------------------------------------------------------------- /logging/log_format.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | type LogFormat int 4 | 5 | const ( 6 | ColorizedOutput LogFormat = iota 7 | PlaintextOutput 8 | JSONOutput 9 | ) 10 | -------------------------------------------------------------------------------- /proxy/server/var.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import "sync/atomic" 4 | 5 | // Process global variables. 6 | var ( 7 | ProcessGeneralLog uint32 8 | ) 9 | 10 | func OpenProcessGeneralQueryLog() bool { 11 | return atomic.LoadUint32(&ProcessGeneralLog) == 1 12 | } 13 | -------------------------------------------------------------------------------- /backend/const.go: -------------------------------------------------------------------------------- 1 | package backend 2 | 3 | const ( 4 | defaultAuthPlugin = "mysql_native_password" 5 | defaultMaxAllowedPacket = 4 << 20 // 4 MiB 6 | minProtocolVersion = 10 7 | maxPacketSize = 1<<24 - 1 8 | timeFormat = "2006-01-02 15:04:05.999999" 9 | ) 10 | -------------------------------------------------------------------------------- /docs/architecture.md: -------------------------------------------------------------------------------- 1 | # 架构设计 2 | 3 | ## 模块划分 4 | 5 | gaea包含四个模块，分别是gaea-proxy、gaea-cc、gaea-agent、gaea-web。gaea-proxy为在线代理，负责承接sql流量，gaea-cc是中控模块，负责gaea-proxy的配置管理及一些后台任务，gaea-agent部署在mysql所在的机器上，负责实例创建、管理、回收等工作，gaea-web是gaea的一个管理界面，使gaea整体使用更加方便。 6 | 7 | ## 架构图 8 | 9 | ![gaea架构图](assets/architecture.png) 10 | -------------------------------------------------------------------------------- /util/math/compare.go: -------------------------------------------------------------------------------- 1 | package math 2 | 3 | // MaxInt64 returns the larger of a and b. 4 | func MaxInt64(a, b int64) int64 { 5 | if a > b { 6 | return a 7 | } 8 | 9 | return b 10 | } 11 | 12 | // MinInt64 returns the smaller of a and b. 13 | func MinInt64(a, b int64) int64 { 14 | if a < b { 15 | return a 16 | } 17 | 18 | return b 19 | } 20 | -------------------------------------------------------------------------------- /util/testleak/add-leaktest.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Usage: add-leaktest.sh pkg/*_test.go 4 | 5 | set -eu 6 | 7 | sed -i'~' -e ' 8 | /^func (s \*test.*Suite) Test.*(c \*C) {/ { 9 | n 10 | /testleak.AfterTest/! i\ 11 | defer testleak.AfterTest(c)() 12 | } 13 | ' $@ 14 | 15 | for i in $@; do 16 | if ! cmp -s $i $i~ ; then 17 | goimports -w $i 18 | fi 19 | echo $i 20 | rm -f $i~ 21 | done 22 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | os: linux 2 | env: 3 | - TZ=Asia/Shanghai 4 | 5 | sudo: required 6 | 7 | language: go 8 | 9 | go: 10 | - 1.13.4 11 | 12 | script: 13 | - export GO111MODULE=on 14 | - make 15 | 16 | deploy: 17 | provider: releases 18 | file: 19 | - bin/gaea 20 | - bin/gaea-cc 21 | overwrite: true 22 | skip_cleanup: true 23 | api_key: $GITHUB_TOKEN 24 | on: 25 | tags: true #发布tag时才进行发包 26 | -------------------------------------------------------------------------------- /genver.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | version=`git log --date=iso --pretty=format:"%cd @%h" -1` 4 | if [ $? -ne 0 ]; then 5 | version="not a git repo" 6 | fi 7 | 8 | compile=`date +"%F %T %z"`" by "`go version` 9 | 10 | cat << EOF | gofmt > core/version.go 11 | package core 12 | 13 | const ( 14 | // Version means gaea version 15 | Version = "$version" 16 | // Compile means gaea compole info 17 | Compile = "$compile" 18 | ) 19 | EOF 20 | -------------------------------------------------------------------------------- /config/configuration_source.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "github.com/XiaoMi/Gaea/provider" 5 | "time" 6 | ) 7 | 8 | // Configuration source provider 9 | type SourceProvider interface { 10 | provider.Provider 11 | OnLoad() 12 | Create(path string, data []byte) error 13 | Update(path string, data []byte) error 14 | UpdateWithTTL(path string, data []byte, ttl time.Duration) error 15 | Delete(path string) error 16 | Read(path string) ([]byte, error) 17 | List(path string) ([]string, error) 18 | Close() error 19 | BasePrefix() string 20 | } 21 | -------------------------------------------------------------------------------- /misc/git/pre-commit: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Runs any hooks in misc/git/hooks, and exits if any of them fail. 4 | set -e 5 | 6 | # This is necessary because the Emacs extensions don't set GIT_DIR. 7 | if [ -z "$GIT_DIR" ]; then 8 | DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) 9 | GIT_DIR="${DIR}/.." 10 | fi 11 | 12 | # This is necessary because the Atom packages don't set GOPATH 13 | if [ -z "$GOPATH" ]; then 14 | GOPATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../../../../.." && pwd ) 15 | export GOPATH 16 | fi 17 | 18 | for hook in $GIT_DIR/../misc/git/hooks/*; do 19 | $hook 20 | done 21 | -------------------------------------------------------------------------------- /etc/gaea_cc.ini: -------------------------------------------------------------------------------- 1 | addr=0.0.0.0:23306 2 | ; basic auth of gaea-cc 3 | admin_username=admin 4 | admin_password=admin 5 | 6 | ; basic auth of gaea-proxy's admin service 7 | proxy_username=test 8 | proxy_password=test 9 | 10 | ;Debug, Trace, Notice, Warn, Fatal, 建议测试采用debug级别，上线采用Notice级别 11 | log_level=Notice 12 | log_path =./logs 13 | log_filename=gaea_cc 14 | log_output=file 15 | 16 | ;coordinator目前支持etcd，coodinator config 17 | coordinator_addr=http://127.0.0.1:2379 18 | username=root 19 | password=root 20 | 21 | ;指定一个的默认gaea集群名称 22 | default_cluster=gaea_default_cluster 23 | 24 | ;encrypt key 25 | encrypt_key=1234abcd5678efg* 26 | -------------------------------------------------------------------------------- /docs/concepts.md: -------------------------------------------------------------------------------- 1 | # 基本概念 2 | 3 | * cluster 4 | 5 | 集群, 按照业务重要程度划分集群, 一个集群内可包含多个gaea-proxy实例, 通过指定gaea-proxy启动时依赖的配置文件中的cluster_name 6 | 确定该proxy所属集群。集群内的proxy实例只为该集群内的namespace提供服务, 起到物理隔离的作用。 7 | 一套集群可为多个namespace提供服务。 8 | 9 | * namespace 10 | 11 | 命名空间，每一个业务系统对应一个namespace，一个namespace对应多个database，业务方可以自由切换。 12 | 每个namespace理论上只会属于一个集群。 13 | 通过gaea-cc配置管理接口, 指定namespace所属集群。 14 | 15 | * slice 16 | 17 | 分片，逻辑上的分组，一个分片包含mysql一主多从。 18 | 19 | * shard 20 | 21 | 分表规则，确定一个表如何分表，包括分表的类型、分布的分片位置。 22 | 23 | * proxy 24 | 25 | 指代理本身，承接线上流量。 26 | 27 | * gaea_cc 28 | 29 | 代理控制模块，主要负责配置下发、实例监控等。 30 | 31 | * gaea_agent 32 | 33 | 部署在mysql实例所在机器，负责实例部署、管理、执行插件等功能。 34 | -------------------------------------------------------------------------------- /proxy/plan/plan_exec_test.go: -------------------------------------------------------------------------------- 1 | package plan 2 | 3 | import ( 4 | "github.com/XiaoMi/Gaea/parser" 5 | "testing" 6 | 7 | "github.com/XiaoMi/Gaea/util" 8 | ) 9 | 10 | func TestExecuteIn(t *testing.T) { 11 | planInfo, _ := preparePlanInfo() 12 | sql := "SELECT * FROM tbl_mycat_murmur WHERE tbl_mycat_murmur.id=5 AND tbl_mycat_murmur.id=4" 13 | stmt, _ := parser.ParseSQL(sql) 14 | plan, err := BuildPlan(stmt, nil, "db_mycat", sql, planInfo.rt, planInfo.seqs) 15 | if err != nil { 16 | t.Fatalf("build plan error: %v", err) 17 | } 18 | ret, err := plan.ExecuteIn(util.NewRequestContext(), nil) 19 | if err != nil { 20 | t.Fatalf("execute error: %v", err) 21 | } 22 | t.Logf("result: %v", ret) 23 | } 24 | -------------------------------------------------------------------------------- /util/sync2/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 Google Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package sync2 provides extra functionality along the same lines as sync. 18 | package sync2 19 | -------------------------------------------------------------------------------- /docs/quickstart.md: -------------------------------------------------------------------------------- 1 | # 快速入门 2 | 3 | ## 编译安装 4 | 5 | gaea基于go开发，基于go modules进行版本管理，并依赖goyacc、gofmt等工具。 6 | 7 | * go >= 1.11 8 | 9 | ```bash 10 | # 如果你已配置GOPATH，同时GO111MODULE设置为auto，请克隆Gaea到GOPATH外的目录 11 | git clone git@github.com:XiaoMi/Gaea.git 12 | 13 | # 如果拉取依赖速度慢，可以配置GOPROXY 14 | # export GOPROXY=https://athens.azurefd.net 15 | 16 | # 编译二进制包 17 | cd Gaea && make 18 | ``` 19 | 20 | ## 执行 21 | 22 | 编译之后在bin目录会有gaea、gaea-cc两个可执行文件。etc目录下为配置文件，如果想快速体验gaea功能，可以采用file配置方式，然后在etc/file/namespace下添加对应租户的json文件，该目录下目前有两个示例，可以直接修改使用。 23 | ./bin/gaea --help显示如下，其中-config是指定配置文件位置，默认为./etc/gaea.ini，具体配置见[配置说明](configuration.md)。 24 | 25 | ```bash 26 | Usage of ./bin/gaea: 27 | -config string 28 | gaea config file (default "etc/gaea.ini") 29 | ``` 30 | -------------------------------------------------------------------------------- /cmd/gaea/main_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Gaea Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package main 15 | 16 | import ( 17 | "testing" 18 | ) 19 | 20 | func Test_Main(t *testing.T) { 21 | main() 22 | } 23 | -------------------------------------------------------------------------------- /util/testleak/check-leaktest.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Usage: check-leaktest.sh 4 | # It needs to run under the github.com/pingcap/tidb directory. 5 | 6 | set -e 7 | 8 | pkgs=$(git grep 'Suite' |grep -vE "Godeps|tags" |awk -F: '{print $1}' | xargs -n1 dirname | sort |uniq) 9 | echo $pkgs 10 | for pkg in ${pkgs}; do 11 | if [ -z "$(ls ${pkg}/*_test.go 2>/dev/null)" ]; then 12 | continue 13 | fi 14 | awk -F'[(]' ' 15 | /func \(s .*Suite\) Test.*C\) {/ { 16 | test = $1"("$2 17 | next 18 | } 19 | 20 | /defer testleak.AfterTest/ { 21 | test = 0 22 | next 23 | } 24 | 25 | { 26 | if (test && (FILENAME != "./tidb_test.go")) { 27 | printf "%s: %s: missing defer testleak.AfterTest\n", FILENAME, test 28 | test = 0 29 | code = 1 30 | } 31 | } 32 | 33 | END { 34 | exit code 35 | } 36 | 37 | ' ${pkg}/*_test.go 38 | done 39 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/XiaoMi/Gaea 2 | 3 | go 1.15 4 | 5 | require ( 6 | github.com/coreos/etcd v3.3.13+incompatible 7 | github.com/emirpasic/gods v1.12.0 8 | github.com/gin-contrib/gzip v0.0.1 9 | github.com/gin-gonic/gin v1.5.0 10 | github.com/go-ini/ini v1.42.0 11 | github.com/golang/mock v1.3.1 12 | github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712 13 | github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011 14 | github.com/pingcap/parser v0.0.0-20200623164729-3a18f1e5dceb 15 | github.com/pingcap/tidb v1.1.0-beta.0.20200630082100-328b6d0a955c 16 | github.com/prometheus/client_golang v1.5.1 17 | github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa // indirect 18 | github.com/stretchr/testify v1.5.1 19 | go.uber.org/config v1.4.0 20 | go.uber.org/multierr v1.5.0 21 | go.uber.org/zap v1.16.0 22 | gopkg.in/ini.v1 v1.42.0 23 | 24 | ) 25 | -------------------------------------------------------------------------------- /stats/util.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Google Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package stats 18 | 19 | import "strings" 20 | 21 | // safeLabel turns a label into a safe label for stats export. 22 | // It is in its own file so it can be customized. 23 | func safeLabel(label string) string { 24 | return strings.Replace(label, ".", "_", -1) 25 | } 26 | -------------------------------------------------------------------------------- /util/string.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Gaea Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package util 16 | 17 | import "bytes" 18 | 19 | func Concat(strings ...string) string { 20 | var buffer bytes.Buffer 21 | for _, s := range strings { 22 | buffer.WriteString(s) 23 | } 24 | return buffer.String() 25 | } 26 | -------------------------------------------------------------------------------- /util/request_context_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Gaea Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package util 15 | 16 | import "testing" 17 | 18 | func Benchmark_RequestContext_Get(b *testing.B) { 19 | var rctx = NewRequestContext() 20 | rctx.Set("key", "value") 21 | for i := 0; i < b.N; i++ { 22 | rctx.Get("key") 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /parser/util.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/pingcap/parser" 5 | "github.com/pingcap/parser/ast" 6 | "github.com/pingcap/parser/format" 7 | _ "github.com/pingcap/tidb/types/parser_driver" 8 | "strings" 9 | ) 10 | 11 | var _testParser *parser.Parser 12 | 13 | func getTesterParser() *parser.Parser { 14 | if _testParser == nil { 15 | _testParser = parser.New() 16 | } 17 | return _testParser 18 | } 19 | 20 | //仅用于测试 21 | func ParseSQL(sql string) (ast.StmtNode, error) { 22 | n, e := getTesterParser().ParseOneStmt(sql, "", "") 23 | return n, e 24 | } 25 | 26 | const resultTableNameFlag format.RestoreFlags = 0 27 | 28 | // NodeToStringWithoutQuote get node text 29 | func NodeToStringWithoutQuote(node ast.Node) (string, error) { 30 | s := &strings.Builder{} 31 | if err := node.Restore(format.NewRestoreCtx(resultTableNameFlag, s)); err != nil { 32 | return "", err 33 | } 34 | return s.String(), nil 35 | } 36 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | ROOT:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) 2 | GAEA_OUT:=$(ROOT)/bin/gaea 3 | GAEA_CC_OUT:=$(ROOT)/bin/gaea-cc 4 | PKG:=$(shell go list -m) 5 | 6 | .PHONY: all build gaea gaea-cc parser clean test build_with_coverage 7 | all: build test 8 | 9 | build: parser gaea gaea-cc 10 | 11 | gaea: 12 | go build -o $(GAEA_OUT) $(shell bash gen_ldflags.sh $(GAEA_OUT) $(PKG)/core $(PKG)/cmd/gaea) 13 | 14 | gaea-cc: 15 | go build -o $(GAEA_CC_OUT) $(shell bash gen_ldflags.sh $(GAEA_CC_OUT) $(PKG)/core $(PKG)/cmd/gaea-cc) 16 | 17 | parser: 18 | cd parser && make && cd .. 19 | 20 | clean: 21 | @rm -rf bin 22 | @rm -f .coverage.out .coverage.html 23 | 24 | test: 25 | go test -coverprofile=.coverage.out ./... 26 | go tool cover -func=.coverage.out -o .coverage.func 27 | tail -1 .coverage.func 28 | go tool cover -html=.coverage.out -o .coverage.html 29 | 30 | build_with_coverage: 31 | go test -c cmd/gaea/main.go cmd/gaea/main_test.go -coverpkg ./... -covermode=count -o bin/gaea 32 | -------------------------------------------------------------------------------- /example/namespace/test_namespace_1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "test_namespace_1", 3 | "online": true, 4 | "read_only": false, 5 | "allowed_dbs": { 6 | "sbtest1": true 7 | }, 8 | "default_phy_dbs": { 9 | "sbtest1": "open_test" 10 | }, 11 | "slow_sql_time": "1000", 12 | "allowed_ip": null, 13 | "slices": [ 14 | { 15 | "name": "slice-0", 16 | "user_name": "root", 17 | "password": "123456", 18 | "master": "127.0.0.1:3306", 19 | "capacity": 12, 20 | "max_capacity": 24, 21 | "idle_timeout": 60 22 | } 23 | ], 24 | "shard_rules": null, 25 | "users": [ 26 | { 27 | "user_name": "root", 28 | "password": "root", 29 | "namespace": "test_namespace_1", 30 | "rw_flag": 2, 31 | "rw_split": 0, 32 | "other_property": 0 33 | } 34 | ], 35 | "default_slice": "slice-0", 36 | "global_sequences": null 37 | } 38 | -------------------------------------------------------------------------------- /util/cache/perf_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 Google Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package cache 18 | 19 | import ( 20 | "testing" 21 | ) 22 | 23 | type MyValue []byte 24 | 25 | func (mv MyValue) Size() int { 26 | return cap(mv) 27 | } 28 | 29 | func BenchmarkGet(b *testing.B) { 30 | cache := NewLRUCache(64 * 1024 * 1024) 31 | value := make(MyValue, 1000) 32 | cache.Set("stuff", value) 33 | for i := 0; i < b.N; i++ { 34 | val, ok := cache.Get("stuff") 35 | if !ok { 36 | panic("error") 37 | } 38 | _ = val 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /etc/file/namespace/test_namespace_1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "test_namespace_1", 3 | "online": true, 4 | "read_only": false, 5 | "allowed_dbs": { 6 | "sbtest1": true 7 | }, 8 | "slow_sql_time": "1000", 9 | "black_sql": [ 10 | "" 11 | ], 12 | "allowed_ip": null, 13 | "slices": [ 14 | { 15 | "name": "slice-0", 16 | "user_name": "test1", 17 | "password": "test1", 18 | "master": "127.0.0.1:3308", 19 | "slaves": null, 20 | "statistic_slaves": null, 21 | "capacity": 12, 22 | "max_capacity": 24, 23 | "idle_timeout": 60 24 | } 25 | ], 26 | "shard_rules": null, 27 | "users": [ 28 | { 29 | "user_name": "front_user1", 30 | "password": "front_password1", 31 | "namespace": "test_namespace_1", 32 | "rw_flag": 2, 33 | "rw_split": 1, 34 | "other_property": 0 35 | } 36 | ], 37 | "default_slice": "slice-0", 38 | "global_sequences": null 39 | } 40 | -------------------------------------------------------------------------------- /etc/file/namespace/test_namespace_2.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "test_namespace_2", 3 | "online": true, 4 | "read_only": false, 5 | "allowed_dbs": { 6 | "sbtest1": true 7 | }, 8 | "slow_sql_time": "1000", 9 | "black_sql": [ 10 | "" 11 | ], 12 | "allowed_ip": null, 13 | "slices": [ 14 | { 15 | "name": "slice-0", 16 | "user_name": "test2", 17 | "password": "test2", 18 | "master": "127.0.0.1:3309", 19 | "slaves": null, 20 | "statistic_slaves": null, 21 | "capacity": 12, 22 | "max_capacity": 24, 23 | "idle_timeout": 60 24 | } 25 | ], 26 | "shard_rules": null, 27 | "users": [ 28 | { 29 | "user_name": "front_user2", 30 | "password": "front_password2", 31 | "namespace": "test_namespace_2", 32 | "rw_flag": 2, 33 | "rw_split": 1, 34 | "other_property": 0 35 | } 36 | ], 37 | "default_slice": "slice-0", 38 | "global_sequences": null 39 | } 40 | -------------------------------------------------------------------------------- /docs/compatibility.md: -------------------------------------------------------------------------------- 1 | # Gaea兼容范围 2 | 3 | ## 协议兼容性 4 | 5 | Gaea支持text协议和binary协议. 6 | 7 | ## SQL兼容性 8 | 9 | Gaea对分表和非分表的兼容性有所不同. 非分表理论上支持所有DML语句, 部分ADMIN语句. 10 | 11 | 对分表情况, Gaea本身的定位是**轻量级, 高性能**, 因此采用轻量的分表实现方式, 对一条SQL的执行, 只做字段改写和结果聚合, 不做SQL语义上的改写和多条SQL结果集的拼接计算. 12 | 13 | **以下支持/不支持操作均指分表情况.** 14 | 15 | ### SELECT 16 | 17 | 明确支持以下操作: 18 | 19 | - JOIN操作支持一个父表和多个关联子表, 以及全局表. 20 | - 聚合函数支持SUM, MAX, MIN, COUNT, 且必须出现在最外层. 21 | - WHERE语句的条件支持AND, OR, 操作符支持=, >, >=, <, <=, <=>, IN, NOT IN, LIKE, NOT LIKE. 22 | - 支持GROUP BY. 23 | 24 | 明确不支持以下操作: 25 | 26 | - 不支持跨分片JOIN. JOIN中非分片键相关的条件, 只改写表名, 不计算路由, 走默认的广播路由. 27 | - JOIN USING不支持指定表名或DB名. 28 | - 表别名不允许与表名重复. 29 | - select animals.id from animals, test1.xm_order_extend as animals; 30 | - 这句SQL在MySQL中被认为是正确的, 但是gaea会明确拒绝这种操作. 31 | 32 | ### INSERT 33 | 34 | 明确不支持以下操作: 35 | 36 | - 不明确指定列名的INSERT 37 | - 跨分片批量INSERT 38 | - INSERT INTO SELECT 39 | 40 | ### UPDATE 41 | 42 | 明确不支持以下操作: 43 | 44 | - UPDATE多个表 45 | 46 | 47 | ## 事务兼容性 48 | 49 | - Gaea目前未实现分布式事务, 只支持单分片事务, 使用跨分片事务会报错. 50 | - 不支持SAVEPOINT, RELEASE SAVEPOINT, ROLLBACK TO SAVEPOINT **TODO** 51 | -------------------------------------------------------------------------------- /stats/variable_interface.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Vitess Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package stats 18 | 19 | // Variable is the minimal interface which each type in this "stats" package 20 | // must implement. 21 | // When integrating the Vitess stats types ("variables") with the different 22 | // monitoring systems, you can rely on this interface. 23 | type Variable interface { 24 | // Help returns the description of the variable. 25 | Help() string 26 | 27 | // String must implement String() from the expvar.Var interface. 28 | String() string 29 | } 30 | -------------------------------------------------------------------------------- /util/testleak/fake.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | // +build !leak 14 | 15 | package testleak 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/pingcap/check" 21 | ) 22 | 23 | // BeforeTest is a dummy implementation when build tag 'leak' is not set. 24 | func BeforeTest() { 25 | } 26 | 27 | // AfterTest is a dummy implementation when build tag 'leak' is not set. 28 | func AfterTest(c *check.C) func() { 29 | return func() { 30 | } 31 | } 32 | 33 | // AfterTestT is used after all the test cases is finished. 34 | func AfterTestT(t *testing.T) func() { 35 | return func() { 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /etc/gaea.ini: -------------------------------------------------------------------------------- 1 | ; config type, etcd/file, you can test gaea with file type, you shoud use etcd in production 2 | config_type=etcd 3 | ;file config path, 具体配置放到file_config_path的namespace目录下，该下级目录为固定目录 4 | file_config_path=./etc/file 5 | 6 | ;coordinator addr 7 | coordinator_addr=http://127.0.0.1:2379 8 | ;etcd user config 9 | username=root 10 | password=root 11 | 12 | ;environ 13 | environ=local 14 | ;service name 15 | service_name=gaea_proxy 16 | ;gaea_proxy cluster name 17 | cluster_name=gaea_default_cluster 18 | 19 | ;log config 20 | log_path=./logs 21 | log_level=Notice 22 | log_filename=gaea 23 | log_output=file 24 | 25 | ;admin addr 26 | admin_addr=0.0.0.0:13307 27 | ; basic auth 28 | admin_user=admin 29 | admin_password=admin 30 | 31 | ;proxy addr 32 | proto_type=tcp4 33 | proxy_addr=0.0.0.0:13306 34 | proxy_charset=utf8 35 | ;slow sql time, when execute time is higher than this, log it, unit: ms 36 | slow_sql_time=100 37 | ;close session after session timeout, unit: seconds 38 | session_timeout=3600 39 | 40 | ;stats conf 41 | stats_enabled=true 42 | ;stats interval 43 | stats_interval=10 44 | 45 | ;encrypt key 46 | encrypt_key=1234abcd5678efg* 47 | -------------------------------------------------------------------------------- /gen_version.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ROOT="$(pwd)" 4 | 5 | if BUILD_GIT_REVISION=$(git rev-parse HEAD 2> /dev/null); then if ! git diff-index --quiet HEAD; then BUILD_GIT_REVISION=${BUILD_GIT_REVISION}"-dirty" 6 | fi 7 | else 8 | BUILD_GIT_REVISION=unknown 9 | fi 10 | 11 | # Check for local changes 12 | if git diff-index --quiet HEAD --; then 13 | tree_status="Clean" 14 | else 15 | tree_status="Modified" 16 | fi 17 | 18 | # XXX This needs to be updated to accomodate tags added after building, rather than prior to builds 19 | RELEASE_TAG=$(git describe --match '[0-9]*\.[0-9]*\.[0-9]*' --exact-match --tags 2> /dev/null || echo "") 20 | 21 | # security wanted VERSION='unknown' 22 | VERSION="${BUILD_GIT_REVISION}" 23 | if [[ -n "${RELEASE_TAG}" ]]; then 24 | VERSION="${RELEASE_TAG}" 25 | elif [[ -n ${MY_VERSION} ]]; then 26 | VERSION="${MY_VERSION}" 27 | fi 28 | 29 | # used by pkg/version 30 | echo buildVersion "${VERSION}" 31 | echo buildGitRevision "${BUILD_GIT_REVISION}" 32 | echo buildUser "$(whoami)" 33 | echo buildHost "$(hostname -f)" 34 | echo buildStatus "${tree_status}" 35 | echo buildTime "$(date +%Y-%m-%d--%T)" 36 | -------------------------------------------------------------------------------- /mysql/error_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package mysql 15 | 16 | import ( 17 | "github.com/pingcap/check" 18 | ) 19 | 20 | var _ = check.Suite(&testSQLErrorSuite{}) 21 | 22 | type testSQLErrorSuite struct { 23 | } 24 | 25 | func (s *testSQLErrorSuite) TestSQLError(c *check.C) { 26 | e := NewErrf(ErrNoDB, "no db error") 27 | c.Assert(len(e.Error()), check.Greater, 0) 28 | 29 | e = NewErrf(0, "customized error") 30 | c.Assert(len(e.Error()), check.Greater, 0) 31 | 32 | e = NewDefaultError(ErrNoDB) 33 | c.Assert(len(e.Error()), check.Greater, 0) 34 | 35 | e = NewDefaultError(0, "customized error") 36 | c.Assert(len(e.Error()), check.Greater, 0) 37 | } 38 | -------------------------------------------------------------------------------- /backend/interface.go: -------------------------------------------------------------------------------- 1 | package backend 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/XiaoMi/Gaea/mysql" 8 | ) 9 | 10 | type PooledConnect interface { 11 | Recycle() 12 | Reconnect() error 13 | Close() 14 | IsClosed() bool 15 | UseDB(db string) error 16 | Execute(sql string) (*mysql.Result, error) 17 | SetAutoCommit(v uint8) error 18 | Begin() error 19 | Commit() error 20 | Rollback() error 21 | SetCharset(charset string, collation mysql.CollationID) (bool, error) 22 | FieldList(table string, wildcard string) ([]*mysql.Field, error) 23 | GetAddr() string 24 | SetSessionVariables(frontend *mysql.SessionVariables) (bool, error) 25 | WriteSetStatement() error 26 | } 27 | 28 | type ConnectionPool interface { 29 | Open() 30 | Addr() string 31 | Close() 32 | Get(ctx context.Context) (PooledConnect, error) 33 | Put(pc PooledConnect) 34 | 35 | SetCapacity(capacity int) (err error) 36 | SetIdleTimeout(idleTimeout time.Duration) 37 | StatsJSON() string 38 | Capacity() int64 39 | Available() int64 40 | Active() int64 41 | InUse() int64 42 | MaxCap() int64 43 | WaitCount() int64 44 | WaitTime() time.Duration 45 | IdleTimeout() time.Duration 46 | IdleClosed() int64 47 | } 48 | -------------------------------------------------------------------------------- /models/sequence.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Gaea Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package models 16 | 17 | // GlobalSequence means source of global sequences with different types 18 | type GlobalSequence struct { 19 | DB string `json:"db"` 20 | Table string `json:"table"` 21 | Type string `json:"type"` // 全局序列号类型,目前只兼容mycat的数据库方式 22 | SliceName string `json:"slice_name"` // 对应sequence表所在的分片，默认都在0号片 23 | PKName string `json:"pk_name"` // 全局序列号字段名称 24 | } 25 | 26 | // Encode means encode for easy use 27 | func (p *GlobalSequence) Encode() []byte { 28 | return JSONEncode(p) 29 | } 30 | -------------------------------------------------------------------------------- /models/encode_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Gaea Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package models 15 | 16 | import ( 17 | "testing" 18 | ) 19 | 20 | type TestStruct struct { 21 | Host string `json:"host"` 22 | Port int `json:"port"` 23 | } 24 | 25 | var a = ` 26 | { 27 | "host": "127.0.0.1", 28 | "port": 8888 29 | } 30 | ` 31 | 32 | func TestJSONEncode(t *testing.T) { 33 | ts := &TestStruct{Host: "127.0.0.1", Port: 8888} 34 | JSONEncode(ts) 35 | } 36 | 37 | func TestJSONDecode(t *testing.T) { 38 | var b TestStruct 39 | if err := JSONDecode(&b, []byte(a)); err != nil { 40 | t.Fatalf("test jsonDecode error, %v", err) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /models/proxy_monitor_metric.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Gaea Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package models 16 | 17 | // ProxyMonitorMetric proxy register information 18 | type ProxyMonitorMetric struct { 19 | Token string `json:"token"` //目前同AdminAddr 20 | StartTime string `json:"start_time"` 21 | 22 | IP string `json:"ip"` 23 | AdminPort string `json:"admin_port"` 24 | ProxyPort string `json:"proxy_port"` 25 | 26 | Pid int `json:"pid"` 27 | Pwd string `json:"pwd"` 28 | Sys string `json:"sys"` 29 | } 30 | 31 | // Encode encode jsosn 32 | func (p *ProxyMonitorMetric) Encode() []byte { 33 | return JSONEncode(p) 34 | } 35 | -------------------------------------------------------------------------------- /docs/grafana.md: -------------------------------------------------------------------------------- 1 | # 监控配置 2 | 3 | gaea proxy基于prometheus做统计数据的存储，使用grafana实现数据可视化展示。 4 | 5 | ## 监控说明 6 | 7 | ### gaea proxy监控 8 | 9 | [proxy grafana配置模板](template/gaea_proxy.json) 10 | 11 | proxy监控概览这部分主要展示gaea proxy的整体运行情况，主要包含以下几个监控项: 12 | 13 | - 集群QPS 14 | - 业务流量 15 | - 业务请求耗时 16 | - SQL 错误数 17 | - CPU 负载 18 | - 内存 负载 19 | - 流量负载 20 | - 会话数 21 | - 业务会话数 22 | - 协程数量 23 | - GC 停顿时间 24 | - 堆对象数量 25 | 26 | 27 | ### 租户各指标监控 28 | 29 | [namespace grafana配置模板](template/gaea_namespace.json) 30 | 31 | 导入模板之前需要把模板里的gaea_test_namespace 替换为实际使用的namespace 32 | 33 | 租户指标监控主要展示某个namespace的统计数据，主要包含以下几个监控项: 34 | 35 | - QPS 36 | - 流量 37 | - SQL耗时 38 | - SQL错误数 39 | - 高耗时SQL指纹 40 | - 错误SQL指纹 41 | - 连接数 42 | - 空闲连接数 43 | - 连接等待队列 44 | 45 | 46 | ## prometheus配置说明 47 | 48 | ``` 49 | - job_name: 'gaea_proxy' 50 | metrics_path: '/api/metric/metrics' 51 | static_configs: 52 | - targets: ["admin_addr1"] 53 | - targets: ["admin_addr2"] 54 | - targets: ["admin_addr3"] 55 | basic_auth: 56 | username: admin_user 57 | password: admin_password 58 | ``` 59 | 需要修改admin_addr,admin_user,admin_password与gaea.ini中的以下几项保持一致。 60 | ``` 61 | ;管理地址 62 | admin_addr=0.0.0.0:13307 63 | ;basic auth 64 | admin_user=admin 65 | admin_password=admin 66 | ``` 67 | ## 68 | 69 | 70 | -------------------------------------------------------------------------------- /models/encode.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Gaea Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package models 16 | 17 | import ( 18 | "encoding/json" 19 | "github.com/XiaoMi/Gaea/logging" 20 | ) 21 | 22 | // JSONEncode return json encoding of v 23 | func JSONEncode(v interface{}) []byte { 24 | b, err := json.MarshalIndent(v, "", " ") 25 | if err != nil { 26 | //TODO panic 27 | logging.DefaultLogger.Fatalf("encode to json failed, %v", err) 28 | return nil 29 | } 30 | return b 31 | } 32 | 33 | // JSONDecode parses the JSON-encoded data and stores the result in the value pointed to by v 34 | func JSONDecode(v interface{}, data []byte) error { 35 | return json.Unmarshal(data, v) 36 | } 37 | -------------------------------------------------------------------------------- /util/hack/hack_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The kingshard Authors. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | // not use this file except in compliance with the License. You may obtain 5 | // a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package hack 16 | 17 | import ( 18 | "bytes" 19 | "testing" 20 | ) 21 | 22 | func TestString(t *testing.T) { 23 | b := []byte("hello world") 24 | a := String(b) 25 | 26 | if a != "hello world" { 27 | t.Fatal(a) 28 | } 29 | 30 | b[0] = 'a' 31 | 32 | if a != "aello world" { 33 | t.Fatal(a) 34 | } 35 | 36 | b = append(b, "abc"...) 37 | if a != "aello world" { 38 | t.Fatal(a) 39 | } 40 | } 41 | 42 | func TestByte(t *testing.T) { 43 | a := "hello world" 44 | 45 | b := Slice(a) 46 | 47 | if !bytes.Equal(b, []byte("hello world")) { 48 | t.Fatal(string(b)) 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /gen_ldflags.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | VERBOSE=${VERBOSE:-"0"} 4 | V="" 5 | if [[ "${VERBOSE}" == "1" ]];then 6 | V="-x" 7 | set -x 8 | fi 9 | 10 | ROOT="$(pwd)" 11 | 12 | OUT=${1:?"output path"} 13 | VERSION_PACKAGE=${2:?"version go package"} # istio.io/istio/pkg/version 14 | BUILDPATH=${3:?"path to build"} 15 | 16 | set -e 17 | 18 | GOOS=${GOOS:-linux} 19 | GOARCH=${GOARCH:-amd64} 20 | GOBINARY=${GOBINARY:-go} 21 | GOPKG="$GOPATH/pkg" 22 | BUILDINFO=${BUILDINFO:-""} 23 | STATIC=${STATIC:-1} 24 | LDFLAGS="-extldflags -static" 25 | GOBUILDFLAGS=${GOBUILDFLAGS:-""} 26 | GCFLAGS=${GCFLAGS:-} 27 | export CGO_ENABLED=0 28 | 29 | if [[ "${STATIC}" != "1" ]];then 30 | LDFLAGS="" 31 | fi 32 | 33 | # gather buildinfo if not already provided 34 | # For a release build BUILDINFO should be produced 35 | # at the beginning of the build and used throughout 36 | if [[ -z ${BUILDINFO} ]];then 37 | BUILDINFO=$(mktemp) 38 | ${ROOT}/gen_version.sh > ${BUILDINFO} 39 | fi 40 | 41 | # BUILD LD_VERSIONFLAGS 42 | LD_VERSIONFLAGS="" 43 | while read line; do 44 | read SYMBOL VALUE < <(echo $line) 45 | LD_VERSIONFLAGS=${LD_VERSIONFLAGS}" -X ${VERSION_PACKAGE}.${SYMBOL}=${VALUE}" 46 | done < "${BUILDINFO}" 47 | 48 | echo -pkgdir=${GOPKG}/${GOOS}_${GOARCH} -ldflags "\"${LDFLAGS} ${LD_VERSIONFLAGS}\"" ${BUILDPATH} 49 | -------------------------------------------------------------------------------- /docs/multi-tenant.md: -------------------------------------------------------------------------------- 1 | # 多租户的设计与实现 2 | 3 | ## 背景 4 | 5 | gaea多租户是为了实现一套gaea集群，可以接入多个业务系统的不同数据库，方便部署、运维。gaea多租户为软多租户，一个租户称为一个namespace，多个namespace之间存在于一套gaea proxy集群内，所以是一种软隔离。我们也可以为一些重要等级业务系统单独部署一套gaea集群，甚至一套业务系统对应一套gaea集群实现物理隔离。 6 | 7 | ## 接入方式 8 | 9 | mysql的授权方式为用户名+密码+ip+数据库，接入gaea的情况下，授权方式为用户名+密码确定唯一一个namespace，ip则在白名单IP/IP段起作用，如果未配置白名单IP，则默认对所有IP生效。所以，不同的业务系统，用户名可以有相同的，但是用户名+密码要保证是唯一的，密码我们内部是根据一定的规则随机生成的，并且会校验是否重复。 10 | 11 | ## 实现原理 12 | 13 | ### 主要结构 14 | 15 | 在授权阶段还未能确定对应的namespace，所以user和namespace的配置是分别加载的，授权的实现主要依赖于UserManager结构体，其定义如下 16 | 17 | ```golang 18 | type UserManager struct { 19 | users map[string][]string // key: user name, value: user password, same user may have different password, so array of passwords is needed 20 | userNamespaces map[string]string // key: UserName+Password, value: name of namespace 21 | } 22 | ``` 23 | 24 | ### 配置加载过程 25 | 26 | 在系统初始化阶段，会依次加载对应user、namespace配置到一个全局Manager内，其中user部分用以授权检查，整个配置通过滚动数组的方式实现了无锁热加载，具体实现可以参照[gaea配置热加载实现原理](config-reloading.md)一章。 27 | 28 | ### 校验过程 29 | 30 | 其中users同一个用户名对应一个string数组，用以处理同一用户名不同密码的情形，而userNamespaces则用以通过用户名+密码快速获取对应的namespace名称。在验证阶段，首先通过CheckUser检查用户名是否存在，不存在则直接授权失败。然后，通过CheckPassword，依次对比确定是否可以找到对应密码，如果找不到，则最终授权失败；如果找到，则授权检查通过并记录对应的会话信息。 31 | 32 | ## 结语 33 | 34 | gaea的多租户确实为部署、运维带来了不少方便，后续也会考虑支持kubernetes的部署、调度多租户等，但是当下的多租户结构不会发生太大变化。 -------------------------------------------------------------------------------- /proxy/plan/plan_unshard_test.go: -------------------------------------------------------------------------------- 1 | package plan 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/XiaoMi/Gaea/backend" 7 | ) 8 | 9 | func TestUnshardPlan(t *testing.T) { 10 | ns, err := preparePlanInfo() 11 | if err != nil { 12 | t.Fatalf("prepare namespace error: %v", err) 13 | } 14 | tests := []SQLTestcase{ 15 | { 16 | db: "db_mycat", 17 | sql: `select * from tbl_unshard_a as a join db_mycat.tbl_unshard_b as b on a.id = b.id`, 18 | sqls: map[string]map[string][]string{ 19 | backend.DefaultSlice: { 20 | "db_mycat_0": {"SELECT * FROM `tbl_unshard_a` AS `a` JOIN `db_mycat_0`.`tbl_unshard_b` AS `b` ON `a`.`id`=`b`.`id`"}, 21 | }, 22 | }, 23 | }, 24 | } 25 | for _, test := range tests { 26 | t.Run(test.sql, getTestFunc(ns, test)) 27 | } 28 | } 29 | 30 | func TestUnshardPlanWithoutDB(t *testing.T) { 31 | ns, err := preparePlanInfo() 32 | if err != nil { 33 | t.Fatalf("prepare namespace error: %v", err) 34 | } 35 | tests := []SQLTestcase{ 36 | { 37 | db: "db_mycat", 38 | sql: `select * from tbl_unshard limit 10`, 39 | sqls: map[string]map[string][]string{ 40 | backend.DefaultSlice: { 41 | "db_mycat_0": {"SELECT * FROM `tbl_unshard` LIMIT 10"}, 42 | }, 43 | }, 44 | }, 45 | } 46 | for _, test := range tests { 47 | t.Run(test.sql, getTestFunc(ns, test)) 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /backend/direct_connection_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Gaea Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package backend 15 | 16 | import ( 17 | "bytes" 18 | "testing" 19 | ) 20 | 21 | func TestAppendSetVariable(t *testing.T) { 22 | var buf bytes.Buffer 23 | appendSetVariable(&buf, "charset", "utf8") 24 | t.Log(buf.String()) 25 | appendSetVariable(&buf, "autocommit", 1) 26 | t.Log(buf.String()) 27 | appendSetVariableToDefault(&buf, "sql_mode") 28 | t.Log(buf.String()) 29 | } 30 | 31 | func TestAppendSetVariable2(t *testing.T) { 32 | var buf bytes.Buffer 33 | appendSetCharset(&buf, "utf8", "utf8_general_ci") 34 | t.Log(buf.String()) 35 | appendSetVariable(&buf, "autocommit", 1) 36 | t.Log(buf.String()) 37 | appendSetVariableToDefault(&buf, "sql_mode") 38 | t.Log(buf.String()) 39 | } 40 | -------------------------------------------------------------------------------- /misc/git/hooks/goimports: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2017 Google Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # git goimports pre-commit hook 17 | # 18 | # To use, store as .git/hooks/pre-commit inside your repository and make sure 19 | # it has execute permissions. 20 | # 21 | # This script does not handle file names that contain spaces. 22 | gofiles=$(git diff --cached --name-only --diff-filter=d | grep '.go$') 23 | 24 | [ -z "$gofiles" ] && exit 0 25 | unformatted=$(goimports -l=true $gofiles 2>&1 | awk -F: '{print $1}') 26 | [ -z "$unformatted" ] && exit 0 27 | 28 | # Some files are not goimports'd. Print message and fail. 29 | 30 | echo >&2 "Go files must be formatted with goimports. Please run:" 31 | echo >&2 32 | echo -n >&2 " goimports -w" 33 | for fn in $unformatted; do 34 | echo -n >&2 " $PWD/$fn" 35 | done 36 | echo 37 | 38 | exit 1 39 | -------------------------------------------------------------------------------- /core/string_builder.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "fmt" 5 | "github.com/XiaoMi/Gaea/util" 6 | "strings" 7 | ) 8 | 9 | type StringBuilder struct { 10 | buffer strings.Builder 11 | } 12 | 13 | func NewStringBuilder(s ...string) *StringBuilder { 14 | sb := StringBuilder{} 15 | if s != nil && len(s) > 0 { 16 | _, _ = sb.buffer.WriteString(fmt.Sprint(s)) 17 | } 18 | return &sb 19 | } 20 | 21 | func (w *StringBuilder) Clear() { 22 | w.buffer.Reset() 23 | } 24 | 25 | func (w *StringBuilder) WriteLine(value ...interface{}) { 26 | for _, v := range value { 27 | w.Write(v) 28 | } 29 | w.buffer.WriteString(util.LineSeparator) 30 | } 31 | 32 | func (w *StringBuilder) Write(value interface{}) { 33 | a, isString := value.(string) 34 | if isString { 35 | _, _ = w.buffer.WriteString(a) 36 | return 37 | } 38 | 39 | b, isBuilder := value.(strings.Builder) 40 | if isBuilder { 41 | _, _ = w.buffer.WriteString(b.String()) 42 | return 43 | } 44 | _, _ = w.buffer.WriteString(fmt.Sprint(value)) 45 | 46 | } 47 | 48 | func (w *StringBuilder) WriteLineF(format string, args ...interface{}) { 49 | w.WriteFormat(format, args...) 50 | w.buffer.WriteString(util.LineSeparator) 51 | } 52 | 53 | func (w *StringBuilder) WriteFormat(format string, arg ...interface{}) { 54 | _, _ = w.buffer.WriteString(fmt.Sprintf(format, arg...)) 55 | } 56 | 57 | func (w *StringBuilder) String() string { 58 | return w.buffer.String() 59 | } 60 | -------------------------------------------------------------------------------- /proxy/plan/decorator_binary_operation_expr.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Gaea Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package plan 16 | 17 | import ( 18 | "github.com/pingcap/parser/ast" 19 | driver "github.com/pingcap/tidb/types/parser_driver" 20 | ) 21 | 22 | // BinaryOperationFieldtype declares field type of binary operation 23 | type BinaryOperationFieldtype int 24 | 25 | // Expr type 26 | const ( 27 | UnsupportExpr BinaryOperationFieldtype = iota 28 | ValueExpr 29 | ColumnNameExpr 30 | FuncCallExpr 31 | ) 32 | 33 | func getExprNodeTypeInBinaryOperation(n ast.ExprNode) BinaryOperationFieldtype { 34 | switch n.(type) { 35 | case *ast.ColumnNameExpr: 36 | return ColumnNameExpr 37 | case *driver.ValueExpr: 38 | return ValueExpr 39 | case *ast.FuncCallExpr: 40 | return FuncCallExpr 41 | default: 42 | return UnsupportExpr 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /proxy/server/executor_stmt_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Gaea Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package server 16 | 17 | import ( 18 | "testing" 19 | ) 20 | 21 | func Test_calcParams(t *testing.T) { 22 | sql := "update micf_order_0 set order_status=4, update_time=1541831505" + 23 | "where\n" + 24 | "order_id in ('1321989216361392') and\n" + 25 | "project_id = 371 and\n" + 26 | "order_status = 2" 27 | paramCount, offsets, err := calcParams(sql) 28 | t.Log(paramCount) 29 | t.Log(offsets) 30 | t.Log(err) 31 | if err != nil { 32 | t.Logf("test calcParams failed, %v\n", err) 33 | } 34 | 35 | sql = "select * from t1 where id = ? and col = ?" 36 | paramCount, offsets, err = calcParams(sql) 37 | t.Log(paramCount) 38 | t.Log(offsets) 39 | t.Log(err) 40 | if err != nil || paramCount != 2 { 41 | t.Logf("test calcParams failed, %v\n", err) 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /config/source/etcd_source_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Gaea Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package source 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/coreos/etcd/client" 21 | ) 22 | 23 | func Test_isErrNoNode(t *testing.T) { 24 | err := client.Error{} 25 | err.Code = client.ErrorCodeKeyNotFound 26 | if !isErrNoNode(err) { 27 | t.Fatalf("test isErrNoNode failed, %v", err) 28 | } 29 | err.Code = client.ErrorCodeNotFile 30 | if isErrNoNode(err) { 31 | t.Fatalf("test isErrNoNode failed, %v", err) 32 | } 33 | } 34 | 35 | func Test_isErrNodeExists(t *testing.T) { 36 | err := client.Error{} 37 | err.Code = client.ErrorCodeNodeExist 38 | if !isErrNodeExists(err) { 39 | t.Fatalf("test isErrNodeExists failed, %v", err) 40 | } 41 | err.Code = client.ErrorCodeNotFile 42 | if isErrNodeExists(err) { 43 | t.Fatalf("test isErrNodeExists failed, %v", err) 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /proxy/router/shard_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Gaea Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package router 16 | 17 | import "testing" 18 | 19 | func TestGetString(t *testing.T) { 20 | tests := []struct { 21 | v interface{} 22 | vs string 23 | }{ 24 | {int(-1), "-1"}, 25 | {int64(-1), "-1"}, 26 | {"-1", "-1"}, 27 | {[]byte("-1"), "-1"}, 28 | {int(0), "0"}, 29 | {int64(0), "0"}, 30 | {uint(0), "0"}, 31 | {uint64(0), "0"}, 32 | {"0", "0"}, 33 | {[]byte("0"), "0"}, 34 | {int(1), "1"}, 35 | {int64(1), "1"}, 36 | {uint(1), "1"}, 37 | {uint64(1), "1"}, 38 | {"1", "1"}, 39 | {[]byte("1"), "1"}, 40 | } 41 | for _, test := range tests { 42 | t.Run(test.vs, func(t *testing.T) { 43 | actualVs := GetString(test.v) 44 | if actualVs != test.vs { 45 | t.Errorf("not equal, v: %v, expect: %s, actual: %s", test.v, test.vs, actualVs) 46 | } 47 | }) 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /stats/ring.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 Google Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package stats 18 | 19 | // RingInt64 Ring of int64 values 20 | // Not thread safe 21 | type RingInt64 struct { 22 | position int 23 | values []int64 24 | } 25 | 26 | // NewRingInt64 create ring 27 | func NewRingInt64(capacity int) *RingInt64 { 28 | return &RingInt64{values: make([]int64, 0, capacity)} 29 | } 30 | 31 | // Add add value into ring 32 | func (ri *RingInt64) Add(val int64) { 33 | if len(ri.values) == cap(ri.values) { 34 | ri.values[ri.position] = val 35 | ri.position = (ri.position + 1) % cap(ri.values) 36 | } else { 37 | ri.values = append(ri.values, val) 38 | } 39 | } 40 | 41 | // Values return values in ring 42 | func (ri *RingInt64) Values() (values []int64) { 43 | values = make([]int64, len(ri.values)) 44 | for i := 0; i < len(ri.values); i++ { 45 | values[i] = ri.values[(ri.position+i)%cap(ri.values)] 46 | } 47 | return values 48 | } 49 | -------------------------------------------------------------------------------- /util/crypto/xaes_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Gaea Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package crypto 16 | 17 | import ( 18 | "encoding/base64" 19 | "testing" 20 | ) 21 | 22 | func TestEncryptECB(t *testing.T) { 23 | 24 | key := "1234abcd5678efg*" 25 | msg := "mQSa0mS1Gi1Q8VCVLHrbU_izaHqzaPmh" 26 | data, err := EncryptECB(key, []byte(msg)) 27 | if err != nil { 28 | t.Fatalf("encrypt failed, err:%v", err) 29 | return 30 | } 31 | base64Str := base64.StdEncoding.EncodeToString(data) 32 | 33 | t.Logf("encrypt succ, data: %v, data str: %s, len:%v", data, base64Str, len(data)) 34 | 35 | data, _ = base64.StdEncoding.DecodeString(base64Str) 36 | origin, err := DecryptECB(key, data) 37 | if err != nil { 38 | t.Fatalf("decrypt failed, err:%v", err) 39 | return 40 | } 41 | 42 | if string(origin) != msg { 43 | t.Fatalf("origin not equal msg") 44 | } 45 | t.Log("decrypt succ") 46 | } 47 | -------------------------------------------------------------------------------- /mysql/type_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package mysql 15 | 16 | import ( 17 | "github.com/pingcap/check" 18 | ) 19 | 20 | var _ = check.Suite(&testTypeSuite{}) 21 | 22 | type testTypeSuite struct{} 23 | 24 | func (s *testTypeSuite) TestFlags(c *check.C) { 25 | c.Assert(HasNotNullFlag(NotNullFlag), check.IsTrue) 26 | c.Assert(HasUniKeyFlag(UniqueKeyFlag), check.IsTrue) 27 | c.Assert(HasNotNullFlag(NotNullFlag), check.IsTrue) 28 | c.Assert(HasNoDefaultValueFlag(NoDefaultValueFlag), check.IsTrue) 29 | c.Assert(HasAutoIncrementFlag(AutoIncrementFlag), check.IsTrue) 30 | c.Assert(HasUnsignedFlag(UnsignedFlag), check.IsTrue) 31 | c.Assert(HasZerofillFlag(ZerofillFlag), check.IsTrue) 32 | c.Assert(HasBinaryFlag(BinaryFlag), check.IsTrue) 33 | c.Assert(HasPriKeyFlag(PriKeyFlag), check.IsTrue) 34 | c.Assert(HasMultipleKeyFlag(MultipleKeyFlag), check.IsTrue) 35 | c.Assert(HasTimestampFlag(TimestampFlag), check.IsTrue) 36 | c.Assert(HasOnUpdateNowFlag(OnUpdateNowFlag), check.IsTrue) 37 | } 38 | -------------------------------------------------------------------------------- /stats/multidimensional_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 Google Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreedto in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package stats 18 | 19 | import ( 20 | "reflect" 21 | "testing" 22 | "time" 23 | ) 24 | 25 | func TestMultiTimingsCounterFor(t *testing.T) { 26 | clear() 27 | mtm := NewMultiTimings("multitimings3", "help", []string{"dim1", "dim2"}) 28 | 29 | mtm.Add([]string{"tag1a", "tag1b"}, 500*time.Microsecond) 30 | mtm.Add([]string{"tag1a", "tag2b"}, 500*time.Millisecond) 31 | mtm.Add([]string{"tag2a", "tag2b"}, 500*time.Millisecond) 32 | 33 | cases := []struct { 34 | dim string 35 | want map[string]int64 36 | }{ 37 | {"dim1", map[string]int64{"tag1a": 2, "tag2a": 1, "All": 3}}, 38 | {"dim2", map[string]int64{"tag1b": 1, "tag2b": 2, "All": 3}}, 39 | } 40 | for _, c := range cases { 41 | counts := CounterForDimension(mtm, c.dim).Counts() 42 | if !reflect.DeepEqual(c.want, counts) { 43 | t.Errorf("mtm.CounterFor(%q).Counts()=%v, want %v", c.dim, counts, c.want) 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /util/sync2/atomic_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 Google Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package sync2 18 | 19 | import ( 20 | "testing" 21 | ) 22 | 23 | func TestAtomicString(t *testing.T) { 24 | var s AtomicString 25 | if s.Get() != "" { 26 | t.Errorf("want empty, got %s", s.Get()) 27 | } 28 | s.Set("a") 29 | if s.Get() != "a" { 30 | t.Errorf("want a, got %s", s.Get()) 31 | } 32 | if s.CompareAndSwap("b", "c") { 33 | t.Errorf("want false, got true") 34 | } 35 | if s.Get() != "a" { 36 | t.Errorf("want a, got %s", s.Get()) 37 | } 38 | if !s.CompareAndSwap("a", "c") { 39 | t.Errorf("want true, got false") 40 | } 41 | if s.Get() != "c" { 42 | t.Errorf("want c, got %s", s.Get()) 43 | } 44 | } 45 | 46 | func TestAtomicBool(t *testing.T) { 47 | b := NewAtomicBool(true) 48 | if !b.Get() { 49 | t.Error("b.Get: false, want true") 50 | } 51 | b.Set(false) 52 | if b.Get() { 53 | t.Error("b.Get: true, want false") 54 | } 55 | b.Set(true) 56 | if !b.Get() { 57 | t.Error("b.Get: false, want true") 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /stats/kebab_case_converter_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Vitess Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package stats 18 | 19 | import "testing" 20 | 21 | func TestToKebabCase(t *testing.T) { 22 | var kebabCaseTest = []struct{ input, output string }{ 23 | {"Camel", "camel"}, 24 | {"Camel", "camel"}, 25 | {"CamelCase", "camel-case"}, 26 | {"CamelCaseAgain", "camel-case-again"}, 27 | {"CCamel", "c-camel"}, 28 | {"CCCamel", "cc-camel"}, 29 | {"CAMEL_CASE", "camel-case"}, 30 | {"camel-case", "camel-case"}, 31 | {"0", "0"}, 32 | {"0.0", "0_0"}, 33 | {"JSON", "json"}, 34 | } 35 | 36 | for _, tt := range kebabCaseTest { 37 | if got, want := toKebabCase(tt.input), tt.output; got != want { 38 | t.Errorf("want '%s', got '%s'", want, got) 39 | } 40 | } 41 | } 42 | 43 | func TestMemoize(t *testing.T) { 44 | key := "Test" 45 | if memoizer.memo[key] != "" { 46 | t.Errorf("want '', got '%s'", memoizer.memo[key]) 47 | } 48 | toKebabCase(key) 49 | if memoizer.memo[key] != "test" { 50 | t.Errorf("want 'test', got '%s'", memoizer.memo[key]) 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /stats/multidimensional.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 Google Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreedto in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package stats 18 | 19 | import ( 20 | "fmt" 21 | "strings" 22 | ) 23 | 24 | // MultiTracker is a CountTracker that tracks counts grouping them by 25 | // more than one dimension. 26 | type MultiTracker interface { 27 | CountTracker 28 | Labels() []string 29 | } 30 | 31 | // CounterForDimension returns a CountTracker for the provided 32 | // dimension. It will panic if the dimension isn't a legal label for 33 | // mt. 34 | func CounterForDimension(mt MultiTracker, dimension string) CountTracker { 35 | for i, lab := range mt.Labels() { 36 | if lab == dimension { 37 | return wrappedCountTracker{ 38 | f: func() map[string]int64 { 39 | result := make(map[string]int64) 40 | for k, v := range mt.Counts() { 41 | if k == "All" { 42 | result[k] = v 43 | continue 44 | } 45 | result[strings.Split(k, ".")[i]] += v 46 | } 47 | return result 48 | }, 49 | } 50 | } 51 | } 52 | 53 | panic(fmt.Sprintf("label %v is not one of %v", dimension, mt.Labels())) 54 | } 55 | -------------------------------------------------------------------------------- /stats/snake_case_converter_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Vitess Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package stats 18 | 19 | import "testing" 20 | 21 | func TestToSnakeCase(t *testing.T) { 22 | var snakeCaseTest = []struct{ input, output string }{ 23 | {"Camel", "camel"}, 24 | {"Camel", "camel"}, 25 | {"CamelCase", "camel_case"}, 26 | {"CamelCaseAgain", "camel_case_again"}, 27 | {"CCamel", "c_camel"}, 28 | {"CCCamel", "cc_camel"}, 29 | {"CAMEL_CASE", "camel_case"}, 30 | {"camel-case", "camel_case"}, 31 | {"0", "0"}, 32 | {"0.0", "0_0"}, 33 | {"JSON", "json"}, 34 | } 35 | 36 | for _, tt := range snakeCaseTest { 37 | if got, want := toSnakeCase(tt.input), tt.output; got != want { 38 | t.Errorf("want '%s', got '%s'", want, got) 39 | } 40 | } 41 | } 42 | 43 | func TestSnakeMemoize(t *testing.T) { 44 | key := "Test" 45 | if snakeMemoizer.memo[key] != "" { 46 | t.Errorf("want '', got '%s'", snakeMemoizer.memo[key]) 47 | } 48 | toSnakeCase(key) 49 | if snakeMemoizer.memo[key] != "test" { 50 | t.Errorf("want 'test', got '%s'", snakeMemoizer.memo[key]) 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /proxy/plan/decorator_limit.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Gaea Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package plan 16 | 17 | import ( 18 | "github.com/pingcap/parser/ast" 19 | driver "github.com/pingcap/tidb/types/parser_driver" 20 | ) 21 | 22 | // NeedRewriteLimitOrCreateRewrite check if SelectStmt need rewrite limit clause, 23 | // if need, create a rewritten limit clause. 24 | // count == -1代表没有Limit子句 25 | func NeedRewriteLimitOrCreateRewrite(stmt *ast.SelectStmt) (bool, int64, int64, *ast.Limit) { 26 | limit := stmt.Limit 27 | if limit == nil { 28 | return false, -1, -1, nil 29 | } 30 | 31 | count := limit.Count.(*driver.ValueExpr).GetInt64() 32 | 33 | if limit.Offset == nil { 34 | return false, 0, count, nil 35 | } 36 | 37 | offset := limit.Offset.(*driver.ValueExpr).GetInt64() 38 | 39 | if offset == 0 { 40 | return false, 0, count, nil 41 | } 42 | 43 | newCount := count + offset 44 | nv := &driver.ValueExpr{} 45 | nv.SetInt64(newCount) 46 | newLimit := &ast.Limit{ 47 | Count: nv, 48 | } 49 | return true, offset, count, newLimit 50 | } 51 | -------------------------------------------------------------------------------- /util/padding.go: -------------------------------------------------------------------------------- 1 | /* 2 | The MIT License (MIT) 3 | 4 | Copyright (c) 2014 Coda Hale 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in 14 | all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | THE SOFTWARE. 23 | */ 24 | 25 | package util 26 | 27 | func times(str string, n int) (out string) { 28 | for i := 0; i < n; i++ { 29 | out += str 30 | } 31 | return 32 | } 33 | 34 | // Left left-pads the string with pad up to len runes 35 | // len may be exceeded if 36 | func Left(str string, length int, pad string) string { 37 | return times(pad, length-len(str)) + str 38 | } 39 | 40 | // Right right-pads the string with pad up to len runes 41 | func Right(str string, length int, pad string) string { 42 | return str + times(pad, length-len(str)) 43 | } 44 | -------------------------------------------------------------------------------- /models/store_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Gaea Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package models 16 | 17 | import ( 18 | "github.com/XiaoMi/Gaea/provider" 19 | "testing" 20 | ) 21 | 22 | func newTestStore() *provider.Store { 23 | c := provider.NewClient(provider.ConfigEtcd, "127.0.0.1:2381", "test", "test", "") 24 | return provider.NewStore(c) 25 | } 26 | 27 | func TestNewStore(t *testing.T) { 28 | c := provider.NewClient(provider.ConfigEtcd, "127.0.0.1:2381", "test", "test", "") 29 | store := provider.NewStore(c) 30 | defer store.Close() 31 | if store == nil { 32 | t.Fatalf("test NewStore failed") 33 | } 34 | } 35 | 36 | func TestNamespaceBase(t *testing.T) { 37 | store := newTestStore() 38 | defer store.Close() 39 | base := store.NamespaceBase() 40 | if base != "/gaea/namespace" { 41 | t.Fatalf("test NamespaceBase failed, %v", base) 42 | } 43 | } 44 | 45 | func TestNamespacePath(t *testing.T) { 46 | store := newTestStore() 47 | defer store.Close() 48 | path := store.NamespacePath("test") 49 | if path != "/gaea/namespace/test" { 50 | t.Fatalf("test NamespacePath failed, %v", path) 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /proxy/sequence/sequence.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Gaea Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package sequence 16 | 17 | import "fmt" 18 | 19 | // Sequence is interface of global sequences with different types 20 | type Sequence interface { 21 | GetPKName() string 22 | NextSeq() (int64, error) 23 | } 24 | 25 | type SequenceManager struct { 26 | sequences map[string]map[string]Sequence 27 | } 28 | 29 | func NewSequenceManager() *SequenceManager { 30 | return &SequenceManager{ 31 | sequences: make(map[string]map[string]Sequence), 32 | } 33 | } 34 | 35 | func (s *SequenceManager) SetSequence(db, table string, seq Sequence) error { 36 | if _, ok := s.sequences[db]; !ok { 37 | s.sequences[db] = make(map[string]Sequence) 38 | } 39 | if _, ok := s.sequences[db][table]; ok { 40 | return fmt.Errorf("already set sequence, db: %s, table: %s", db, table) 41 | } 42 | 43 | s.sequences[db][table] = seq 44 | return nil 45 | } 46 | 47 | func (s *SequenceManager) GetSequence(db, table string) (Sequence, bool) { 48 | dbSeq, ok := s.sequences[db] 49 | if !ok { 50 | return nil, false 51 | } 52 | seq, ok := dbSeq[table] 53 | return seq, ok 54 | } 55 | -------------------------------------------------------------------------------- /config/boot_config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "github.com/XiaoMi/Gaea/core" 5 | "github.com/XiaoMi/Gaea/logging" 6 | "github.com/XiaoMi/Gaea/util" 7 | "go.uber.org/config" 8 | "os" 9 | "path/filepath" 10 | ) 11 | 12 | var logger = logging.GetLogger("config") 13 | 14 | type BootConfig struct { 15 | Provider string 16 | } 17 | 18 | func LoadConfig() *BootConfig { 19 | var sources []config.YAMLOption 20 | 21 | files := defaultFileLocations() 22 | 23 | var sb = core.NewStringBuilder() 24 | sb.WriteLine() 25 | sb.WriteLine("Search configuration locations:") 26 | for _, f := range files { 27 | if util.FileExists(f) { 28 | sources = append(sources, config.File(f)) 29 | sb.WriteLine("[Found]:", f) 30 | } else { 31 | sb.WriteLine("[Not Found]:", f) 32 | } 33 | } 34 | 35 | bootCnf := &BootConfig{ 36 | Provider: "file", 37 | } 38 | 39 | if len(sources) > 0 { 40 | var err error 41 | var yaml *config.YAML 42 | if yaml, err = config.NewYAML(sources...); err == nil { 43 | err = yaml.Get("config").Populate(bootCnf) 44 | } 45 | if err != nil { 46 | logger.Warn("Load boot config file fault.", util.LineSeparator, err) 47 | } 48 | } 49 | 50 | return bootCnf 51 | } 52 | 53 | func defaultFileLocations() []string { 54 | files := make(map[string]bool, 3) 55 | if !util.IsWindows() { 56 | files["/etc/go-sharding/config.yaml"] = false 57 | files["/etc/go-sharding/config.yml"] = false 58 | } 59 | dir, err := os.Getwd() 60 | if err == nil { 61 | files[filepath.Join(dir, "config.yaml")] = false 62 | } else { 63 | files["config.yaml"] = false 64 | } 65 | 66 | result := make([]string, len(files)) 67 | i := 0 68 | for k, _ := range files { 69 | result[i] = k 70 | i++ 71 | } 72 | return result 73 | } 74 | -------------------------------------------------------------------------------- /util/sync2/semaphore_flaky_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 Google Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package sync2 18 | 19 | import ( 20 | "testing" 21 | "time" 22 | ) 23 | 24 | func TestSemaNoTimeout(t *testing.T) { 25 | s := NewSemaphore(1, 0) 26 | s.Acquire() 27 | released := false 28 | go func() { 29 | time.Sleep(10 * time.Millisecond) 30 | released = true 31 | s.Release() 32 | }() 33 | s.Acquire() 34 | if !released { 35 | t.Errorf("release: false, want true") 36 | } 37 | } 38 | 39 | func TestSemaTimeout(t *testing.T) { 40 | s := NewSemaphore(1, 5*time.Millisecond) 41 | s.Acquire() 42 | go func() { 43 | time.Sleep(10 * time.Millisecond) 44 | s.Release() 45 | }() 46 | if s.Acquire() { 47 | t.Errorf("Acquire: true, want false") 48 | } 49 | time.Sleep(10 * time.Millisecond) 50 | if !s.Acquire() { 51 | t.Errorf("Acquire: false, want true") 52 | } 53 | } 54 | 55 | func TestSemaTryAcquire(t *testing.T) { 56 | s := NewSemaphore(1, 0) 57 | if !s.TryAcquire() { 58 | t.Errorf("TryAcquire: false, want true") 59 | } 60 | if s.TryAcquire() { 61 | t.Errorf("TryAcquire: true, want false") 62 | } 63 | s.Release() 64 | if !s.TryAcquire() { 65 | t.Errorf("TryAcquire: false, want true") 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /misc/git/hooks/govet: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Copyright 2017 Google Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # git go vet pre-commit hook 17 | # 18 | # To use, store as .git/hooks/pre-commit inside your repository and make sure 19 | # it has execute permissions. 20 | 21 | if [ -z "$GOPATH" ]; then 22 | echo "ERROR: pre-commit hook for go vet: \$GOPATH is empty. Please run 'source dev.env' to set the correct \$GOPATH." 23 | exit 1 24 | fi 25 | 26 | # This script does not handle file names that contain spaces. 27 | gofiles=$(git diff --cached --name-only --diff-filter=d | grep '.go$') 28 | 29 | # If any checks are found to be useless, they can be disabled here. 30 | # See the output of "go tool vet" for a list of flags. 31 | vetflags="-all=true" 32 | 33 | errors= 34 | 35 | # Run on one file at a time because a single invocation of "go tool vet" 36 | # with multiple files requires the files to all be in one package. 37 | for gofile in $gofiles 38 | do 39 | if ! go tool vet $vetflags $gofile 2>&1; then 40 | errors=YES 41 | fi 42 | done 43 | 44 | [ -z "$errors" ] && exit 0 45 | 46 | echo 47 | echo "Please fix the go vet warnings above. To disable certain checks, change vetflags in misc/git/hooks/govet." 48 | exit 1 49 | -------------------------------------------------------------------------------- /util/request_context.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Gaea Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package util 16 | 17 | import ( 18 | "sync" 19 | ) 20 | 21 | const ( 22 | // StmtType stmt type 23 | StmtType = "stmtType" // SQL类型, 值类型为int (对应parser.Preview()得到的值) 24 | // FromSlave if read from slave 25 | FromSlave = "fromSlave" // 读写分离标识, 值类型为int, false = 0, true = 1 26 | ) 27 | 28 | // RequestContext means request scope context with values 29 | // thread safe 30 | type RequestContext struct { 31 | lock *sync.RWMutex 32 | ctx map[string]interface{} 33 | } 34 | 35 | // NewRequestContext return request scopre context 36 | func NewRequestContext() *RequestContext { 37 | return &RequestContext{ctx: make(map[string]interface{}, 2), lock: new(sync.RWMutex)} 38 | } 39 | 40 | // Get return context in RequestContext 41 | func (reqCtx *RequestContext) Get(key string) interface{} { 42 | reqCtx.lock.RLock() 43 | v, ok := reqCtx.ctx[key] 44 | reqCtx.lock.RUnlock() 45 | if ok { 46 | return v 47 | } 48 | return nil 49 | } 50 | 51 | // Set set value with specific key 52 | func (reqCtx *RequestContext) Set(key string, value interface{}) { 53 | reqCtx.lock.Lock() 54 | reqCtx.ctx[key] = value 55 | reqCtx.lock.Unlock() 56 | } 57 | -------------------------------------------------------------------------------- /util/timer/randticker.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 Google Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package timer 18 | 19 | import ( 20 | "math/rand" 21 | "time" 22 | ) 23 | 24 | // RandTicker is just like time.Ticker, except that 25 | // it adds randomness to the events. 26 | type RandTicker struct { 27 | C <-chan time.Time 28 | done chan struct{} 29 | } 30 | 31 | // NewRandTicker creates a new RandTicker. d is the duration, 32 | // and variance specifies the variance. The ticker will tick 33 | // every d +/- variance. 34 | func NewRandTicker(d, variance time.Duration) *RandTicker { 35 | c := make(chan time.Time, 1) 36 | done := make(chan struct{}) 37 | go func() { 38 | rnd := rand.New(rand.NewSource(time.Now().UnixNano())) 39 | for { 40 | vr := time.Duration(rnd.Int63n(int64(2*variance)) - int64(variance)) 41 | tmr := time.NewTimer(d + vr) 42 | select { 43 | case <-tmr.C: 44 | select { 45 | case c <- time.Now(): 46 | default: 47 | } 48 | case <-done: 49 | tmr.Stop() 50 | close(c) 51 | return 52 | } 53 | } 54 | }() 55 | return &RandTicker{ 56 | C: c, 57 | done: done, 58 | } 59 | } 60 | 61 | // Stop stops the ticker and closes the underlying channel. 62 | func (tkr *RandTicker) Stop() { 63 | close(tkr.done) 64 | } 65 | -------------------------------------------------------------------------------- /docs/prepare.md: -------------------------------------------------------------------------------- 1 | # prepare的设计与实现 2 | 3 | ## 背景 4 | 5 | 应用端使用prepare主要考虑通过固定sql模板，在执行sql时只传输参数，减少数据包传输大小，提升sql执行效率。对于非分库分表的情况，我们可以直接通过转发execute(对应后端连接可能是prepare+execute+close)的方式进行支持。但是对于分库分表的情形，需要计算路由，重写sql，支持起来会非常麻烦。商城目前的分库分表中间件是mycat，而mycat是支持prepare的，而gaea的prepare方案也是参照mycat，即将prepare statements的执行转换为sql的执行，然后在应答阶段，根据文本的应答内容构造二进制应答内容，返回给客户端，从而统一了分库分表的处理逻辑。 6 | 7 | ## prepare 8 | 9 | gaea在接到preprae请求后，首先计算参数个数、参数偏移位置和stmt-id。然后根据以上数据，构造statement对象并保存在SessionExecutor的stmts内，stmts为一个map，key为stmt-id，value即为构造的statement对象。 10 | 11 | prepare阶段主要是计算、保存execute需要使用的变量信息，prepare应答数据内也会包含这些变量信息。 12 | 13 | ## execute 14 | 15 | execute请求时会携带prepare应答返回的stmt-id，服务端根据stmt-id从SessionExecutor的stmts中查询对应的statement信息。根据statement信息的参数个数、偏移和execute上传的参数值，进行关联绑定，然后rewrite一条同等含义的sql。同时，为了安全性考虑，也会进行特殊字符过滤，防止比如sql注入的发生。 16 | 17 | 生成sql之后，无论是分表还是非分表，我们都可以调用handleQuery进行统一的处理，避免了因为要支持prepare，而存在两套计算分库、分表路由的逻辑。 18 | 19 | 处理完成之后，需要进行文本应答协议到二进制应答协议的转换，相关实现在BuildBinaryResultset内。 20 | 21 | execute执行完成之后，执行ResetParams，重新初始化send_long_data对应的args字段，病返回应答。 22 | 23 | ## send_long_data 24 | 25 | send_long_data不是必须的，但是如果execute有多个参数，且不止一个参数长度比较大，一次execute可能达到mysql max-payload-length，但是如果分多次，每次只发送一个，这样就绕过了max-payload-length，send_long_data就是基于这样的背景产生的。 26 | 27 | 客户端发送send_long_data报文，会携带stmt-id、param-id(参数位置)，我们根据stmt-id参数可以检索prepare阶段存储的stmt信息，根据param-id和对应上送的数据，可以建立一个k-v映射，存储在Stmt.args(interface slice)。在上述execute执行阶段，也会根据参数位置从Stmt.args查询对应位置的参数值，进行关联绑定。 28 | 29 | send_long_data不需要应答。 30 | 31 | ## close 32 | 33 | close的处理逻辑比较简单，服务端收到close请求后，删除prepare阶段stmt-id及其数据的对应关系。 34 | 35 | ## 总结 36 | 37 | gaea对于prepare的处理初衷还是考虑协议的兼容和简化处理逻辑，对于client->proxy->mysql这样接口来说，client->proxy是prepare协议，proxy->mysql是文本协议，所以整体来看在gaea环境下使用prepare性能提升有限，还是建议直接使用sql。 38 | 39 | ## 参考资料 40 | 41 | [mysql prepare statements 官方文档](https://dev.mysql.com/doc/internals/en/prepared-statements.html) -------------------------------------------------------------------------------- /util/types.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Gaea Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package util 16 | 17 | import ( 18 | "github.com/pingcap/parser/format" 19 | "strings" 20 | 21 | "github.com/pingcap/tidb/types" 22 | driver "github.com/pingcap/tidb/types/parser_driver" 23 | ) 24 | 25 | var EscapeRestoreFlags = format.RestoreStringSingleQuotes | format.RestoreStringEscapeBackslash | format.RestoreKeyWordUppercase | format.RestoreNameBackQuotes 26 | 27 | // GetValueExprResult copy from ValueExpr.Restore() 28 | // TODO: 分表列是否需要支持等值比较NULL 29 | func GetValueExprResult(n *driver.ValueExpr) (interface{}, error) { 30 | switch n.Kind() { 31 | case types.KindNull: 32 | return nil, nil // TODO: null str or nil? 33 | case types.KindInt64: 34 | return n.GetInt64(), nil 35 | case types.KindUint64: 36 | return n.GetUint64(), nil 37 | case types.KindFloat32: 38 | return n.GetFloat32(), nil 39 | case types.KindFloat64: 40 | return n.GetFloat64(), nil 41 | case types.KindString, types.KindBytes: 42 | return n.GetString(), nil 43 | default: 44 | s := &strings.Builder{} 45 | ctx := format.NewRestoreCtx(EscapeRestoreFlags, s) 46 | err := n.Restore(ctx) 47 | if err != nil { 48 | return nil, err 49 | } 50 | return s.String(), nil 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /util/timer/randticker_flaky_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 Google Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package timer 18 | 19 | import ( 20 | "testing" 21 | "time" 22 | ) 23 | 24 | const ( 25 | testDuration = 100 * time.Millisecond 26 | testVariance = 20 * time.Millisecond 27 | ) 28 | 29 | func TestTick(t *testing.T) { 30 | tkr := NewRandTicker(testDuration, testVariance) 31 | for i := 0; i < 5; i++ { 32 | start := time.Now() 33 | end := <-tkr.C 34 | diff := start.Add(testDuration).Sub(end) 35 | tolerance := testVariance + 20*time.Millisecond 36 | if diff < -tolerance || diff > tolerance { 37 | t.Errorf("start: %v, end: %v, diff %v. Want <%v tolerenace", start, end, diff, tolerance) 38 | } 39 | } 40 | tkr.Stop() 41 | _, ok := <-tkr.C 42 | if ok { 43 | t.Error("Channel was not closed") 44 | } 45 | } 46 | 47 | func TestTickSkip(t *testing.T) { 48 | tkr := NewRandTicker(10*time.Millisecond, 1*time.Millisecond) 49 | time.Sleep(35 * time.Millisecond) 50 | end := <-tkr.C 51 | diff := time.Now().Sub(end) 52 | if diff < 20*time.Millisecond { 53 | t.Errorf("diff: %v, want >20ms", diff) 54 | } 55 | 56 | // This tick should be up-to-date 57 | end = <-tkr.C 58 | diff = time.Now().Sub(end) 59 | if diff > 1*time.Millisecond { 60 | t.Errorf("diff: %v, want <1ms", diff) 61 | } 62 | tkr.Stop() 63 | } 64 | -------------------------------------------------------------------------------- /stats/counter_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Vitess Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package stats 18 | 19 | import ( 20 | "expvar" 21 | "testing" 22 | ) 23 | 24 | func TestCounter(t *testing.T) { 25 | var gotname string 26 | var gotv *Counter 27 | clear() 28 | Register(func(name string, v expvar.Var) { 29 | gotname = name 30 | gotv = v.(*Counter) 31 | }) 32 | v := NewCounter("Int", "help") 33 | if gotname != "Int" { 34 | t.Errorf("want Int, got %s", gotname) 35 | } 36 | if gotv != v { 37 | t.Errorf("want %#v, got %#v", v, gotv) 38 | } 39 | v.Add(1) 40 | if v.Get() != 1 { 41 | t.Errorf("want 1, got %v", v.Get()) 42 | } 43 | if v.String() != "1" { 44 | t.Errorf("want 1, got %v", v.Get()) 45 | } 46 | v.Reset() 47 | if v.Get() != 0 { 48 | t.Errorf("want 0, got %v", v.Get()) 49 | } 50 | } 51 | 52 | func TestGaugeFunc(t *testing.T) { 53 | var gotname string 54 | var gotv *GaugeFunc 55 | clear() 56 | Register(func(name string, v expvar.Var) { 57 | gotname = name 58 | gotv = v.(*GaugeFunc) 59 | }) 60 | 61 | v := NewGaugeFunc("name", "help", func() int64 { 62 | return 1 63 | }) 64 | if gotname != "name" { 65 | t.Errorf("want name, got %s", gotname) 66 | } 67 | if gotv != v { 68 | t.Errorf("want %#v, got %#v", v, gotv) 69 | } 70 | if v.String() != "1" { 71 | t.Errorf("want 1, got %v", v.String()) 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /proxy/plan/merge_result_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Gaea Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package plan 16 | 17 | import ( 18 | "fmt" 19 | "testing" 20 | 21 | "github.com/XiaoMi/Gaea/mysql" 22 | ) 23 | 24 | func TestLimitSelectResult(t *testing.T) { 25 | tests := []struct { 26 | offset int64 27 | count int64 28 | retLen int64 29 | trimedRetLen int64 30 | }{ 31 | {-1, -1, 5, 5}, 32 | {0, -1, 5, 5}, 33 | {0, 3, 5, 3}, 34 | {0, 10, 5, 5}, 35 | {3, 10, 5, 2}, 36 | {3, 1, 5, 1}, 37 | {4, 1, 5, 1}, 38 | {5, 1, 5, 0}, 39 | {5, 10, 5, 0}, 40 | } 41 | 42 | for _, test := range tests { 43 | t.Run(fmt.Sprintf("%d:%d", test.offset, test.count), func(t *testing.T) { 44 | info := &SelectPlan{ 45 | offset: test.offset, 46 | count: test.count, 47 | } 48 | 49 | ret := &mysql.Result{ 50 | Resultset: &mysql.Resultset{ 51 | Values: make([][]interface{}, test.retLen), 52 | RowDatas: make([]mysql.RowData, test.retLen), 53 | }, 54 | } 55 | 56 | if err := limitSelectResult(info, ret); err != nil { 57 | t.Fatalf("limitSelectResult error: %v", err) 58 | } 59 | 60 | if int64(len(ret.Values)) != test.trimedRetLen { 61 | t.Errorf("len Values not equal, expect: %d, actual: %d", test.trimedRetLen, len(ret.Values)) 62 | } 63 | }) 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /logging/logger.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | import ( 4 | "go.uber.org/zap" 5 | "go.uber.org/zap/zapcore" 6 | "os" 7 | "sync" 8 | ) 9 | 10 | var loggerMutex sync.RWMutex // guards access to global logger state 11 | 12 | // loggers is the set of loggers in the system 13 | var loggers = make(map[string]*zap.SugaredLogger) 14 | 15 | var levels = make(map[string]zap.AtomicLevel) 16 | var defaultLevel zapcore.Level = zapcore.InfoLevel 17 | var output = zapcore.Lock(os.Stdout) 18 | 19 | var logCore = newCore(ColorizedOutput, output, defaultLevel) 20 | 21 | /** 22 | func newLogger(options []zap.Option) (*zap.Logger, error) { 23 | var level zapcore.Level 24 | err := (&level).UnmarshalText([]byte(*loggerLevelPtr)) 25 | if err != nil { 26 | return nil, err 27 | } 28 | 29 | conf := zap.NewProductionConfig() 30 | 31 | // Use logger profile if set on command line before falling back 32 | // to default based on build type. 33 | switch *loggerProfilePtr { 34 | case "dev": 35 | conf = zap.NewDevelopmentConfig() 36 | case "prod": 37 | conf = zap.NewProductionConfig() 38 | default: 39 | if version.IsDevBuild() { 40 | conf = zap.NewDevelopmentConfig() 41 | } 42 | } 43 | 44 | conf.Encoding = *loggerFormatPtr 45 | if conf.Encoding == "console" { 46 | // Human-readable timestamps for console format of logs. 47 | conf.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder 48 | } 49 | 50 | conf.Level.SetLevel(level) 51 | return conf.Build(options...) 52 | } 53 | 54 | */ 55 | 56 | var DefaultLogger = GetLogger("sharding-proxy") 57 | 58 | func GetLogger(name string) *zap.SugaredLogger { 59 | loggerMutex.Lock() 60 | defer loggerMutex.Unlock() 61 | log, ok := loggers[name] 62 | if !ok { 63 | levels[name] = zap.NewAtomicLevelAt(defaultLevel) 64 | 65 | log = zap.New(logCore, zap.AddCaller()). 66 | WithOptions(zap.IncreaseLevel(levels[name])). 67 | Named(name). 68 | Sugar() 69 | 70 | loggers[name] = log 71 | } 72 | 73 | return log 74 | } 75 | -------------------------------------------------------------------------------- /stats/kebab_case_converter.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Vitess Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package stats 18 | 19 | import ( 20 | "regexp" 21 | "strings" 22 | "sync" 23 | ) 24 | 25 | // toKebabCase produces a monitoring compliant name from the 26 | // original. It converts CamelCase to camel-case, 27 | // and CAMEL_CASE to camel-case. For numbers, it 28 | // converts 0.5 to v0_5. 29 | func toKebabCase(name string) (hyphenated string) { 30 | memoizer.Lock() 31 | defer memoizer.Unlock() 32 | if hyphenated = memoizer.memo[name]; hyphenated != "" { 33 | return hyphenated 34 | } 35 | hyphenated = name 36 | for _, converter := range kebabConverters { 37 | hyphenated = converter.re.ReplaceAllString(hyphenated, converter.repl) 38 | } 39 | hyphenated = strings.ToLower(hyphenated) 40 | memoizer.memo[name] = hyphenated 41 | return 42 | } 43 | 44 | var kebabConverters = []struct { 45 | re *regexp.Regexp 46 | repl string 47 | }{ 48 | // example: LC -> L-C (e.g. CamelCase -> Camel-Case). 49 | {regexp.MustCompile("([a-z])([A-Z])"), "$1-$2"}, 50 | // example: CCa -> C-Ca (e.g. CCamel -> C-Camel). 51 | {regexp.MustCompile("([A-Z])([A-Z][a-z])"), "$1-$2"}, 52 | {regexp.MustCompile("_"), "-"}, 53 | {regexp.MustCompile("\\."), "_"}, 54 | } 55 | 56 | var memoizer = memoizerType{ 57 | memo: make(map[string]string), 58 | } 59 | 60 | type memoizerType struct { 61 | sync.Mutex 62 | memo map[string]string 63 | } 64 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Go-Sharding 2 | 3 | 4 | ## 简介 5 | 数据库分库分表中间件，尽可能兼容 [ShardingSphere](https://github.com/apache/shardingsphere) 的 golang 实现， 6 | 基于小米 [Gaea](https://github.com/XiaoMi/Gaea) 魔改，但是路由算法支持 ShardingSphere 的 inline 表达式风格，而不是 Mycat/kingshard 这类晦涩而又不灵活的配置，移除多租户功能（配置太复杂了，部署多套即可） 7 | 8 | --- 9 | 10 | 已经决定全部重写，小米的代码仅参考逻辑！！！ 11 | 12 | > 最新的开发进度和代码看[**这里**](https://github.com/endink/go-sharding/tree/refactoring) 13 | 14 | --- 15 | ## 为什么造这个轮子 16 | 17 | 尝试了 ShardingSphere Proxy, 其有着糟糕的 insert 性能和 CPU 100% 问题，官方 issue 里 CPU 问题给出的回复是让升级到一个 5.0 alpha 版本试试，注意，是试试， 18 | 这样的处理实在无法让人接受，我们有大量 insert 性能操作，使用 ShardingSphere Proxy 后造成**数十倍**的性能下降，注意，是数十倍，而且 CPU 100%，通过 jprofiler 观察 19 | 其 Command 部分占用了所有的 CPU , 尤其是 SqlParser， 核心组件的严重性能问题除了官方更新，自己基本是无法解决的。 20 | 21 | ### 大胆猜测 ShardingSphere 性能问题的根源： 22 | 23 | 1. Netty 工作线程池中使用 Jdbc 这种同步客户端可能是其底层设计上的最大问题，ShardingSphere 并没有用 Netty 客户端自己实现一个简易的 Mysql 客户端，而是偷懒用了 jdbc hikaricp, 24 | 这种重重叠叠的线程池造成 CPU 居高不下, 至少从 CPU 角度看是不应该用 JDBC 的。 25 | 26 | 2. 直接 JDBC 的问题还在于从 MYSQL 请求到的二进制数据被 JDBC 解析一次，合并回去又要自己封装一次，相当于一份数据多次序列化反序列化，对于 Proxy 来说简直是噩梦， 27 | 多的不是，CPU 和字节拷贝的内存占用已经无形中多了很多，内存池化只能缓解，底层设计的邋遢上层基本无力优化，自己解析数据的好处在于处理合并时候可以直接操作二进制包，较少系统开销。 28 | 29 | 3. ShardingSphere 代码中并没有对列数据值的逻辑运算的优化编排，造成改写时候逻辑复杂，也就是缺少执行计划概念，这个问题最好的参考是 youtube 的 Vitess 项目。 30 | 31 | 32 | 总之，同样的场景，换用 Gaea 测试后性能恢复正常，但是其固化的分片方式又不太满足我们的需求，因此决定基于它造一个轮子。 33 | 34 | ## 全面的重构 35 | 36 | 小米的代码搬运了 kingshard、Vitess、tidb 等开源项目大量代码，查询计划部分通过表数组索引保存数据到装饰器造成代码难于阅读， 37 | Router 接口强绑定表索引分片方式，使得自己实现特殊分片逻辑成为不可能的任务，好在其解析 SQL 经过生产历练补漏了很多细节，具参考价值 38 | 39 | ## 改造任务进度 40 | 41 | - [x] 移除小米自己的 logger, 使用 uber zap 42 | - [x] 支持 Mysql 8 登录认证（jdbc 测试通过） 43 | - [x] 支持 Mysql Workbench 连接 44 | - [x] 移除粘贴过来的的 SqlParser 代码， 使用 go module 直接引用 tidb 项目，方便升级 45 | - [ ] 逻辑表呈现（使用管理工具时合并分片表为逻辑表） 46 | - [ ] 重构路由和查询计划，支持 ShardingSphere 配置风格 47 | - [ ] inline 表达式支持（进行中...） 48 | - [ ] range 路由支持 （进行中...） 49 | - [ ] 分片计划查看特定 SQL 支持 50 | - [ ] 支持分布式事务 51 | - [ ] 其他优化 52 | 53 | 54 | ## 当前可用性 55 | 56 | Main 分支保证可用性，已支持 Mysql 8.0.X 登录协议，dev 分支实现新特性，个人临时工作保存，可能长期无法正常编译 57 | -------------------------------------------------------------------------------- /models/slice.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Gaea Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package models 16 | 17 | import "errors" 18 | 19 | // Slice means source model of slice 20 | type Slice struct { 21 | Name string `json:"name"` 22 | UserName string `json:"user_name"` 23 | Password string `json:"password"` 24 | Master string `json:"master"` 25 | Slaves []string `json:"slaves"` 26 | StatisticSlaves []string `json:"statistic_slaves"` 27 | 28 | Capacity int `json:"capacity"` // connection pool capacity 29 | MaxCapacity int `json:"max_capacity"` // max connection pool capacity 30 | IdleTimeout int `json:"idle_timeout"` // close backend direct connection after idle_timeout,unit: seconds 31 | } 32 | 33 | func (s *Slice) verify() error { 34 | if s.Name == "" { 35 | return errors.New("must specify slice name") 36 | } 37 | 38 | if s.UserName == "" { 39 | return errors.New("missing user") 40 | } 41 | 42 | if s.Master == "" && len(s.Slaves) == 0 { 43 | return errors.New("both master and slaves empty") 44 | } 45 | 46 | for _, slave := range s.Slaves { 47 | if slave == "" { 48 | return errors.New("illegal slave addr") 49 | } 50 | } 51 | 52 | if s.Capacity <= 0 { 53 | return errors.New("connection pool capacity should be > 0") 54 | } 55 | 56 | if s.MaxCapacity <= 0 { 57 | return errors.New("max connection pool capactiy should be > 0") 58 | } 59 | 60 | return nil 61 | } 62 | -------------------------------------------------------------------------------- /util/hack/hack.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The kingshard Authors. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | // not use this file except in compliance with the License. You may obtain 5 | // a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package hack 16 | 17 | import ( 18 | "reflect" 19 | "strconv" 20 | "strings" 21 | "unsafe" 22 | ) 23 | 24 | // String no copy to change slice to string 25 | // use your own risk 26 | func String(b []byte) (s string) { 27 | pbytes := (*reflect.SliceHeader)(unsafe.Pointer(&b)) 28 | pstring := (*reflect.StringHeader)(unsafe.Pointer(&s)) 29 | pstring.Data = pbytes.Data 30 | pstring.Len = pbytes.Len 31 | return 32 | } 33 | 34 | // Slice no copy to change string to slice 35 | // use your own risk 36 | func Slice(s string) (b []byte) { 37 | pbytes := (*reflect.SliceHeader)(unsafe.Pointer(&b)) 38 | pstring := (*reflect.StringHeader)(unsafe.Pointer(&s)) 39 | pbytes.Data = pstring.Data 40 | pbytes.Len = pstring.Len 41 | pbytes.Cap = pstring.Len 42 | return 43 | } 44 | 45 | // IsSQLSep check if parser seperation 46 | func IsSQLSep(r rune) bool { 47 | return r == ' ' || r == ',' || 48 | r == '\t' || r == '/' || 49 | r == '\n' || r == '\r' 50 | } 51 | 52 | // ArrayToString translate array to string 53 | func ArrayToString(array []int) string { 54 | if len(array) == 0 { 55 | return "" 56 | } 57 | var strArray []string 58 | for _, v := range array { 59 | strArray = append(strArray, strconv.FormatInt(int64(v), 10)) 60 | } 61 | 62 | return strings.Join(strArray, ", ") 63 | } 64 | 65 | // Abs int64 abs 66 | func Abs(n int64) int64 { 67 | y := n >> 63 68 | return (n ^ y) - y 69 | } 70 | -------------------------------------------------------------------------------- /misc/git/hooks/gofmt: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2017 Google Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # git gofmt pre-commit hook 17 | # 18 | # To use, store as .git/hooks/pre-commit inside your repository and make sure 19 | # it has execute permissions. 20 | # 21 | # This script does not handle file names that contain spaces. 22 | gofiles=$(git diff --cached --name-only --diff-filter=d | grep '.go$') 23 | 24 | [ -z "$gofiles" ] && exit 0 25 | unformatted=$(gofmt -s -l $gofiles 2>&1) 26 | [ -z "$unformatted" ] && exit 0 27 | 28 | # Some files are not gofmt'd. Print command to fix them and fail. 29 | 30 | # Deduplicate files first in case a file has multiple errors. 31 | files=$( 32 | # Split the "gofmt" output on newlines only. 33 | OLDIFS=$IFS 34 | IFS=' 35 | ' 36 | for line in $unformatted; do 37 | # Strip everything after the first ':', including it. 38 | # Example output for $line: 39 | # go/vt/vttablet/tabletserver/txserializer/tx_serializer_test.go:241:60: expected ';', found 'IDENT' wg 40 | echo ${line/:*/} 41 | done | 42 | # Remove duplicates. 43 | sort -u 44 | IFS=$OLDIFS 45 | ) 46 | 47 | echo >&2 48 | echo >&2 "Go files must be formatted with gofmt. Please run:" 49 | echo >&2 50 | echo >&2 -n " gofmt -s -w" 51 | 52 | for f in $files; do 53 | # Print " \" after the "gofmt" above and each filename (except for the last one). 54 | echo >&2 " \\" 55 | echo >&2 -n " $PWD/$f" 56 | done 57 | echo >&2 58 | 59 | echo >&2 60 | echo >&2 "If gofmt fails and outputs errors, you have to fix them manually." 61 | echo >&2 62 | 63 | exit 1 64 | -------------------------------------------------------------------------------- /models/cc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Gaea Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package models 16 | 17 | import ( 18 | "strings" 19 | 20 | "github.com/go-ini/ini" 21 | ) 22 | 23 | // CCConfig means gaea cc source 24 | type CCConfig struct { 25 | Addr string `ini:"addr"` 26 | AdminUserName string `ini:"admin_username"` 27 | AdminPassword string `ini:"admin_password"` 28 | ProxyUserName string `ini:"proxy_username"` 29 | ProxyPassword string `ini:"proxy_password"` 30 | // etcd 相关配置 31 | CoordinatorAddr string `ini:"coordinator_addr"` 32 | CoordinatorRoot string `ini:"coordinator_root"` 33 | UserName string `ini:"username"` 34 | Password string `ini:"password"` 35 | 36 | DefaultCluster string `ini:"default_cluster"` 37 | 38 | LogPath string `ini:"log_path"` 39 | LogLevel string `ini:"log_level"` 40 | LogFileName string `ini:"log_filename"` 41 | LogOutput string `ini:"log_output"` 42 | 43 | EncryptKey string `ini:"encrypt_key"` 44 | } 45 | 46 | // ParseCCConfig parser gaea cc source from file 47 | func ParseCCConfig(cfgFile string) (*CCConfig, error) { 48 | cfg, err := ini.Load(cfgFile) 49 | 50 | if err != nil { 51 | return nil, err 52 | } 53 | 54 | ccConfig := new(CCConfig) 55 | err = cfg.MapTo(ccConfig) 56 | if ccConfig.DefaultCluster == "" && ccConfig.CoordinatorRoot != "" { 57 | ccConfig.DefaultCluster = strings.TrimPrefix(ccConfig.CoordinatorRoot, "/") 58 | } 59 | return ccConfig, err 60 | } 61 | 62 | // Verify verify cc source 63 | func (cc *CCConfig) Verify() error { 64 | return nil 65 | } 66 | -------------------------------------------------------------------------------- /cmd/gaea-cc/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Gaea Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "flag" 19 | "fmt" 20 | "github.com/XiaoMi/Gaea/cc/proxy" 21 | "os" 22 | "os/signal" 23 | "sync" 24 | "syscall" 25 | 26 | "github.com/XiaoMi/Gaea/cc" 27 | "github.com/XiaoMi/Gaea/core" 28 | 29 | "github.com/XiaoMi/Gaea/models" 30 | ) 31 | 32 | var ccConfigFile = flag.String("c", "./etc/gaea_cc.ini", "gaea cc配置") 33 | var info = flag.Bool("info", false, "show info of gaea-cc") 34 | 35 | func main() { 36 | flag.Parse() 37 | if *info { 38 | fmt.Printf("Build Version Information:%s\n", core.Info.LongForm()) 39 | return 40 | } 41 | 42 | fmt.Printf("Build Version Information:%s\n", core.Info.LongForm()) 43 | 44 | // 初始化配置 45 | ccConfig, err := models.ParseCCConfig(*ccConfigFile) 46 | if err != nil { 47 | fmt.Printf("parse cc source failed, %v\n", err) 48 | } 49 | 50 | // 构造服务实例 51 | s, err := cc.NewServer(ccConfig.Addr, ccConfig) 52 | if err != nil { 53 | proxy.ControllerLogger.Fatalf("create server failed, %v", err) 54 | return 55 | } 56 | 57 | c := make(chan os.Signal, 1) 58 | signal.Notify(c, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGPIPE) 59 | wg := new(sync.WaitGroup) 60 | wg.Add(1) 61 | go func() { 62 | defer wg.Done() 63 | for { 64 | sig := <-c 65 | if sig == syscall.SIGINT || sig == syscall.SIGTERM || sig == syscall.SIGQUIT { 66 | proxy.ControllerLogger.Infof("got signal %d, quit", sig) 67 | s.Close() 68 | return 69 | } 70 | proxy.ControllerLogger.Infof("ignore signal %d", sig) 71 | } 72 | }() 73 | 74 | s.Run() 75 | wg.Wait() 76 | } 77 | -------------------------------------------------------------------------------- /stats/snake_case_converter.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Vitess Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package stats 18 | 19 | import ( 20 | "regexp" 21 | "strings" 22 | ) 23 | 24 | // GetSnakeName calls toSnakeName on the passed in string. It produces 25 | // a snake-cased name from the provided camel-cased name. 26 | // It memoizes the transformation and returns the stored result if available. 27 | func GetSnakeName(name string) string { 28 | return toSnakeCase(name) 29 | } 30 | 31 | // toSnakeCase produces a monitoring compliant name from the original. 32 | // For systems (like Prometheus) that ask for snake-case names. 33 | // It converts CamelCase to camel_case, and CAMEL_CASE to camel_case. 34 | // For numbers, it converts 0.5 to v0_5. 35 | func toSnakeCase(name string) (hyphenated string) { 36 | snakeMemoizer.Lock() 37 | defer snakeMemoizer.Unlock() 38 | if hyphenated = snakeMemoizer.memo[name]; hyphenated != "" { 39 | return hyphenated 40 | } 41 | hyphenated = name 42 | for _, converter := range snakeConverters { 43 | hyphenated = converter.re.ReplaceAllString(hyphenated, converter.repl) 44 | } 45 | hyphenated = strings.ToLower(hyphenated) 46 | snakeMemoizer.memo[name] = hyphenated 47 | return 48 | } 49 | 50 | var snakeConverters = []struct { 51 | re *regexp.Regexp 52 | repl string 53 | }{ 54 | // example: LC -> L_C (e.g. CamelCase -> Camel_Case). 55 | {regexp.MustCompile("([a-z])([A-Z])"), "${1}_${2}"}, 56 | // example: CCa -> C_Ca (e.g. CCamel -> C_Camel). 57 | {regexp.MustCompile("([A-Z])([A-Z][a-z])"), "${1}_${2}"}, 58 | {regexp.MustCompile("\\."), "_"}, 59 | {regexp.MustCompile("-"), "_"}, 60 | } 61 | 62 | var snakeMemoizer = memoizerType{ 63 | memo: make(map[string]string), 64 | } 65 | -------------------------------------------------------------------------------- /util/ip_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The kingshard Authors. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | // not use this file except in compliance with the License. You may obtain 5 | // a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | package util 15 | 16 | import ( 17 | "fmt" 18 | "net" 19 | "testing" 20 | ) 21 | 22 | func TestParseIP(t *testing.T) { 23 | ip := net.ParseIP("abcdefg") 24 | fmt.Println(ip) 25 | fmt.Println(ip.String()) 26 | } 27 | 28 | func TestCreateIPInfoIPSuccess(t *testing.T) { 29 | addr := "127.0.0.1" 30 | info, err := ParseIPInfo(addr) 31 | if err != nil { 32 | t.FailNow() 33 | } 34 | if info.isIPNet { 35 | t.FailNow() 36 | } 37 | if addr != info.info { 38 | t.FailNow() 39 | } 40 | if addr != info.ip.String() { 41 | t.FailNow() 42 | } 43 | } 44 | 45 | func TestCreateIPInfoIPError(t *testing.T) { 46 | addr := "127.255.256.1" 47 | if _, err := ParseIPInfo(addr); err == nil { 48 | t.FailNow() 49 | } 50 | } 51 | 52 | func TestCreateIPInfoIPError2(t *testing.T) { 53 | addr := "abcdefg" 54 | if _, err := ParseIPInfo(addr); err == nil { 55 | t.FailNow() 56 | } 57 | } 58 | 59 | func TestCreateIPInfoIPNetSuccess(t *testing.T) { 60 | addr := "192.168.122.1/24" 61 | netAddr := "192.168.122.0/24" 62 | info, err := ParseIPInfo(addr) 63 | if err != nil { 64 | t.FailNow() 65 | } 66 | if !info.isIPNet { 67 | t.FailNow() 68 | } 69 | if addr != info.info { 70 | t.FailNow() 71 | } 72 | if netAddr != info.ipNet.String() { 73 | t.FailNow() 74 | } 75 | } 76 | 77 | func TestCreateIPInfoIPNetError(t *testing.T) { 78 | addr := "192.168.122.1/" 79 | if _, err := ParseIPInfo(addr); err == nil { 80 | t.FailNow() 81 | } 82 | } 83 | 84 | func TestCreateIPInfoIPNetError2(t *testing.T) { 85 | addr := "192.168.122.1/35" 86 | if _, err := ParseIPInfo(addr); err == nil { 87 | t.FailNow() 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /util/requests/api_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Gaea Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package requests 16 | 17 | import ( 18 | "encoding/json" 19 | "net/http/httputil" 20 | "testing" 21 | ) 22 | 23 | func TestAddParameters(t *testing.T) { 24 | url := "http://127.0.0.1:8080" 25 | expect := "http://127.0.0.1:8080?a=test1&b=test2" 26 | params := make(map[string]string, 2) 27 | params["a"] = "test1" 28 | params["b"] = "test2" 29 | r := AddParameters(url, params) 30 | if r != expect { 31 | t.Errorf("test AddParameters failed, expect: %s, result: %s", expect, r) 32 | } 33 | } 34 | 35 | func TestBuildHTTPRequest(t *testing.T) { 36 | url := "http://127.0.0.1:8080" 37 | method := Put 38 | 39 | header := make(map[string]string, 2) 40 | header["Content-Type"] = "application/json" 41 | 42 | params := make(map[string]string, 2) 43 | params["a"] = "test1" 44 | params["b"] = "test2" 45 | 46 | type ReqBody struct { 47 | Field1 string `json:"field_1"` 48 | Field2 int `json:"field_2"` 49 | } 50 | rb := ReqBody{Field1: "contest1", Field2: 99} 51 | body, _ := json.Marshal(rb) 52 | 53 | req := NewRequest(url, method, header, params, body) 54 | req.SetBasicAuth("test", "test") 55 | httpReq, err := BuildHTTPRequest(req) 56 | if err != nil { 57 | t.Errorf("test BuildHTTPRequest failed, %v", err) 58 | } 59 | 60 | httpReqDump, err := httputil.DumpRequest(httpReq, true) 61 | if err != nil { 62 | t.Errorf("test BuildHTTPRequest failed, %v", err) 63 | } 64 | t.Logf("dumped http request: %s\n", string(httpReqDump)) 65 | } 66 | 67 | func TestEncodeURL(t *testing.T) { 68 | url := EncodeURL("127.0.0.1:8080", "/test/test/:%d", 1) 69 | if url != "http://127.0.0.1:8080/test/test/:1" { 70 | t.Fatalf("test EncodeURL failed, %s", url) 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /util/bucketpool/bucketpool.go: -------------------------------------------------------------------------------- 1 | package bucketpool 2 | 3 | import ( 4 | "math" 5 | "sync" 6 | ) 7 | 8 | type sizedPool struct { 9 | size int 10 | pool sync.Pool 11 | } 12 | 13 | func newSizedPool(size int) *sizedPool { 14 | return &sizedPool{ 15 | size: size, 16 | pool: sync.Pool{ 17 | New: func() interface{} { return makeSlicePointer(size) }, 18 | }, 19 | } 20 | } 21 | 22 | // Pool is actually multiple pools which store buffers of specific size. 23 | // i.e. it can be three pools which return buffers 32K, 64K and 128K. 24 | type Pool struct { 25 | minSize int 26 | maxSize int 27 | pools []*sizedPool 28 | } 29 | 30 | // New returns Pool which has buckets from minSize to maxSize. 31 | // Buckets increase with the power of two, i.e with multiplier 2: [2b, 4b, 16b, ... , 1024b] 32 | // Last pool will always be capped to maxSize. 33 | func New(minSize, maxSize int) *Pool { 34 | if maxSize < minSize { 35 | panic("maxSize can't be less than minSize") 36 | } 37 | const multiplier = 2 38 | var pools []*sizedPool 39 | curSize := minSize 40 | for curSize < maxSize { 41 | pools = append(pools, newSizedPool(curSize)) 42 | curSize *= multiplier 43 | } 44 | pools = append(pools, newSizedPool(maxSize)) 45 | return &Pool{ 46 | minSize: minSize, 47 | maxSize: maxSize, 48 | pools: pools, 49 | } 50 | } 51 | 52 | func (p *Pool) findPool(size int) *sizedPool { 53 | if size > p.maxSize { 54 | return nil 55 | } 56 | idx := int(math.Ceil(math.Log2(float64(size) / float64(p.minSize)))) 57 | if idx < 0 { 58 | idx = 0 59 | } 60 | if idx > len(p.pools)-1 { 61 | return nil 62 | } 63 | return p.pools[idx] 64 | } 65 | 66 | // Get returns pointer to []byte which has len size. 67 | // If there is no bucket with buffers >= size, slice will be allocated. 68 | func (p *Pool) Get(size int) *[]byte { 69 | sp := p.findPool(size) 70 | if sp == nil { 71 | return makeSlicePointer(size) 72 | } 73 | buf := sp.pool.Get().(*[]byte) 74 | *buf = (*buf)[:size] 75 | return buf 76 | } 77 | 78 | // Put returns pointer to slice to some bucket. Discards slice for which there is no bucket 79 | func (p *Pool) Put(b *[]byte) { 80 | sp := p.findPool(cap(*b)) 81 | if sp == nil { 82 | return 83 | } 84 | *b = (*b)[:cap(*b)] 85 | sp.pool.Put(b) 86 | } 87 | 88 | func makeSlicePointer(size int) *[]byte { 89 | data := make([]byte, size) 90 | return &data 91 | } 92 | -------------------------------------------------------------------------------- /util/timer/timer_flaky_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 Google Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package timer 18 | 19 | import ( 20 | "sync/atomic" 21 | "testing" 22 | "time" 23 | ) 24 | 25 | const ( 26 | half = time.Duration(500e5) 27 | quarter = time.Duration(250e5) 28 | tenth = time.Duration(100e5) 29 | ) 30 | 31 | var numcalls int32 32 | 33 | func f() { 34 | atomic.AddInt32(&numcalls, 1) 35 | } 36 | 37 | func TestWait(t *testing.T) { 38 | atomic.StoreInt32(&numcalls, 0) 39 | timer := NewTimer(quarter) 40 | timer.Start(f) 41 | defer timer.Stop() 42 | time.Sleep(tenth) 43 | if atomic.LoadInt32(&numcalls) != 0 { 44 | t.Errorf("want 0, received %v", numcalls) 45 | } 46 | time.Sleep(quarter) 47 | if atomic.LoadInt32(&numcalls) != 1 { 48 | t.Errorf("want 1, received %v", numcalls) 49 | } 50 | time.Sleep(quarter) 51 | if atomic.LoadInt32(&numcalls) != 2 { 52 | t.Errorf("want 1, received %v", numcalls) 53 | } 54 | } 55 | 56 | func TestReset(t *testing.T) { 57 | atomic.StoreInt32(&numcalls, 0) 58 | timer := NewTimer(half) 59 | timer.Start(f) 60 | defer timer.Stop() 61 | timer.SetInterval(quarter) 62 | time.Sleep(tenth) 63 | if atomic.LoadInt32(&numcalls) != 0 { 64 | t.Errorf("want 0, received %v", numcalls) 65 | } 66 | time.Sleep(quarter) 67 | if atomic.LoadInt32(&numcalls) != 1 { 68 | t.Errorf("want 1, received %v", numcalls) 69 | } 70 | } 71 | 72 | func TestIndefinite(t *testing.T) { 73 | atomic.StoreInt32(&numcalls, 0) 74 | timer := NewTimer(0) 75 | timer.Start(f) 76 | defer timer.Stop() 77 | timer.TriggerAfter(quarter) 78 | time.Sleep(tenth) 79 | if atomic.LoadInt32(&numcalls) != 0 { 80 | t.Errorf("want 0, received %v", numcalls) 81 | } 82 | time.Sleep(quarter) 83 | if atomic.LoadInt32(&numcalls) != 1 { 84 | t.Errorf("want 1, received %v", numcalls) 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /util/util.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Gaea Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package util 16 | 17 | import ( 18 | "fmt" 19 | "os" 20 | "runtime" 21 | "strconv" 22 | "strings" 23 | "sync/atomic" 24 | "time" 25 | ) 26 | 27 | var LineSeparator string = "\n" 28 | 29 | func init() { 30 | if IsWindows() { 31 | LineSeparator = "\r\n" 32 | } 33 | } 34 | 35 | // BoolIndex rolled array switch mark 36 | type BoolIndex struct { 37 | index int32 38 | } 39 | 40 | // Set set index value 41 | func (b *BoolIndex) Set(index bool) { 42 | if index { 43 | atomic.StoreInt32(&b.index, 1) 44 | } else { 45 | atomic.StoreInt32(&b.index, 0) 46 | } 47 | } 48 | 49 | // Get return current, next, current bool value 50 | func (b *BoolIndex) Get() (int32, int32, bool) { 51 | index := atomic.LoadInt32(&b.index) 52 | if index == 1 { 53 | return 1, 0, true 54 | } 55 | return 0, 1, false 56 | } 57 | 58 | // ItoString interface to string 59 | func ItoString(a interface{}) (bool, string) { 60 | switch a.(type) { 61 | case nil: 62 | return false, "NULL" 63 | case []byte: 64 | return true, string(a.([]byte)) 65 | default: 66 | return false, fmt.Sprintf("%v", a) 67 | } 68 | } 69 | 70 | // Int2TimeDuration convert int to Time.Duration 71 | func Int2TimeDuration(t int) (time.Duration, error) { 72 | tmp := strconv.Itoa(t) 73 | tmp = tmp + "s" 74 | idleTimeout, err := time.ParseDuration(tmp) 75 | if err != nil { 76 | return 0, err 77 | 78 | } 79 | return idleTimeout, nil 80 | } 81 | 82 | func IsWindows() bool { 83 | sysType := runtime.GOOS 84 | 85 | return strings.ToLower(sysType) == "windows" 86 | 87 | } 88 | 89 | func FileExists(name string) bool { 90 | if _, err := os.Stat(name); err != nil { 91 | if os.IsNotExist(err) { 92 | return false 93 | } 94 | } 95 | return true 96 | } 97 | -------------------------------------------------------------------------------- /util/murmur.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Gaea Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package util 16 | 17 | const ( 18 | c1 = 0xcc9e2d51 19 | c2 = 0x1b873593 20 | ) 21 | 22 | // MurmurHash copy from guava Murmur3_32HashFunction 23 | type MurmurHash struct { 24 | seed int 25 | } 26 | 27 | // NewMurmurHash constructor of MurmurHash 28 | func NewMurmurHash(seed int) *MurmurHash { 29 | return &MurmurHash{ 30 | seed: seed, 31 | } 32 | } 33 | 34 | // HashUnencodedChars check if has unencoded chars 35 | func (m *MurmurHash) HashUnencodedChars(inputStr string) int { 36 | input := []rune(inputStr) 37 | h1 := int32(m.seed) 38 | 39 | for i := 1; i < len(input); i += 2 { 40 | k1 := int32(input[i-1] | (input[i] << 16)) 41 | k1 = mixK1(k1) 42 | h1 = mixH1(h1, k1) 43 | } 44 | 45 | if len(input)&1 == 1 { 46 | k1 := int32(input[len(input)-1]) 47 | k1 = mixK1(k1) 48 | h1 ^= k1 49 | } 50 | 51 | return int(fmix(h1, int32(2*len(input)))) 52 | } 53 | 54 | func mixK1(k1 int32) int32 { 55 | k1_64 := int64(k1) 56 | k1_64 *= c1 57 | k1 = int32(k1_64) 58 | k1 = rotateLeft(k1, 15) 59 | k1_64 = int64(k1) 60 | k1_64 *= c2 61 | return int32(k1_64) 62 | } 63 | 64 | func mixH1(h1, k1 int32) int32 { 65 | h1 ^= k1 66 | h1 = rotateLeft(h1, 13) 67 | h1_64 := int64(h1) 68 | h1_64 = h1_64*5 + 0xe6546b64 69 | return int32(h1_64) 70 | } 71 | 72 | func fmix(h1, length int32) int32 { 73 | uh1 := uint32(h1) 74 | ulength := uint32(length) 75 | uh1 ^= ulength 76 | uh1 ^= uh1 >> 16 77 | uh1 *= 0x85ebca6b 78 | uh1 ^= uh1 >> 13 79 | uh1 *= 0xc2b2ae35 80 | uh1 ^= uh1 >> 16 81 | return int32(uh1) 82 | } 83 | 84 | func rotateLeft(i, distance int32) int32 { 85 | ui := uint32(i) 86 | udistance := uint32(distance) 87 | a1 := ui << udistance 88 | b1 := ui >> (32 - udistance) 89 | return int32(a1 | b1) 90 | } 91 | -------------------------------------------------------------------------------- /provider/registry.go: -------------------------------------------------------------------------------- 1 | package provider 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "sync" 7 | ) 8 | 9 | var onceReg sync.Once 10 | var instance Registry 11 | 12 | type Registry interface { 13 | TryLoad(tp Type, name string) (Provider, bool) 14 | Load(tp Type, name string) Provider 15 | Register(tp Type, provider Provider) error 16 | LoadOrStore(tp Type, name string, creation func() Provider) (actual Provider, loaded bool) 17 | LoadAndDelete(tp Type, name string) (value Provider, loaded bool) 18 | Delete(tp Type, name string) 19 | } 20 | 21 | func DefaultRegistry() Registry { 22 | onceReg.Do(func() { 23 | instance = ®istry{sync.Map{}} 24 | }) 25 | return instance 26 | } 27 | 28 | type registry struct { 29 | mp sync.Map 30 | } 31 | 32 | func getFullName(tp Type, name string) string { 33 | return fmt.Sprintf("%d:%s", int(tp), name) 34 | } 35 | 36 | func (r *registry) TryLoad(tp Type, name string) (Provider, bool) { 37 | fullName := getFullName(tp, name) 38 | v, ok := r.mp.Load(fullName) 39 | if ok { 40 | p, ok := v.(Provider) 41 | return p, ok 42 | } 43 | return nil, ok 44 | } 45 | 46 | func (r *registry) Load(tp Type, name string) Provider { 47 | fullName := getFullName(tp, name) 48 | v, ok := r.mp.Load(fullName) 49 | if !ok { 50 | panic(fmt.Errorf("provider named '%s' was not found", name)) 51 | } 52 | return v.(Provider) 53 | } 54 | 55 | func (r *registry) Register(tp Type, provider Provider) error { 56 | if provider == nil { 57 | return errors.New("provider can not be null") 58 | } 59 | n := provider.GetName() 60 | fullName := getFullName(tp, n) 61 | if len(n) > 0 { 62 | r.mp.Store(fullName, provider) 63 | return nil 64 | } else { 65 | return errors.New("provider name can not be empty") 66 | } 67 | } 68 | 69 | func (r *registry) LoadOrStore(tp Type, name string, creation func() Provider) (actual Provider, loaded bool) { 70 | v, ok := r.TryLoad(tp, name) 71 | if !ok { 72 | v = creation() 73 | r.mp.Store(name, v) 74 | } 75 | return v, ok 76 | } 77 | 78 | func (r *registry) LoadAndDelete(tp Type, name string) (value Provider, loaded bool) { 79 | fullName := getFullName(tp, name) 80 | v, ok := r.mp.LoadAndDelete(fullName) 81 | if ok { 82 | p, ok := v.(Provider) 83 | return p, ok 84 | } else { 85 | return nil, ok 86 | } 87 | } 88 | 89 | func (r *registry) Delete(tp Type, name string) { 90 | fullName := getFullName(tp, name) 91 | r.mp.Delete(fullName) 92 | } 93 | -------------------------------------------------------------------------------- /mysql/util_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The kingshard Authors. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | // not use this file except in compliance with the License. You may obtain 5 | // a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | // Copyright 2019 The Gaea Authors. All Rights Reserved. 16 | // 17 | // Licensed under the Apache License, Version 2.0 (the "License"); 18 | // you may not use this file except in compliance with the License. 19 | // You may obtain a copy of the License at 20 | // 21 | // http://www.apache.org/licenses/LICENSE-2.0 22 | // 23 | // Unless required by applicable law or agreed to in writing, software 24 | // distributed under the License is distributed on an "AS IS" BASIS, 25 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 26 | // See the License for the specific language governing permissions and 27 | // limitations under the License. 28 | 29 | package mysql 30 | 31 | import ( 32 | "encoding/hex" 33 | "testing" 34 | 35 | "github.com/XiaoMi/Gaea/util/hack" 36 | ) 37 | 38 | func TestCalcPassword(t *testing.T) { 39 | /* 40 | // **** JDBC **** 41 | seed: 42 | @jx=d_3z42;sS$YrS)p| 43 | hex: 44 | 406a783d645f337a34323b73532459725329707c 45 | pass: 46 | kingshard 47 | scramble: 48 | fbc71db5ac3d7b51048d1a1d88c1677f34bcca11 49 | */ 50 | test, _ := RandomBuf(20) 51 | hexTest := hex.EncodeToString(test) 52 | t.Logf("rnd seed: %s, %s", hack.String(test), hexTest) 53 | 54 | seed := hack.Slice("@jx=d_3z42;sS$YrS)p|") 55 | hexSeed := hex.EncodeToString(seed) 56 | 57 | t.Logf("seed: %s equal %s, pass: %v", "406a783d645f337a34323b73532459725329707c", hexSeed, "406a783d645f337a34323b73532459725329707c" == hexSeed) 58 | scramble := CalcPassword(seed, hack.Slice("kingshard")) 59 | 60 | hexScramble := hex.EncodeToString(scramble) 61 | t.Logf("scramble: %s equal %s, pass: %v", "fbc71db5ac3d7b51048d1a1d88c1677f34bcca11", hexScramble, "fbc71db5ac3d7b51048d1a1d88c1677f34bcca11" == hexScramble) 62 | } 63 | -------------------------------------------------------------------------------- /util/sync2/semaphore.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 Google Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package sync2 18 | 19 | // What's in a name? Channels have all you need to emulate a counting 20 | // semaphore with a boatload of extra functionality. However, in some 21 | // cases, you just want a familiar API. 22 | 23 | import ( 24 | "time" 25 | ) 26 | 27 | // Semaphore is a counting semaphore with the option to 28 | // specify a timeout. 29 | type Semaphore struct { 30 | slots chan struct{} 31 | timeout time.Duration 32 | } 33 | 34 | // NewSemaphore creates a Semaphore. The count parameter must be a positive 35 | // number. A timeout of zero means that there is no timeout. 36 | func NewSemaphore(count int, timeout time.Duration) *Semaphore { 37 | sem := &Semaphore{ 38 | slots: make(chan struct{}, count), 39 | timeout: timeout, 40 | } 41 | for i := 0; i < count; i++ { 42 | sem.slots <- struct{}{} 43 | } 44 | return sem 45 | } 46 | 47 | // Acquire returns true on successful acquisition, and 48 | // false on a timeout. 49 | func (sem *Semaphore) Acquire() bool { 50 | if sem.timeout == 0 { 51 | <-sem.slots 52 | return true 53 | } 54 | tm := time.NewTimer(sem.timeout) 55 | defer tm.Stop() 56 | select { 57 | case <-sem.slots: 58 | return true 59 | case <-tm.C: 60 | return false 61 | } 62 | } 63 | 64 | // TryAcquire acquires a semaphore if it's immediately available. 65 | // It returns false otherwise. 66 | func (sem *Semaphore) TryAcquire() bool { 67 | select { 68 | case <-sem.slots: 69 | return true 70 | default: 71 | return false 72 | } 73 | } 74 | 75 | // Release releases the acquired semaphore. You must 76 | // not release more than the number of semaphores you've 77 | // acquired. 78 | func (sem *Semaphore) Release() { 79 | sem.slots <- struct{}{} 80 | } 81 | 82 | // Size returns the current number of available slots. 83 | func (sem *Semaphore) Size() int { 84 | return len(sem.slots) 85 | } 86 | -------------------------------------------------------------------------------- /stats/histogram_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 Google Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package stats 18 | 19 | import ( 20 | "expvar" 21 | "testing" 22 | ) 23 | 24 | func TestHistogram(t *testing.T) { 25 | clear() 26 | h := NewHistogram("hist1", "help", []int64{1, 5}) 27 | for i := 0; i < 10; i++ { 28 | h.Add(int64(i)) 29 | } 30 | want := `{"1": 2, "5": 6, "inf": 10, "Count": 10, "Total": 45}` 31 | if h.String() != want { 32 | t.Errorf("got %v, want %v", h.String(), want) 33 | } 34 | counts := h.Counts() 35 | counts["Count"] = h.Count() 36 | counts["Total"] = h.Total() 37 | for k, want := range map[string]int64{ 38 | "1": 2, 39 | "5": 4, 40 | "inf": 4, 41 | "Count": 10, 42 | "Total": 45, 43 | } { 44 | if got := counts[k]; got != want { 45 | t.Errorf("histogram counts [%v]: got %d, want %d", k, got, want) 46 | } 47 | } 48 | if got, want := h.CountLabel(), "Count"; got != want { 49 | t.Errorf("got %v, want %v", got, want) 50 | } 51 | if got, want := h.TotalLabel(), "Total"; got != want { 52 | t.Errorf("got %v, want %v", got, want) 53 | } 54 | } 55 | 56 | func TestGenericHistogram(t *testing.T) { 57 | clear() 58 | h := NewGenericHistogram( 59 | "histgen", 60 | "help", 61 | []int64{1, 5}, 62 | []string{"one", "five", "max"}, 63 | "count", 64 | "total", 65 | ) 66 | want := `{"one": 0, "five": 0, "max": 0, "count": 0, "total": 0}` 67 | if got := h.String(); got != want { 68 | t.Errorf("got %v, want %v", got, want) 69 | } 70 | } 71 | 72 | func TestHistogramHook(t *testing.T) { 73 | var gotname string 74 | var gotv *Histogram 75 | clear() 76 | Register(func(name string, v expvar.Var) { 77 | gotname = name 78 | gotv = v.(*Histogram) 79 | }) 80 | 81 | name := "hist2" 82 | v := NewHistogram(name, "help", []int64{1}) 83 | if gotname != name { 84 | t.Errorf("got %v; want %v", gotname, name) 85 | } 86 | if gotv != v { 87 | t.Errorf("got %#v, want %#v", gotv, v) 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /models/user.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Gaea Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package models 16 | 17 | import ( 18 | "errors" 19 | "fmt" 20 | "strings" 21 | ) 22 | 23 | // 用户只读标识 24 | // read write privileges 25 | const ( 26 | // 只读 27 | ReadOnly = 1 28 | // 可读可写 29 | ReadWrite = 2 30 | ) 31 | 32 | // 用户读写分离标识 33 | const ( 34 | // NoReadWriteSplit 非读写分离 35 | NoReadWriteSplit = 0 36 | // ReadWriteSplit 读写分离 37 | ReadWriteSplit = 1 38 | // StatisticUser 统计用户 39 | StatisticUser = 1 40 | ) 41 | 42 | // User meand user struct 43 | type User struct { 44 | UserName string `json:"user_name"` 45 | Password string `json:"password"` 46 | Namespace string `json:"namespace"` 47 | RWFlag int `json:"rw_flag"` //1: 只读 2:读写 48 | RWSplit int `json:"rw_split"` //0: 不采用读写分离 1:读写分离 49 | OtherProperty int `json:"other_property"` // 1:统计用户 50 | } 51 | 52 | func (p *User) verify() error { 53 | if p.UserName == "" { 54 | return errors.New("missing user name") 55 | } 56 | p.UserName = strings.TrimSpace(p.UserName) 57 | 58 | if p.Namespace == "" { 59 | return fmt.Errorf("missing namespace name: %s", p.UserName) 60 | } 61 | p.Namespace = strings.TrimSpace(p.Namespace) 62 | 63 | if p.Password == "" { 64 | return fmt.Errorf("missing password: [%s]%s", p.Namespace, p.UserName) 65 | } 66 | p.Password = strings.TrimSpace(p.Password) 67 | 68 | if p.RWFlag != ReadOnly && p.RWFlag != ReadWrite { 69 | return fmt.Errorf("invalid RWFlag, user: %s, rwflag: %d", p.UserName, p.RWFlag) 70 | } 71 | 72 | if p.RWSplit != NoReadWriteSplit && p.RWSplit != ReadWriteSplit { 73 | return fmt.Errorf("invalid RWSplit, user: %s, rwsplit: %d", p.UserName, p.RWSplit) 74 | } 75 | 76 | if p.OtherProperty != StatisticUser && p.OtherProperty != 0 { 77 | return fmt.Errorf("invalid other property, user: %s, %d", p.UserName, p.OtherProperty) 78 | } 79 | 80 | return nil 81 | } 82 | -------------------------------------------------------------------------------- /misc/git/hooks/golint: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2017 Google Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # git golint pre-commit hook 17 | # 18 | # To use, store as .git/hooks/pre-commit inside your repository and make sure 19 | # it has execute permissions. 20 | 21 | if [ -z "$GOPATH" ]; then 22 | echo "ERROR: pre-commit hook for golint: \$GOPATH is empty. Please run 'source dev.env' to set the correct \$GOPATH." 23 | exit 1 24 | fi 25 | 26 | if [ -z "$(which golint)" ]; then 27 | echo "golint not found, please run: go get github.com/golang/lint/golint" 28 | exit 1 29 | fi 30 | 31 | # This script does not handle file names that contain spaces. 32 | gofiles=$(git diff --cached --name-only --diff-filter=d | grep '.go$') 33 | echo $gofiles 34 | 35 | errors= 36 | 37 | # Run on one file at a time because a single invocation of golint 38 | # with multiple files requires the files to all be in one package. 39 | gofiles_with_warnings=() 40 | for gofile in $gofiles 41 | do 42 | errcount=$(golint $gofile | wc -l) 43 | if [ "$errcount" -gt "0" ]; then 44 | errors=YES 45 | echo "$errcount suggestions for:" 46 | echo "golint $gofile" 47 | gofiles_with_warnings+=($gofile) 48 | fi 49 | done 50 | 51 | [ -z "$errors" ] && exit 0 52 | 53 | # git doesn't give us access to user input, so let's steal it. 54 | exec < /dev/tty 55 | if [[ $? -eq 0 ]]; then 56 | # interactive shell. Prompt the user. 57 | echo 58 | echo "Lint suggestions were found. They're not enforced, but we're pausing" 59 | echo "to let you know before they get clobbered in the scrollback buffer." 60 | echo 61 | read -r -p 'Press enter to cancel, "s" to step through the warnings or type "ack" to continue: ' 62 | if [ "$REPLY" = "ack" ]; then 63 | exit 0 64 | fi 65 | if [ "$REPLY" = "s" ]; then 66 | first_file="true" 67 | for gofile in "${gofiles_with_warnings[@]}" 68 | do 69 | echo 70 | if [ "$first_file" != "true" ]; then 71 | echo "Press enter to show the warnings for the next file." 72 | read 73 | fi 74 | golint $gofile 75 | first_file="false" 76 | done 77 | fi 78 | else 79 | # non-interactive shell (e.g. called from Eclipse). Just display the errors. 80 | for gofile in "${gofiles_with_warnings[@]}" 81 | do 82 | golint $gofile 83 | done 84 | fi 85 | exit 1 86 | -------------------------------------------------------------------------------- /proxy/plan/route_result.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Gaea Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package plan 16 | 17 | import ( 18 | "fmt" 19 | ) 20 | 21 | // RouteResult is the route result of a statement 22 | // 遍历AST之后得到的路由结果 23 | // db, table唯一确定了一个路由, 这里只记录分片表的db和table, 如果是关联表, 必须关联到同一个父表 24 | type RouteResult struct { 25 | db string 26 | table string 27 | 28 | currentIndex int // 当前遍历indexes位置下标 29 | indexes []int // 分片索引列表, 是有序的 30 | } 31 | 32 | // NewRouteResult constructor of RouteResult 33 | func NewRouteResult(db, table string, initIndexes []int) *RouteResult { 34 | return &RouteResult{ 35 | db: db, 36 | table: table, 37 | indexes: initIndexes, 38 | } 39 | } 40 | 41 | // Check check if the db and table is valid 42 | func (r *RouteResult) Check(db, table string) error { 43 | if r.db != db { 44 | return fmt.Errorf("db not equal, origin: %v, current: %v", r.db, db) 45 | } 46 | if r.table != table { 47 | return fmt.Errorf("table not equal, origin: %v, current: %v", r.table, table) 48 | } 49 | return nil 50 | } 51 | 52 | // Inter inter indexes with origin indexes in RouteResult 53 | // 如果是关联表, db, table需要用父表的db和table 54 | func (r *RouteResult) Inter(indexes []int) { 55 | r.indexes = interList(r.indexes, indexes) 56 | } 57 | 58 | // Union union indexes with origin indexes in RouteResult 59 | // 如果是关联表, db, table需要用父表的db和table 60 | func (r *RouteResult) Union(indexes []int) { 61 | r.indexes = unionList(r.indexes, indexes) 62 | } 63 | 64 | // GetShardIndexes get shard indexes 65 | func (r *RouteResult) GetShardIndexes() []int { 66 | return r.indexes 67 | } 68 | 69 | // GetCurrentTableIndex get current table index 70 | func (r *RouteResult) GetCurrentTableIndex() (int, error) { 71 | if r.currentIndex >= len(r.indexes) { 72 | return -1, fmt.Errorf("table index out of range") 73 | } 74 | return r.indexes[r.currentIndex], nil 75 | } 76 | 77 | // Next get next table index 78 | func (r *RouteResult) Next() int { 79 | idx := r.currentIndex 80 | r.currentIndex++ 81 | return r.indexes[idx] 82 | } 83 | 84 | // HasNext check if has next index 85 | func (r *RouteResult) HasNext() bool { 86 | return r.currentIndex < len(r.indexes) 87 | } 88 | 89 | // Reset reset the cursor of index 90 | func (r *RouteResult) Reset() { 91 | r.currentIndex = 0 92 | } 93 | -------------------------------------------------------------------------------- /mysql/resultset_sort_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The kingshard Authors. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | // not use this file except in compliance with the License. You may obtain 5 | // a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package mysql 16 | 17 | import ( 18 | "fmt" 19 | "reflect" 20 | "sort" 21 | "testing" 22 | ) 23 | 24 | func TestResultsetSort(t *testing.T) { 25 | r1 := new(Resultset) 26 | r2 := new(Resultset) 27 | 28 | r1.Values = [][]interface{}{ 29 | {int64(1), "a", []byte("aa")}, 30 | {int64(2), "a", []byte("bb")}, 31 | {int64(3), "c", []byte("bb")}, 32 | } 33 | 34 | r1.RowDatas = []RowData{ 35 | RowData([]byte("1")), 36 | RowData([]byte("2")), 37 | RowData([]byte("3")), 38 | } 39 | 40 | s := new(ResultsetSorter) 41 | 42 | s.Resultset = r1 43 | 44 | s.sk = []SortKey{ 45 | {Column: 0, Direction: SortDesc}, 46 | } 47 | 48 | sort.Sort(s) 49 | 50 | r2.Values = [][]interface{}{ 51 | {int64(3), "c", []byte("bb")}, 52 | {int64(2), "a", []byte("bb")}, 53 | {int64(1), "a", []byte("aa")}, 54 | } 55 | 56 | r2.RowDatas = []RowData{ 57 | RowData([]byte("3")), 58 | RowData([]byte("2")), 59 | RowData([]byte("1")), 60 | } 61 | 62 | if !reflect.DeepEqual(r1, r2) { 63 | t.Fatal(fmt.Sprintf("%v %v", r1, r2)) 64 | } 65 | 66 | s.sk = []SortKey{ 67 | {Column: 1, Direction: SortAsc}, 68 | {Column: 2, Direction: SortDesc}, 69 | } 70 | 71 | sort.Sort(s) 72 | 73 | r2.Values = [][]interface{}{ 74 | {int64(2), "a", []byte("bb")}, 75 | {int64(1), "a", []byte("aa")}, 76 | {int64(3), "c", []byte("bb")}, 77 | } 78 | 79 | r2.RowDatas = []RowData{ 80 | RowData([]byte("2")), 81 | RowData([]byte("1")), 82 | RowData([]byte("3")), 83 | } 84 | 85 | if !reflect.DeepEqual(r1, r2) { 86 | t.Fatal(fmt.Sprintf("%v %v", r1, r2)) 87 | } 88 | 89 | s.sk = []SortKey{ 90 | {Column: 1, Direction: SortAsc}, 91 | {Column: 2, Direction: SortAsc}, 92 | } 93 | 94 | sort.Sort(s) 95 | 96 | r2.Values = [][]interface{}{ 97 | {int64(1), "a", []byte("aa")}, 98 | {int64(2), "a", []byte("bb")}, 99 | {int64(3), "c", []byte("bb")}, 100 | } 101 | 102 | r2.RowDatas = []RowData{ 103 | RowData([]byte("1")), 104 | RowData([]byte("2")), 105 | RowData([]byte("3")), 106 | } 107 | 108 | if !reflect.DeepEqual(r1, r2) { 109 | t.Fatal(fmt.Sprintf("%v %v", r1, r2)) 110 | } 111 | 112 | } 113 | -------------------------------------------------------------------------------- /proxy/sequence/mysql.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Gaea Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package sequence 16 | 17 | import ( 18 | "errors" 19 | "fmt" 20 | "strconv" 21 | "strings" 22 | "sync" 23 | 24 | "github.com/XiaoMi/Gaea/backend" 25 | ) 26 | 27 | // MySQLSequence struct of sequence number with specific sequence name 28 | type MySQLSequence struct { 29 | slice *backend.Slice 30 | pkName string 31 | seqName string 32 | lock *sync.Mutex 33 | curr int64 34 | max int64 35 | sql string 36 | } 37 | 38 | // NewMySQLSequence init sequence item 39 | // TODO: 直接注入slice需要考虑关闭的问题, 目前是在Namespace中管理Slice的关闭的. 如果单独使用MySQLSequence, 需要注意. 40 | func NewMySQLSequence(slice *backend.Slice, seqName, pkName string) *MySQLSequence { 41 | t := &MySQLSequence{ 42 | slice: slice, 43 | seqName: seqName, 44 | pkName: pkName, 45 | lock: new(sync.Mutex), 46 | curr: 0, 47 | max: 0, 48 | sql: "SELECT mycat_seq_nextval('" + seqName + "') as seq_val", 49 | } 50 | return t 51 | } 52 | 53 | // NextSeq get next sequence number 54 | func (s *MySQLSequence) NextSeq() (int64, error) { 55 | s.lock.Lock() 56 | defer s.lock.Unlock() 57 | if s.curr >= s.max { 58 | err := s.getSeqFromDB() 59 | if err != nil { 60 | return 0, err 61 | } 62 | } 63 | s.curr++ 64 | return s.curr, nil 65 | } 66 | 67 | // GetPKName return sequence column 68 | func (s *MySQLSequence) GetPKName() string { 69 | return s.pkName 70 | } 71 | 72 | func (s *MySQLSequence) getSeqFromDB() error { 73 | conn, err := s.slice.GetMasterConn() 74 | if err != nil { 75 | return err 76 | } 77 | defer conn.Recycle() 78 | 79 | err = conn.UseDB("mycat") 80 | if err != nil { 81 | return err 82 | } 83 | 84 | r, err := conn.Execute(s.sql) 85 | if err != nil { 86 | return err 87 | } 88 | 89 | ret, err := r.Resultset.GetString(0, 0) 90 | if err != nil { 91 | return err 92 | } 93 | 94 | ns := strings.Split(ret, ",") 95 | if len(ns) != 2 { 96 | return errors.New(fmt.Sprintf("invalid mycat sequence value %s %s", s.seqName, ret)) 97 | } 98 | 99 | curr, _ := strconv.ParseInt(ns[0], 10, 64) 100 | incr, _ := strconv.ParseInt(ns[1], 10, 64) 101 | s.max = curr + incr 102 | s.curr = curr 103 | return nil 104 | } 105 | -------------------------------------------------------------------------------- /core/version.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Gaea Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package core 16 | 17 | import ( 18 | "fmt" 19 | "runtime" 20 | ) 21 | 22 | // The following fields are populated at build time using -ldflags -X. 23 | // Note that DATE is omitted for reproducible builds 24 | var ( 25 | buildVersion = "unknown" 26 | buildGitRevision = "unknown" 27 | buildUser = "unknown" 28 | buildHost = "unknown" 29 | buildStatus = "unknown" 30 | buildTime = "unknown" 31 | ) 32 | 33 | // BuildInfo describes version information about the binary build. 34 | type BuildInfo struct { 35 | Version string `json:"version"` 36 | GitRevision string `json:"revision"` 37 | User string `json:"user"` 38 | Host string `json:"host"` 39 | GolangVersion string `json:"golang_version"` 40 | BuildStatus string `json:"status"` 41 | BuildTime string `json:"time"` 42 | } 43 | 44 | var ( 45 | // Info exports the build version information. 46 | Info BuildInfo 47 | ) 48 | 49 | // String produces a single-line version info 50 | // 51 | // This looks like: 52 | // 53 | // ``` 54 | // user@host--- 55 | // ``` 56 | func (b BuildInfo) String() string { 57 | return fmt.Sprintf("%v@%v-%v-%v-%v-%v", 58 | b.User, 59 | b.Host, 60 | b.Version, 61 | b.GitRevision, 62 | b.BuildStatus, 63 | b.BuildTime) 64 | } 65 | 66 | // LongForm returns a multi-line version information 67 | // 68 | // This looks like: 69 | // 70 | // ``` 71 | // Version: 72 | // GitRevision: 73 | // User: user@host 74 | // GolangVersion: go1.10.2 75 | // BuildStatus: 76 | // ``` 77 | func (b BuildInfo) LongForm() string { 78 | return fmt.Sprintf(`Version: %v 79 | GitRevision: %v 80 | User: %v@%v 81 | GolangVersion: %v 82 | BuildStatus: %v 83 | BuildTime: %v 84 | `, 85 | b.Version, 86 | b.GitRevision, 87 | b.User, 88 | b.Host, 89 | b.GolangVersion, 90 | b.BuildStatus, 91 | b.BuildTime) 92 | } 93 | 94 | func init() { 95 | Info = BuildInfo{ 96 | Version: buildVersion, 97 | GitRevision: buildGitRevision, 98 | User: buildUser, 99 | Host: buildHost, 100 | GolangVersion: runtime.Version(), 101 | BuildStatus: buildStatus, 102 | BuildTime: buildTime, 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /cc/proxy/proxy_api.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Gaea Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package proxy 16 | 17 | import ( 18 | "encoding/json" 19 | 20 | "github.com/XiaoMi/Gaea/util/requests" 21 | ) 22 | 23 | // APIClient api client 24 | type APIClient struct { 25 | addr string 26 | user string 27 | password string 28 | } 29 | 30 | // NewAPIClient create api client 31 | func NewAPIClient(addr, user, password string) *APIClient { 32 | return &APIClient{addr: addr, user: user, password: password} 33 | } 34 | 35 | // PrepareConfig send prepare source 36 | func (c *APIClient) PrepareConfig(name string) error { 37 | url := c.encodeURL("/api/proxy/source/prepare/%s", name) 38 | return requests.SendPut(url, c.user, c.password) 39 | } 40 | 41 | // CommitConfig send commit source 42 | func (c *APIClient) CommitConfig(name string) error { 43 | url := c.encodeURL("/api/proxy/source/commit/%s", name) 44 | return requests.SendPut(url, c.user, c.password) 45 | } 46 | 47 | // DelNamespace send delete namespace to proxy 48 | func (c *APIClient) DelNamespace(name string) error { 49 | url := c.encodeURL("/api/proxy/namespace/delete/%s", name) 50 | return requests.SendPut(url, c.user, c.password) 51 | } 52 | 53 | // GetNamespaceSQLFingerprint return parser fingerprint of specific namespace 54 | func (c *APIClient) GetNamespaceSQLFingerprint(name string) (*SQLFingerprint, error) { 55 | var reply SQLFingerprint 56 | url := c.encodeURL("/api/proxy/stats/sessionsqlfingerprint/%s", name) 57 | resp, err := requests.SendGet(url, c.user, c.password) 58 | if err != nil { 59 | return nil, err 60 | } 61 | if resp != nil && resp.Body != nil { 62 | json.Unmarshal(resp.Body, &reply) 63 | } 64 | return &reply, err 65 | } 66 | 67 | func (c *APIClient) proxyConfigFingerprint() (string, error) { 68 | r := "" 69 | url := c.encodeURL("/api/proxy/source/fingerprint") 70 | resp, err := requests.SendGet(url, c.user, c.password) 71 | if err != nil { 72 | return r, err 73 | } 74 | if resp != nil && resp.Body != nil { 75 | json.Unmarshal(resp.Body, &r) 76 | } 77 | return r, err 78 | } 79 | 80 | // Ping ping proxy 81 | func (c *APIClient) Ping() error { 82 | url := c.encodeURL("/api/proxy/ping") 83 | _, err := requests.SendGet(url, c.user, c.password) 84 | if err != nil { 85 | return err 86 | } 87 | return nil 88 | } 89 | 90 | func (c *APIClient) encodeURL(format string, args ...interface{}) string { 91 | return requests.EncodeURL(c.addr, format, args...) 92 | } 93 | -------------------------------------------------------------------------------- /cmd/gaea/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Gaea Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "flag" 19 | "fmt" 20 | "github.com/XiaoMi/Gaea/logging" 21 | "github.com/XiaoMi/Gaea/util" 22 | "os" 23 | "os/signal" 24 | "sync" 25 | "syscall" 26 | 27 | "github.com/XiaoMi/Gaea/core" 28 | "github.com/XiaoMi/Gaea/models" 29 | "github.com/XiaoMi/Gaea/proxy/server" 30 | ) 31 | 32 | func main() { 33 | var defaultConfigFilePath = "/etc/gaea.ini" 34 | if util.IsWindows() { 35 | defaultConfigFilePath = "gaea.ini" 36 | } 37 | 38 | var configFile = flag.String("source", defaultConfigFilePath, "gaea source file") 39 | var info = flag.Bool("info", false, "show info of gaea") 40 | flag.Parse() 41 | 42 | if *info { 43 | fmt.Printf("Build Version Information:%s\n", core.Info.LongForm()) 44 | return 45 | } 46 | 47 | fmt.Printf("Build Version Information:%s\n", core.Info.LongForm()) 48 | var cfg *models.Proxy 49 | if !util.FileExists(*configFile) { 50 | cfg = models.DefaultProxy() 51 | } else { 52 | // init source of gaea proxy 53 | c, err := models.ParseProxyConfigFromFile(*configFile) 54 | if err != nil { 55 | fmt.Printf("parse source file error:%v\n", err.Error()) 56 | return 57 | } 58 | cfg = c 59 | } 60 | 61 | // init manager 62 | mgr, err := server.LoadAndCreateManager(cfg) 63 | if err != nil { 64 | logging.DefaultLogger.Fatalf("init manager failed, error: %v", err) 65 | return 66 | } 67 | 68 | svr, err := server.NewServer(cfg, mgr) 69 | if err != nil { 70 | logging.DefaultLogger.Fatalf("NewServer error, quit. error: %s", err.Error()) 71 | return 72 | } 73 | 74 | sc := make(chan os.Signal, 1) 75 | signal.Notify(sc, 76 | syscall.SIGINT, 77 | syscall.SIGTERM, 78 | syscall.SIGQUIT, 79 | syscall.SIGPIPE, 80 | //syscall.SIGUSR1, 81 | ) 82 | 83 | var wg sync.WaitGroup 84 | wg.Add(1) 85 | go func() { 86 | defer wg.Done() 87 | for { 88 | sig := <-sc 89 | if sig == syscall.SIGINT || sig == syscall.SIGTERM || sig == syscall.SIGQUIT { 90 | logging.DefaultLogger.Infof("Got signal %d, quit", sig) 91 | _ = svr.Close() 92 | break 93 | } else if sig == syscall.SIGPIPE { 94 | logging.DefaultLogger.Infof("Ignore broken pipe signal") 95 | } 96 | //} else if sig == syscall.SIGUSR1 { 97 | // log.Infof("Got update source signal") 98 | //} 99 | } 100 | }() 101 | _ = svr.Run() 102 | wg.Wait() 103 | } 104 | -------------------------------------------------------------------------------- /config/source/file_source.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Gaea Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package source 16 | 17 | import ( 18 | "errors" 19 | "github.com/XiaoMi/Gaea/logging" 20 | "io/ioutil" 21 | "os" 22 | "strings" 23 | "time" 24 | ) 25 | 26 | const ( 27 | defaultFilePath = "./etc/file" 28 | ) 29 | 30 | // File source provider for configuration 31 | type fileSource struct { 32 | Prefix string 33 | } 34 | 35 | // New constructor of etcdSource 36 | func NewFileSource(path string) (*fileSource, error) { 37 | if strings.TrimSpace(path) == "" { 38 | path = defaultFilePath 39 | } 40 | if err := checkDir(path); err != nil { 41 | logging.DefaultLogger.Warnf("check file source directory failed, %v", err) 42 | return nil, err 43 | } 44 | return &fileSource{Prefix: path}, nil 45 | } 46 | 47 | func checkDir(path string) error { 48 | if strings.TrimSpace(path) == "" { 49 | return errors.New("invalid path") 50 | } 51 | stat, err := os.Stat(path) 52 | if err != nil { 53 | return err 54 | } 55 | 56 | if !stat.IsDir() { 57 | return errors.New("invalid path, should be a directory") 58 | } 59 | 60 | return nil 61 | } 62 | 63 | // Close do nothing 64 | func (c *fileSource) Close() error { 65 | return nil 66 | } 67 | 68 | // Create do nothing 69 | func (c *fileSource) Create(path string, data []byte) error { 70 | return nil 71 | } 72 | 73 | // Update do nothing 74 | func (c *fileSource) Update(path string, data []byte) error { 75 | return nil 76 | } 77 | 78 | // UpdateWithTTL update path with data and ttl 79 | func (c *fileSource) UpdateWithTTL(path string, data []byte, ttl time.Duration) error { 80 | return nil 81 | } 82 | 83 | // Delete delete path 84 | func (c *fileSource) Delete(path string) error { 85 | return nil 86 | } 87 | 88 | // Read read file data 89 | func (c *fileSource) Read(file string) ([]byte, error) { 90 | value, err := ioutil.ReadFile(file) 91 | if err != nil { 92 | return nil, err 93 | } 94 | return value, nil 95 | } 96 | 97 | // List list path, return slice of all files 98 | func (c *fileSource) List(path string) ([]string, error) { 99 | r := make([]string, 0) 100 | files, err := ioutil.ReadDir(path) 101 | if err != nil { 102 | return r, err 103 | } 104 | 105 | for _, f := range files { 106 | r = append(r, f.Name()) 107 | } 108 | 109 | return r, nil 110 | } 111 | 112 | // BasePrefix return base prefix 113 | func (c *fileSource) BasePrefix() string { 114 | return c.Prefix 115 | } 116 | -------------------------------------------------------------------------------- /docs/sequence-id.md: -------------------------------------------------------------------------------- 1 | # 全局序列号说明 2 | 3 | ## 原理 4 | 5 | 参考mycat生成全局唯一序列号的设计，在数据库中建立一张表，存放sequence名称(name)，sequence当前值(current_value)，步长(increment int类型每次读取多少个sequence，假设为K)等信息； 6 | 7 | Sequence获取步骤： 8 | 9 | 1).第一次使用该sequence时，根据传入的sequence名称，从数据库这张表中读取current_value，和increment到gaea中，并将数据库中的current_value设置为原current_value值+increment值（实现方式是基于后续的存储函数） 10 | 11 | 2).gaea将读取到current_value+increment作为本次要使用的sequence值，下次使用时，自动加1，当使用increment次后，执行步骤1)相同的操作. 12 | 13 | gaea只会修改和查询这张表，使用前需要按照下文中配置table的步骤在这张表中插入一条记录。 14 | 若某次读取的sequence没有用完，系统就停掉了，则这次读取的sequence剩余值不会再使用。 15 | 16 | ## 如何使用 17 | 18 | 如：tbl_user_info的id列使用全局自增序列号 19 | ``` 20 | insert into gaea_test.tbl_user_info set name="zhangsan", age=15, id = nextval(); 21 | ``` 22 | 23 | ## 如何配置 24 | 在指定的slice的master上操作，完成以下配置。 25 | ### 配置db 26 | db: mycat 27 | ``` 28 | create database mycat; 29 | ``` 30 | ### 配置table 31 | table: mycat_sequence 32 | ``` 33 | DROP TABLE IF EXISTS MYCAT_SEQUENCE; 34 | CREATE TABLE MYCAT_SEQUENCE (name VARCHAR(50) NOT NULL,current_value INT NOT NULL,increment INT NOT NULL DEFAULT 100, PRIMARY KEY(name)) ENGINE=InnoDB; 35 | ``` 36 | 37 | ### 初始化table 38 | ``` 39 | INSERT INTO MYCAT_SEQUENCE(name,current_value,increment) VALUES ('GAEA_TEST.TBL_USER_INFO', -99, 100); 40 | ``` 41 | name由大写的db名称和table名称组成 42 | increment可根据参考业务qps峰值配置 43 | 例： 44 | 如果要在database gaea_test里的tbl_user_info表中使用全局唯一序列号 45 | name=>GAEA_TEST.TBL_USER_INFO 46 | ``` 47 | mysql> select * from mycat_sequence; 48 | +-------------------------+---------------+-----------+ 49 | | name | current_value | increment | 50 | +-------------------------+---------------+-----------+ 51 | | GAEA_TEST.TBL_USER_INFO | 700 | 100 | 52 | +-------------------------+---------------+-----------+ 53 | ``` 54 | 55 | ### 配置函数 56 | - 获取当前sequence的值： 57 | ``` 58 | DROP FUNCTION IF EXISTS mycat_seq_currval; 59 | DELIMITER $ 60 | CREATE FUNCTION mycat_seq_currval(seq_name VARCHAR(50)) RETURNS varchar(64) CHARSET utf8 61 | DETERMINISTIC 62 | BEGIN 63 | DECLARE retval VARCHAR(64); 64 | SET retval="-999999999,null"; 65 | SELECT concat(CAST(current_value AS CHAR),",",CAST(increment AS CHAR)) INTO retval FROM MYCAT_SEQUENCE WHERE name = seq_name; 66 | RETURN retval; 67 | END $ 68 | DELIMITER ; 69 | ``` 70 | 71 | - 设置sequence值： 72 | ``` 73 | 74 | DROP FUNCTION IF EXISTS mycat_seq_setval; 75 | DELIMITER $ 76 | CREATE FUNCTION mycat_seq_setval(seq_name VARCHAR(50),value INTEGER) RETURNS varchar(64) CHARSET utf8 77 | DETERMINISTIC 78 | BEGIN 79 | UPDATE MYCAT_SEQUENCE 80 | SET current_value = value 81 | WHERE name = seq_name; 82 | RETURN mycat_seq_currval(seq_name); 83 | END $ 84 | DELIMITER ; 85 | ``` 86 | 87 | - 获取下一个sequence值： 88 | 89 | ``` 90 | DROP FUNCTION IF EXISTS mycat_seq_nextval; 91 | DELIMITER $ 92 | CREATE FUNCTION mycat_seq_nextval(seq_name VARCHAR(50)) RETURNS varchar(64) CHARSET utf8 93 | DETERMINISTIC 94 | BEGIN 95 | UPDATE MYCAT_SEQUENCE 96 | SET current_value = current_value + increment WHERE name = seq_name; 97 | RETURN mycat_seq_currval(seq_name); 98 | END $ 99 | DELIMITER ; 100 | ``` 101 | 102 | -------------------------------------------------------------------------------- /logging/locked_multi_core.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | import ( 4 | "go.uber.org/multierr" 5 | "go.uber.org/zap" 6 | "go.uber.org/zap/zapcore" 7 | "sync" 8 | ) 9 | 10 | var _ zapcore.Core = (*lockedMultiCore)(nil) 11 | 12 | type lockedMultiCore struct { 13 | mu sync.RWMutex // guards mutations to cores slice 14 | cores []zapcore.Core 15 | } 16 | 17 | func (l *lockedMultiCore) With(fields []zapcore.Field) zapcore.Core { 18 | l.mu.RLock() 19 | defer l.mu.RUnlock() 20 | sub := &lockedMultiCore{ 21 | cores: make([]zapcore.Core, len(l.cores)), 22 | } 23 | for i := range l.cores { 24 | sub.cores[i] = l.cores[i].With(fields) 25 | } 26 | return sub 27 | } 28 | 29 | func (l *lockedMultiCore) Enabled(lvl zapcore.Level) bool { 30 | l.mu.RLock() 31 | defer l.mu.RUnlock() 32 | for i := range l.cores { 33 | if l.cores[i].Enabled(lvl) { 34 | return true 35 | } 36 | } 37 | return false 38 | } 39 | 40 | func (l *lockedMultiCore) Check(ent zapcore.Entry, ce *zapcore.CheckedEntry) *zapcore.CheckedEntry { 41 | l.mu.RLock() 42 | defer l.mu.RUnlock() 43 | for i := range l.cores { 44 | ce = l.cores[i].Check(ent, ce) 45 | } 46 | return ce 47 | } 48 | 49 | func (l *lockedMultiCore) Write(ent zapcore.Entry, fields []zapcore.Field) error { 50 | l.mu.RLock() 51 | defer l.mu.RUnlock() 52 | var err error 53 | for i := range l.cores { 54 | err = multierr.Append(err, l.cores[i].Write(ent, fields)) 55 | } 56 | return err 57 | } 58 | 59 | func (l *lockedMultiCore) Sync() error { 60 | l.mu.RLock() 61 | defer l.mu.RUnlock() 62 | var err error 63 | for i := range l.cores { 64 | err = multierr.Append(err, l.cores[i].Sync()) 65 | } 66 | return err 67 | } 68 | 69 | func (l *lockedMultiCore) AddCore(core zapcore.Core) { 70 | l.mu.Lock() 71 | defer l.mu.Unlock() 72 | 73 | l.cores = append(l.cores, core) 74 | } 75 | 76 | func (l *lockedMultiCore) DeleteCore(core zapcore.Core) { 77 | l.mu.Lock() 78 | defer l.mu.Unlock() 79 | 80 | w := 0 81 | for i := 0; i < len(l.cores); i++ { 82 | if l.cores[i] == core { 83 | continue 84 | } 85 | l.cores[w] = l.cores[i] 86 | w++ 87 | } 88 | l.cores = l.cores[:w] 89 | } 90 | 91 | func (l *lockedMultiCore) ReplaceCore(original, replacement zapcore.Core) { 92 | l.mu.Lock() 93 | defer l.mu.Unlock() 94 | 95 | for i := 0; i < len(l.cores); i++ { 96 | if l.cores[i] == original { 97 | l.cores[i] = replacement 98 | } 99 | } 100 | } 101 | 102 | func newCore(format LogFormat, ws zapcore.WriteSyncer, level zapcore.Level) zapcore.Core { 103 | encCfg := zap.NewProductionEncoderConfig() 104 | encCfg.EncodeTime = zapcore.ISO8601TimeEncoder 105 | 106 | var encoder zapcore.Encoder 107 | switch format { 108 | case PlaintextOutput: 109 | encCfg.EncodeLevel = zapcore.CapitalLevelEncoder 110 | encoder = zapcore.NewConsoleEncoder(encCfg) 111 | case JSONOutput: 112 | encoder = zapcore.NewJSONEncoder(encCfg) 113 | default: 114 | encCfg.EncodeLevel = zapcore.CapitalColorLevelEncoder 115 | encoder = zapcore.NewConsoleEncoder(encCfg) 116 | } 117 | 118 | return zapcore.NewCore(encoder, ws, zap.NewAtomicLevelAt(level)) 119 | } 120 | -------------------------------------------------------------------------------- /mysql/charset_tidb_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package mysql 15 | 16 | import ( 17 | "testing" 18 | 19 | "github.com/pingcap/check" 20 | 21 | "github.com/XiaoMi/Gaea/util/testleak" 22 | ) 23 | 24 | func TestT(t *testing.T) { 25 | check.CustomVerboseFlag = true 26 | check.TestingT(t) 27 | } 28 | 29 | var _ = check.Suite(&testCharsetSuite{}) 30 | 31 | type testCharsetSuite struct { 32 | } 33 | 34 | func testValidCharset(c *check.C, charset string, collation string, expect bool) { 35 | b := ValidCharsetAndCollation(charset, collation) 36 | c.Assert(b, check.Equals, expect) 37 | } 38 | 39 | func (s *testCharsetSuite) TestValidCharset(c *check.C) { 40 | defer testleak.AfterTest(c)() 41 | tests := []struct { 42 | cs string 43 | co string 44 | succ bool 45 | }{ 46 | {"utf8", "utf8_general_ci", true}, 47 | {"", "utf8_general_ci", true}, 48 | {"utf8mb4", "utf8mb4_bin", true}, 49 | {"latin1", "latin1_bin", true}, 50 | {"utf8", "utf8_invalid_ci", false}, 51 | {"utf16", "utf16_bin", false}, 52 | {"gb2312", "gb2312_chinese_ci", false}, 53 | {"UTF8", "UTF8_BIN", true}, 54 | {"UTF8", "utf8_bin", true}, 55 | {"UTF8MB4", "utf8mb4_bin", true}, 56 | {"UTF8MB4", "UTF8MB4_bin", true}, 57 | {"UTF8MB4", "UTF8MB4_general_ci", true}, 58 | {"Utf8", "uTf8_bIN", true}, 59 | } 60 | for _, tt := range tests { 61 | testValidCharset(c, tt.cs, tt.co, tt.succ) 62 | } 63 | } 64 | 65 | func (s *testCharsetSuite) TestGetAllCharsets(c *check.C) { 66 | defer testleak.AfterTest(c)() 67 | charset := &Charset{"test", "test_bin", nil, "Test", 5} 68 | charsetInfos = append(charsetInfos, charset) 69 | descs := GetAllCharsets() 70 | c.Assert(len(descs), check.Equals, len(charsetInfos)-1) 71 | } 72 | 73 | func testGetDefaultCollation(c *check.C, charset string, expectCollation string, succ bool) { 74 | b, err := GetDefaultCollation(charset) 75 | if !succ { 76 | c.Assert(err, check.NotNil) 77 | return 78 | } 79 | c.Assert(b, check.Equals, expectCollation) 80 | } 81 | 82 | func (s *testCharsetSuite) TestGetDefaultCollation(c *check.C) { 83 | defer testleak.AfterTest(c)() 84 | tests := []struct { 85 | cs string 86 | co string 87 | succ bool 88 | }{ 89 | {"utf8", "utf8_bin", true}, 90 | {"UTF8", "utf8_bin", true}, 91 | {"utf8mb4", "utf8mb4_bin", true}, 92 | {"ascii", "ascii_bin", true}, 93 | {"binary", "binary", true}, 94 | {"latin1", "latin1_bin", true}, 95 | {"invalid_cs", "", false}, 96 | {"", "utf8_bin", false}, 97 | } 98 | for _, tt := range tests { 99 | testGetDefaultCollation(c, tt.cs, tt.co, tt.succ) 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /docs/connection-pool.md: -------------------------------------------------------------------------------- 1 | # gaea proxy后端连接池的设计与实现 2 | 3 | ## 理想的连接池 4 | 5 | 基于go实现连接池的方式有很多种，比如通过chan、通过map+锁的方式，但是从使用者的角度来看，一个优秀的连接池我认为有以下几个特性: 1.有最大连接数和初始连接数限制 2.实际连接数可以在上述范围内动态伸缩 3.有限时间内获取连接且连接可用。在这三个基础之上，可能还包含一些其他非必须特性比如在运行时改变连接数最大限制、暴露连接池的一些状态信息等等。gaea的连接池是基于vitess的resource pool进行封装并添加了连接重试的功能，相关代码在backend和util目录下。 6 | 7 | ## 连接池的创建、使用 8 | 9 | ### 定义 10 | 11 | ConnectionPool定义 12 | 13 | ```golang 14 | // ConnectionPool means connection pool with specific addr 15 | type ConnectionPool struct { 16 | mu sync.RWMutex 17 | connections *util.ResourcePool 18 | 19 | addr string 20 | user string 21 | password string 22 | db string 23 | 24 | charset string 25 | collationID mysql.CollationID 26 | 27 | capacity int // capacity of pool 28 | maxCapacity int // max capacity of pool 29 | idleTimeout time.Duration 30 | } 31 | ``` 32 | 33 | NewConnectionPool定义 34 | 35 | ```golang 36 | // NewConnectionPool create connection pool 37 | func NewConnectionPool(addr, user, password, db string, capacity, maxCapacity int, idleTimeout time.Duration, charset string, collationID mysql.CollationID) *ConnectionPool { 38 | cp := &ConnectionPool{addr: addr, user: user, password: password, db: db, capacity: capacity, maxCapacity: maxCapacity, idleTimeout: idleTimeout, charset: charset, collationID: collationID} 39 | return cp 40 | } 41 | ``` 42 | 43 | Open定义 44 | 45 | ```golang 46 | // Open open connection pool without error, should be called before use the pool 47 | func (cp *ConnectionPool) Open() { 48 | if cp.capacity == 0 { 49 | cp.capacity = DefaultCapacity 50 | } 51 | 52 | if cp.maxCapacity == 0 { 53 | cp.maxCapacity = cp.capacity 54 | } 55 | cp.mu.Lock() 56 | defer cp.mu.Unlock() 57 | cp.connections = util.NewResourcePool(cp.connect, cp.capacity, cp.maxCapacity, cp.idleTimeout) 58 | return 59 | } 60 | ``` 61 | 62 | 每一个连接作为一个资源单位，connections存放所有的连接资源，类型为ResourcePool。capacity定义连接池的初始容量、maxCapacity定义连接池的最大容量，实际后端连接数量会根据使用情况在[0,maxCapacity]之间浮动。idleTimeout为连接空闲关闭时间，当连接不活跃时间达到该值时，连接池将会与后端mysql断开该连接并回收资源。 63 | 64 | ### 创建 65 | 66 | 外部调用者通过NewConnectionPool函数创建一个连接池对象，然后通过Open函数，初始化连接池并与后端mysql建立实际的连接(有的连接池或长连接会在第一次请求时才去建立连接)。连接池的connect方法作为资源池初始化的工厂方法，所以如果你要基于该resource pool实现其他池子时，需要实现该工厂方法。 67 | 68 | ### 使用 69 | 70 | 通过连接池的Get方法可以获取一个连接，Get的入口参数包含context，该context最初设计用来传入一些全局上下文，比如超时上下文，目前gaea的获取连接超时时间是固定的，所以超时上下文也是基于该context在内部构造的。为防止上层一直阻塞在获取连接处，超过getConnTimeout未获取到连接会报超时错误，从而避免发生更严重的状况。如果发生超时次数过多，可以通过配置平台调整最大连接数大小，不同namespace的最大连接数其实是一个经验值，根据不同的业务状态有所不同，支持动态实时调整。 71 | 72 | 拿到连接后，连接池会主动调用tryReuse，用来保证连接是自动提交的状态。应用层需要主动调用initBackendConn，初始化与mysql有关的状态信息，包括use db、charset、session variables、sql mode等。为了提升效率会检测后端连接与前端会话的对应变量值，不一致才会进行设置，在设置的时候也是通过批量发送sql的方式，尽最大可能减少与后端mysql的网络交互。 73 | 74 | 连接使用完成后，需要手动调用recycleBackendConn回收连接，注意: 事务相关的连接是在commit或者rollback的时候进行统一释放。 75 | 76 | ## 动态维护连接 77 | 78 | 动态维护连接其实包含两部分，一部分维护连接池容量，不活跃的连接要主动关闭。连接池的Open函数会调用NewResourcePool函数，NewResourcePool函数会启动一个Timer定时器，定时器通过定期检测连接活跃时间与空闲时间的差值，决定是否关闭连接、回收资源从而实现动态调整连接池的容量。另一部分是保证获取到的连接是有效连接，这里在通过writeEphemeralPacket向后端mysql连接写数据时，如果报错含有"broken pipe"即连接可能无效，则会进行重试，直到连接成功或者重试次数达到三次报错，通过主动监测和连接重试，我们不需要进行定期ping后端连接，就可以保证后端连接是有效的。 79 | 80 | ## 总结 81 | 82 | gaea的连接池实现还是相对简单可依赖的，在使用gaea的过程中，最好将后端mysql的wait_timeout值设置为比gaea idleTimeout长，可以减少不必要的连接重试。 83 | -------------------------------------------------------------------------------- /stats/timings_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 Google Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package stats 18 | 19 | import ( 20 | "expvar" 21 | "strings" 22 | "testing" 23 | "time" 24 | ) 25 | 26 | func TestTimings(t *testing.T) { 27 | clear() 28 | tm := NewTimings("timings1", "help", "category") 29 | tm.Add("tag1", 500*time.Microsecond) 30 | tm.Add("tag1", 1*time.Millisecond) 31 | tm.Add("tag2", 1*time.Millisecond) 32 | want := `{"TotalCount":3,"TotalTime":2500000,"Histograms":{"tag1":{"500000":1,"1000000":2,"5000000":2,"10000000":2,"50000000":2,"100000000":2,"500000000":2,"1000000000":2,"5000000000":2,"10000000000":2,"inf":2,"Count":2,"Time":1500000},"tag2":{"500000":0,"1000000":1,"5000000":1,"10000000":1,"50000000":1,"100000000":1,"500000000":1,"1000000000":1,"5000000000":1,"10000000000":1,"inf":1,"Count":1,"Time":1000000}}}` 33 | if got := tm.String(); got != want { 34 | t.Errorf("got %s, want %s", got, want) 35 | } 36 | } 37 | 38 | func TestMultiTimings(t *testing.T) { 39 | clear() 40 | mtm := NewMultiTimings("maptimings1", "help", []string{"dim1", "dim2"}) 41 | mtm.Add([]string{"tag1a", "tag1b"}, 500*time.Microsecond) 42 | mtm.Add([]string{"tag1a", "tag1b"}, 1*time.Millisecond) 43 | mtm.Add([]string{"tag2a", "tag2b"}, 1*time.Millisecond) 44 | want := `{"TotalCount":3,"TotalTime":2500000,"Histograms":{"tag1a.tag1b":{"500000":1,"1000000":2,"5000000":2,"10000000":2,"50000000":2,"100000000":2,"500000000":2,"1000000000":2,"5000000000":2,"10000000000":2,"inf":2,"Count":2,"Time":1500000},"tag2a.tag2b":{"500000":0,"1000000":1,"5000000":1,"10000000":1,"50000000":1,"100000000":1,"500000000":1,"1000000000":1,"5000000000":1,"10000000000":1,"inf":1,"Count":1,"Time":1000000}}}` 45 | if got := mtm.String(); got != want { 46 | t.Errorf("got %s, want %s", got, want) 47 | } 48 | } 49 | 50 | func TestMultiTimingsDot(t *testing.T) { 51 | clear() 52 | mtm := NewMultiTimings("maptimings2", "help", []string{"label"}) 53 | mtm.Add([]string{"value.dot"}, 500*time.Microsecond) 54 | safe := safeLabel("value.dot") 55 | safeJSON := strings.Replace(safe, "\\", "\\\\", -1) 56 | want := `{"TotalCount":1,"TotalTime":500000,"Histograms":{"` + safeJSON + `":{"500000":1,"1000000":1,"5000000":1,"10000000":1,"50000000":1,"100000000":1,"500000000":1,"1000000000":1,"5000000000":1,"10000000000":1,"inf":1,"Count":1,"Time":500000}}}` 57 | if got := mtm.String(); got != want { 58 | t.Errorf("got %s, want %s", got, want) 59 | } 60 | } 61 | 62 | func TestTimingsHook(t *testing.T) { 63 | var gotname string 64 | var gotv *Timings 65 | clear() 66 | Register(func(name string, v expvar.Var) { 67 | gotname = name 68 | gotv = v.(*Timings) 69 | }) 70 | 71 | name := "timings2" 72 | v := NewTimings(name, "help", "") 73 | if gotname != name { 74 | t.Errorf("got %q, want %q", gotname, name) 75 | } 76 | if gotv != v { 77 | t.Errorf("got %#v, want %#v", gotv, v) 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /util/time_wheel_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Gaea Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package util 16 | 17 | import ( 18 | "strconv" 19 | "sync/atomic" 20 | "testing" 21 | "time" 22 | 23 | "github.com/stretchr/testify/assert" 24 | ) 25 | 26 | type A struct { 27 | a int 28 | b string 29 | isCallbacked int32 30 | } 31 | 32 | func (a *A) callback() { 33 | atomic.StoreInt32(&a.isCallbacked, 1) 34 | } 35 | 36 | func (a *A) getCallbackValue() int32 { 37 | return atomic.LoadInt32(&a.isCallbacked) 38 | } 39 | 40 | func newTimeWheel() *TimeWheel { 41 | tw, err := NewTimeWheel(time.Second, 3600) 42 | if err != nil { 43 | panic(err) 44 | } 45 | tw.Start() 46 | return tw 47 | } 48 | 49 | func TestNewTimeWheel(t *testing.T) { 50 | tests := []struct { 51 | name string 52 | tick time.Duration 53 | bucketNum int 54 | hasErr bool 55 | }{ 56 | {tick: time.Second, bucketNum: 0, hasErr: true}, 57 | {tick: time.Millisecond, bucketNum: 1, hasErr: true}, 58 | {tick: time.Second, bucketNum: 1, hasErr: false}, 59 | } 60 | for _, test := range tests { 61 | t.Run(test.name, func(t *testing.T) { 62 | _, err := NewTimeWheel(test.tick, test.bucketNum) 63 | assert.Equal(t, test.hasErr, err != nil) 64 | }) 65 | } 66 | } 67 | 68 | func TestAdd(t *testing.T) { 69 | tw := newTimeWheel() 70 | a := &A{} 71 | err := tw.Add(time.Second*1, "test", a.callback) 72 | assert.NoError(t, err) 73 | 74 | time.Sleep(time.Millisecond * 500) 75 | assert.Equal(t, int32(0), a.getCallbackValue()) 76 | time.Sleep(time.Second * 2) 77 | assert.Equal(t, int32(1), a.getCallbackValue()) 78 | tw.Stop() 79 | } 80 | 81 | func TestAddMultipleTimes(t *testing.T) { 82 | a := &A{} 83 | tw := newTimeWheel() 84 | for i := 0; i < 4; i++ { 85 | err := tw.Add(time.Second, "test", a.callback) 86 | assert.NoError(t, err) 87 | time.Sleep(time.Millisecond * 500) 88 | t.Logf("current: %d", i) 89 | assert.Equal(t, int32(0), a.getCallbackValue()) 90 | } 91 | 92 | time.Sleep(time.Second * 2) 93 | assert.Equal(t, int32(1), a.getCallbackValue()) 94 | tw.Stop() 95 | } 96 | 97 | func TestRemove(t *testing.T) { 98 | a := &A{a: 10, b: "test"} 99 | tw := newTimeWheel() 100 | err := tw.Add(time.Second*1, a, a.callback) 101 | assert.NoError(t, err) 102 | 103 | time.Sleep(time.Millisecond * 500) 104 | assert.Equal(t, int32(0), a.getCallbackValue()) 105 | err = tw.Remove(a) 106 | assert.NoError(t, err) 107 | time.Sleep(time.Second * 2) 108 | assert.Equal(t, int32(0), a.getCallbackValue()) 109 | tw.Stop() 110 | } 111 | 112 | func BenchmarkAdd(b *testing.B) { 113 | a := &A{} 114 | tw := newTimeWheel() 115 | for i := 0; i < b.N; i++ { 116 | key := "test" + strconv.Itoa(i) 117 | err := tw.Add(time.Second, key, a.callback) 118 | if err != nil { 119 | b.Fatalf("benchmark Add failed, %v", err) 120 | } 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /stats/duration_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Vitess Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package stats 18 | 19 | import ( 20 | "expvar" 21 | "testing" 22 | "time" 23 | ) 24 | 25 | func TestCounterDuration(t *testing.T) { 26 | var gotname string 27 | var gotv *CounterDuration 28 | clear() 29 | Register(func(name string, v expvar.Var) { 30 | gotname = name 31 | gotv = v.(*CounterDuration) 32 | }) 33 | v := NewCounterDuration("CounterDuration", "help") 34 | if gotname != "CounterDuration" { 35 | t.Errorf("want CounterDuration, got %s", gotname) 36 | } 37 | if gotv != v { 38 | t.Errorf("want %#v, got %#v", v, gotv) 39 | } 40 | if v.Get() != 0 { 41 | t.Errorf("want 0, got %v", v.Get()) 42 | } 43 | v.Add(time.Duration(1)) 44 | if v.Get() != 1 { 45 | t.Errorf("want 1, got %v", v.Get()) 46 | } 47 | if v.String() != "1" { 48 | t.Errorf("want 1, got %v", v.Get()) 49 | } 50 | } 51 | 52 | func TestCounterDurationFunc(t *testing.T) { 53 | var gotname string 54 | var gotv *CounterDurationFunc 55 | clear() 56 | Register(func(name string, v expvar.Var) { 57 | gotname = name 58 | gotv = v.(*CounterDurationFunc) 59 | }) 60 | 61 | v := NewCounterDurationFunc("CounterDurationFunc", "help", func() time.Duration { 62 | return time.Duration(1) 63 | }) 64 | if gotname != "CounterDurationFunc" { 65 | t.Errorf("want CounterDurationFunc, got %s", gotname) 66 | } 67 | if gotv != v { 68 | t.Errorf("want %#v, got %#v", v, gotv) 69 | } 70 | if v.String() != "1" { 71 | t.Errorf("want 1, got %v", v.String()) 72 | } 73 | } 74 | 75 | func TestGaugeDuration(t *testing.T) { 76 | var gotname string 77 | var gotv *GaugeDuration 78 | clear() 79 | Register(func(name string, v expvar.Var) { 80 | gotname = name 81 | gotv = v.(*GaugeDuration) 82 | }) 83 | v := NewGaugeDuration("GaugeDuration", "help") 84 | if gotname != "GaugeDuration" { 85 | t.Errorf("want GaugeDuration, got %s", gotname) 86 | } 87 | if gotv != v { 88 | t.Errorf("want %#v, got %#v", v, gotv) 89 | } 90 | v.Set(time.Duration(5)) 91 | if v.Get() != 5 { 92 | t.Errorf("want 5, got %v", v.Get()) 93 | } 94 | v.Add(time.Duration(1)) 95 | if v.Get() != 6 { 96 | t.Errorf("want 6, got %v", v.Get()) 97 | } 98 | if v.String() != "6" { 99 | t.Errorf("want 6, got %v", v.Get()) 100 | } 101 | } 102 | 103 | func TestGaugeDurationFunc(t *testing.T) { 104 | var gotname string 105 | var gotv *GaugeDurationFunc 106 | clear() 107 | Register(func(name string, v expvar.Var) { 108 | gotname = name 109 | gotv = v.(*GaugeDurationFunc) 110 | }) 111 | 112 | v := NewGaugeDurationFunc("GaugeDurationFunc", "help", func() time.Duration { 113 | return time.Duration(1) 114 | }) 115 | 116 | if gotname != "GaugeDurationFunc" { 117 | t.Errorf("want GaugeDurationFunc, got %s", gotname) 118 | } 119 | if gotv != v { 120 | t.Errorf("want %#v, got %#v", v, gotv) 121 | } 122 | if v.String() != "1" { 123 | t.Errorf("want 1, got %v", v.String()) 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /proxy/plan/plan_explain.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Gaea Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package plan 16 | 17 | import ( 18 | "fmt" 19 | "github.com/XiaoMi/Gaea/backend" 20 | "github.com/XiaoMi/Gaea/mysql" 21 | "github.com/XiaoMi/Gaea/proxy/router" 22 | "github.com/XiaoMi/Gaea/proxy/sequence" 23 | "github.com/XiaoMi/Gaea/util" 24 | "github.com/pingcap/parser/ast" 25 | ) 26 | 27 | // constants of ShardType 28 | const ( 29 | ShardTypeUnshard = "unshard" 30 | ShardTypeShard = "shard" 31 | ) 32 | 33 | // ExplainPlan is the plan for explain statement 34 | type ExplainPlan struct { 35 | shardType string 36 | sqls map[string]map[string][]string 37 | } 38 | 39 | func buildExplainPlan(stmt *ast.ExplainStmt, phyDBs map[string]string, db, sql string, r *router.Router, seq *sequence.SequenceManager) (*ExplainPlan, error) { 40 | stmtToExplain := stmt.Stmt 41 | if _, ok := stmtToExplain.(*ast.ExplainStmt); ok { 42 | return nil, fmt.Errorf("nested explain") 43 | } 44 | 45 | p, err := BuildPlan(stmtToExplain, phyDBs, db, sql, r, seq) 46 | if err != nil { 47 | return nil, fmt.Errorf("build plan to explain error: %v", err) 48 | } 49 | 50 | ep := &ExplainPlan{} 51 | 52 | switch pl := p.(type) { 53 | case *SelectPlan: 54 | ep.shardType = ShardTypeShard 55 | ep.sqls = pl.sqls 56 | return ep, nil 57 | case *DeletePlan: 58 | ep.shardType = ShardTypeShard 59 | ep.sqls = pl.sqls 60 | return ep, nil 61 | case *UpdatePlan: 62 | ep.shardType = ShardTypeShard 63 | ep.sqls = pl.sqls 64 | return ep, nil 65 | case *InsertPlan: 66 | ep.shardType = ShardTypeShard 67 | ep.sqls = pl.sqls 68 | return ep, nil 69 | case *UnshardPlan: 70 | ep.shardType = ShardTypeUnshard 71 | ep.sqls = make(map[string]map[string][]string) 72 | dbSQLs := make(map[string][]string) 73 | if phyDB, ok := phyDBs[pl.db]; ok { 74 | pl.db = phyDB 75 | } 76 | dbSQLs[pl.db] = []string{pl.sql} 77 | ep.sqls[backend.DefaultSlice] = dbSQLs 78 | return ep, nil 79 | default: 80 | return nil, fmt.Errorf("unsupport plan to explain, type: %T", p) 81 | } 82 | } 83 | 84 | // ExecuteIn implement Plan 85 | func (p *ExplainPlan) ExecuteIn(*util.RequestContext, Executor) (*mysql.Result, error) { 86 | return createExplainResult(p.shardType, p.sqls), nil 87 | } 88 | 89 | // Size implement Plan 90 | func (p *ExplainPlan) Size() int { 91 | return 1 92 | } 93 | 94 | func createExplainResult(shardType string, sqls map[string]map[string][]string) *mysql.Result { 95 | var rows [][]interface{} 96 | var names = []string{"type", "slice", "db", "parser"} 97 | 98 | for slice, dbSQLs := range sqls { 99 | for db, tableSQLs := range dbSQLs { 100 | for _, sql := range tableSQLs { 101 | row := []interface{}{shardType, slice, db, sql} 102 | rows = append(rows, row) 103 | } 104 | } 105 | } 106 | 107 | r, _ := mysql.BuildResultset(nil, names, rows) 108 | ret := &mysql.Result{ 109 | Resultset: r, 110 | } 111 | 112 | return ret 113 | } 114 | -------------------------------------------------------------------------------- /stats/duration.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Vitess Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package stats 18 | 19 | import ( 20 | "strconv" 21 | "time" 22 | 23 | "github.com/XiaoMi/Gaea/util/sync2" 24 | ) 25 | 26 | // CounterDuration exports a time.Duration as counter. 27 | type CounterDuration struct { 28 | i sync2.AtomicDuration 29 | help string 30 | } 31 | 32 | // NewCounterDuration returns a new CounterDuration. 33 | func NewCounterDuration(name, help string) *CounterDuration { 34 | cd := &CounterDuration{ 35 | help: help, 36 | } 37 | publish(name, cd) 38 | return cd 39 | } 40 | 41 | // Help implements the Variable interface. 42 | func (cd CounterDuration) Help() string { 43 | return cd.help 44 | } 45 | 46 | // String is the implementation of expvar.var. 47 | func (cd CounterDuration) String() string { 48 | return strconv.FormatInt(int64(cd.i.Get()), 10) 49 | } 50 | 51 | // Add adds the provided value to the CounterDuration. 52 | func (cd *CounterDuration) Add(delta time.Duration) { 53 | cd.i.Add(delta) 54 | } 55 | 56 | // Get returns the value. 57 | func (cd *CounterDuration) Get() time.Duration { 58 | return cd.i.Get() 59 | } 60 | 61 | // GaugeDuration exports a time.Duration as gauge. 62 | // In addition to CounterDuration, it also has Set() which allows overriding 63 | // the current value. 64 | type GaugeDuration struct { 65 | CounterDuration 66 | } 67 | 68 | // NewGaugeDuration returns a new GaugeDuration. 69 | func NewGaugeDuration(name, help string) *GaugeDuration { 70 | gd := &GaugeDuration{ 71 | CounterDuration: CounterDuration{ 72 | help: help, 73 | }, 74 | } 75 | publish(name, gd) 76 | return gd 77 | } 78 | 79 | // Set sets the value. 80 | func (gd *GaugeDuration) Set(value time.Duration) { 81 | gd.i.Set(value) 82 | } 83 | 84 | // CounterDurationFunc allows to provide the value via a custom function. 85 | type CounterDurationFunc struct { 86 | F func() time.Duration 87 | help string 88 | } 89 | 90 | // NewCounterDurationFunc creates a new CounterDurationFunc instance and 91 | // publishes it if name is set. 92 | func NewCounterDurationFunc(name string, help string, f func() time.Duration) *CounterDurationFunc { 93 | cf := &CounterDurationFunc{ 94 | F: f, 95 | help: help, 96 | } 97 | 98 | if name != "" { 99 | publish(name, cf) 100 | } 101 | return cf 102 | } 103 | 104 | // Help implements the Variable interface. 105 | func (cf CounterDurationFunc) Help() string { 106 | return cf.help 107 | } 108 | 109 | // String is the implementation of expvar.var. 110 | func (cf CounterDurationFunc) String() string { 111 | return strconv.FormatInt(int64(cf.F()), 10) 112 | } 113 | 114 | // GaugeDurationFunc allows to provide the value via a custom function. 115 | type GaugeDurationFunc struct { 116 | CounterDurationFunc 117 | } 118 | 119 | // NewGaugeDurationFunc creates a new GaugeDurationFunc instance and 120 | // publishes it if name is set. 121 | func NewGaugeDurationFunc(name string, help string, f func() time.Duration) *GaugeDurationFunc { 122 | gf := &GaugeDurationFunc{ 123 | CounterDurationFunc: CounterDurationFunc{ 124 | F: f, 125 | help: help, 126 | }, 127 | } 128 | 129 | if name != "" { 130 | publish(name, gf) 131 | } 132 | return gf 133 | } 134 | -------------------------------------------------------------------------------- /mysql/error.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The kingshard Authors. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | // not use this file except in compliance with the License. You may obtain 5 | // a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | // Package mysql error.go contains error code, SQLSTATE value, and message string. 16 | // The struct SqlError contains above informations and used in programs. 17 | 18 | // Copyright 2019 The Gaea Authors. All Rights Reserved. 19 | // 20 | // Licensed under the Apache License, Version 2.0 (the "License"); 21 | // you may not use this file except in compliance with the License. 22 | // You may obtain a copy of the License at 23 | // 24 | // http://www.apache.org/licenses/LICENSE-2.0 25 | // 26 | // Unless required by applicable law or agreed to in writing, software 27 | // distributed under the License is distributed on an "AS IS" BASIS, 28 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 29 | // See the License for the specific language governing permissions and 30 | // limitations under the License. 31 | 32 | package mysql 33 | 34 | import ( 35 | "errors" 36 | "fmt" 37 | ) 38 | 39 | var ( 40 | // ErrBadConn bad connection error 41 | ErrBadConn = errors.New("connection was bad") 42 | // ErrMalformPacket packet error 43 | ErrMalformPacket = errors.New("Malform packet error") 44 | // ErrTxDone transaction done error 45 | ErrTxDone = errors.New("parser: Transaction has already been committed or rolled back") 46 | ) 47 | 48 | // SQLError contains error code、SQLSTATE and message string 49 | // https://dev.mysql.com/doc/refman/5.7/en/server-error-reference.html 50 | type SQLError struct { 51 | Code uint16 52 | Message string 53 | State string 54 | } 55 | 56 | func (se *SQLError) Error() string { 57 | return fmt.Sprintf("ERROR %d (%s): %s", se.Code, se.State, se.Message) 58 | } 59 | 60 | // SQLCode returns the internal MySQL error code. 61 | func (se *SQLError) SQLCode() uint16 { 62 | return se.Code 63 | } 64 | 65 | // SQLState returns the SQLSTATE value. 66 | func (se *SQLError) SQLState() string { 67 | return se.State 68 | } 69 | 70 | // NewDefaultError default mysql error, must adapt errname message format 71 | func NewDefaultError(errCode uint16, args ...interface{}) *SQLError { 72 | e := new(SQLError) 73 | e.Code = errCode 74 | 75 | if s, ok := MySQLState[errCode]; ok { 76 | e.State = s 77 | } else { 78 | e.State = DefaultMySQLState 79 | } 80 | 81 | if format, ok := MySQLErrName[errCode]; ok { 82 | e.Message = fmt.Sprintf(format, args...) 83 | } else { 84 | e.Message = fmt.Sprint(args...) 85 | } 86 | 87 | return e 88 | } 89 | 90 | // NewError create new error with specified code and message 91 | func NewError(errCode uint16, message string) *SQLError { 92 | e := new(SQLError) 93 | e.Code = errCode 94 | 95 | if s, ok := MySQLState[errCode]; ok { 96 | e.State = s 97 | } else { 98 | e.State = DefaultMySQLState 99 | } 100 | 101 | e.Message = message 102 | 103 | return e 104 | } 105 | 106 | // NewErrf creates a SQL error, with an error code and a format specifier. 107 | func NewErrf(errCode uint16, format string, args ...interface{}) *SQLError { 108 | e := &SQLError{Code: errCode} 109 | 110 | if s, ok := MySQLState[errCode]; ok { 111 | e.State = s 112 | } else { 113 | e.State = DefaultMySQLState 114 | } 115 | 116 | e.Message = fmt.Sprintf(format, args...) 117 | 118 | return e 119 | } 120 | -------------------------------------------------------------------------------- /util/ip.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The kingshard Authors. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | // not use this file except in compliance with the License. You may obtain 5 | // a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package util 16 | 17 | import ( 18 | "errors" 19 | "fmt" 20 | "net" 21 | "strings" 22 | ) 23 | 24 | // IPInfo ip information 25 | type IPInfo struct { 26 | info string 27 | isIPNet bool 28 | ip net.IP 29 | ipNet net.IPNet 30 | } 31 | 32 | // ParseIPInfo parse ip 33 | func ParseIPInfo(v string) (IPInfo, error) { 34 | if ip, ipNet, err := net.ParseCIDR(v); err == nil { 35 | return IPInfo{ 36 | info: v, 37 | isIPNet: true, 38 | ip: ip, 39 | ipNet: *ipNet, 40 | }, nil 41 | } 42 | 43 | if ip := net.ParseIP(v); ip != nil { 44 | return IPInfo{ 45 | info: v, 46 | isIPNet: false, 47 | ip: ip, 48 | }, nil 49 | } 50 | 51 | return IPInfo{}, errors.New("invalid ip address") 52 | } 53 | 54 | // Info return information of ip 55 | func (t *IPInfo) Info() string { 56 | return t.info 57 | } 58 | 59 | // Match check if ip matched 60 | func (t *IPInfo) Match(ip net.IP) bool { 61 | if t.isIPNet { 62 | return t.ipNet.Contains(ip) 63 | } 64 | return t.ip.Equal(ip) 65 | } 66 | 67 | func parseAllowIps(allowIpsStr string) ([]IPInfo, error) { 68 | if len(allowIpsStr) == 0 { 69 | return make([]IPInfo, 0, 10), nil 70 | } 71 | ipVec := strings.Split(allowIpsStr, ",") 72 | allowIpsList := make([]IPInfo, 0, 10) 73 | for _, ipStr := range ipVec { 74 | if ip, err := ParseIPInfo(strings.TrimSpace(ipStr)); err == nil { 75 | allowIpsList = append(allowIpsList, ip) 76 | } 77 | } 78 | return allowIpsList, nil 79 | } 80 | 81 | func resolveAddr(network string, address string, replaceZeroAddr bool) (string, error) { 82 | switch network { 83 | default: 84 | return "", fmt.Errorf("invalid network '%s'", network) 85 | 86 | case "unix", "unixpacket": 87 | return address, nil 88 | 89 | case "tcp", "tcp4", "tcp6": 90 | addr, err := net.ResolveTCPAddr(network, address) 91 | if err != nil { 92 | return "", err 93 | } 94 | if addr.Port != 0 { 95 | //是否为空地址 96 | if !isZeroIPAddr(addr) { 97 | return addr.String(), nil 98 | } 99 | if replaceZeroAddr { 100 | ifaddrs, err := net.InterfaceAddrs() 101 | if err != nil { 102 | return "", err 103 | } 104 | for _, ifaddr := range ifaddrs { 105 | switch in := ifaddr.(type) { 106 | case *net.IPNet: 107 | if in.IP.IsGlobalUnicast() { 108 | addr.IP = in.IP 109 | return addr.String(), nil 110 | } 111 | } 112 | } 113 | } 114 | } 115 | return "", fmt.Errorf("invalid address '%s'", addr.String()) 116 | } 117 | } 118 | 119 | // ResolveAddr return real ip by net interface 120 | func ResolveAddr(network string, locAddr string) (string, error) { 121 | return resolveAddr(network, locAddr, true) 122 | } 123 | 124 | func isZeroIPAddr(addr *net.TCPAddr) bool { 125 | if ipv4 := addr.IP.To4(); ipv4 != nil { 126 | return net.IPv4zero.Equal(ipv4) 127 | } else if ipv6 := addr.IP.To16(); ipv6 != nil { 128 | return net.IPv6zero.Equal(ipv6) 129 | } 130 | return false 131 | } 132 | 133 | // HostName return hostname by ip 134 | func HostName(ip string) (hostname string, err error) { 135 | hostName, err := net.LookupAddr(ip) 136 | if err != nil { 137 | return "", err 138 | } 139 | return hostName[0], err 140 | } 141 | -------------------------------------------------------------------------------- /parser/sql_preview.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "strings" 5 | "unicode" 6 | ) 7 | 8 | type StatementType int 9 | 10 | const ( 11 | StmtSelect StatementType = iota 12 | StmtStream 13 | StmtInsert 14 | StmtReplace 15 | StmtUpdate 16 | StmtDelete 17 | StmtDDL 18 | StmtBegin 19 | StmtCommit 20 | StmtRollback 21 | StmtSet 22 | StmtShow 23 | StmtUse 24 | StmtOther 25 | StmtUnknown 26 | StmtComment 27 | StmtExplain 28 | StmtPriv 29 | StmtSavepoint 30 | StmtRelease 31 | StmtSRollback 32 | ) 33 | 34 | // Preview analyzes the beginning of the query using a simpler and faster 35 | // textual comparison to identify the statement type. 36 | func PreviewSql(sql string) StatementType { 37 | trimmed := StripLeadingComments(sql) 38 | 39 | if strings.Index(trimmed, "/*!") == 0 { 40 | return StmtComment 41 | } 42 | 43 | isNotLetter := func(r rune) bool { return !unicode.IsLetter(r) } 44 | firstWord := strings.TrimLeftFunc(trimmed, isNotLetter) 45 | 46 | if end := strings.IndexFunc(firstWord, unicode.IsSpace); end != -1 { 47 | firstWord = firstWord[:end] 48 | } 49 | // Comparison is done in order of priority. 50 | loweredFirstWord := strings.ToLower(firstWord) 51 | switch loweredFirstWord { 52 | case "select": 53 | return StmtSelect 54 | case "stream": 55 | return StmtStream 56 | case "insert": 57 | return StmtInsert 58 | case "replace": 59 | return StmtReplace 60 | case "update": 61 | return StmtUpdate 62 | case "delete": 63 | return StmtDelete 64 | case "savepoint": 65 | return StmtSavepoint 66 | } 67 | // For the following statements it is not sufficient to rely 68 | // on loweredFirstWord. This is because they are not statements 69 | // in the grammar and we are relying on Preview to parse them. 70 | // For instance, we don't want: "BEGIN JUNK" to be parsed 71 | // as StmtBegin. 72 | trimmedNoComments, _ := SplitMarginComments(trimmed) 73 | switch strings.ToLower(trimmedNoComments) { 74 | case "begin", "start transaction": 75 | return StmtBegin 76 | case "commit": 77 | return StmtCommit 78 | case "rollback": 79 | return StmtRollback 80 | } 81 | switch loweredFirstWord { 82 | case "create", "alter", "rename", "drop", "truncate", "flush": 83 | return StmtDDL 84 | case "set": 85 | return StmtSet 86 | case "show": 87 | return StmtShow 88 | case "use": 89 | return StmtUse 90 | case "describe", "desc", "explain": 91 | return StmtExplain 92 | case "analyze", "repair", "optimize": 93 | return StmtOther 94 | case "grant", "revoke": 95 | return StmtPriv 96 | case "release": 97 | return StmtRelease 98 | case "rollback": 99 | return StmtSRollback 100 | } 101 | return StmtUnknown 102 | } 103 | 104 | func (s StatementType) String() string { 105 | switch s { 106 | case StmtSelect: 107 | return "SELECT" 108 | case StmtStream: 109 | return "STREAM" 110 | case StmtInsert: 111 | return "INSERT" 112 | case StmtReplace: 113 | return "REPLACE" 114 | case StmtUpdate: 115 | return "UPDATE" 116 | case StmtDelete: 117 | return "DELETE" 118 | case StmtDDL: 119 | return "DDL" 120 | case StmtBegin: 121 | return "BEGIN" 122 | case StmtCommit: 123 | return "COMMIT" 124 | case StmtRollback: 125 | return "ROLLBACK" 126 | case StmtSet: 127 | return "SET" 128 | case StmtShow: 129 | return "SHOW" 130 | case StmtUse: 131 | return "USE" 132 | case StmtOther: 133 | return "OTHER" 134 | case StmtPriv: 135 | return "PRIV" 136 | case StmtExplain: 137 | return "EXPLAIN" 138 | case StmtSavepoint: 139 | return "SAVEPOINT" 140 | case StmtSRollback: 141 | return "SAVEPOINT_ROLLBACK" 142 | case StmtRelease: 143 | return "RELEASE" 144 | default: 145 | return "UNKNOWN" 146 | } 147 | } 148 | 149 | func (s StatementType) CanHandleWithoutPlan() bool { 150 | switch s { 151 | case StmtShow, StmtSet, StmtBegin, StmtComment, StmtRollback, StmtUse, StmtPriv, StmtSavepoint, StmtRelease: 152 | return true 153 | } 154 | return false 155 | } 156 | -------------------------------------------------------------------------------- /docs/config-reloading.md: -------------------------------------------------------------------------------- 1 | # gaea配置热加载设计与实现 2 | 3 | ## 背景 4 | 5 | 我们在开始设计gaea的时候列了几个关键词: 配置热加载、多租户、读写分离、分库分表、路由sql，配置热加载就是其中一个非常重要的目标。以往基于配置文件的方式，存在以下几个问题: DBA 6 | 使用起来比较容易出错、多个代理的配置文件维护比较麻烦、配置更新到生效的过程比较漫长、通过接口更新配置也存在文件和内存中配置可能不一致的情况。而gaea配置热加载就是希望能解决以上弊端并通过一个统一的平台进行配置管理。 7 | 8 | 设计之初，我们就把配置分为静态配置和动态配置。静态配置即在运行时不需要、也不可以进行修改的配置内容，比如监听端口、etcd地址、log路径等。动态配置即为运行时需要不断进行更改的配置，比如增删namespace、namespace内实例、用户、配置项的变更等等，动态配置是配置热加载的主角。 9 | 10 | ## 配置结构 11 | 12 | 动态配置最开始加载为models里对应的结构，比如models.Namespace，但是程序运行使用的配置为proxy/server包下的Manager。 13 | 14 | 动态配置对应结构如下: 15 | 16 | ```golang 17 | // Manager contains namespace manager and user manager 18 | type Manager struct { 19 | switchIndex util.BoolIndex 20 | namespaces [2]*NamespaceManager 21 | users [2]*UserManager 22 | statistics *StatisticManager 23 | } 24 | ``` 25 | 26 | Manager是一个全局变量，负责动态配置的管理，按照配置的适用范围分为用户配置、namespace配置和打点统计配置，而switchIndex则作为滚动数组进行配置滚动的标识。 27 | 28 | ### Namespace配置 29 | 30 | 主要包含名称、数据库白名单、用户属性、sql指纹、执行计划、分片配置等。 31 | 32 | ```golang 33 | // NamespaceManager namespace manager 34 | type NamespaceManager struct { 35 | namespaces map[string]*Namespace 36 | } 37 | // Namespace is struct driected used by server 38 | type Namespace struct { 39 | name string 40 | allowedDBs map[string]bool 41 | defaultPhyDBs map[string]string // logicDBName-phyDBName 42 | sqls map[string]string //key: parser fingerprint 43 | slowSQLTime int64 // session slow parser time, millisecond, default 1000 44 | allowips []util.IPInfo 45 | router *router.Router 46 | slices map[string]*backend.Slice // key: slice name 47 | userProperties map[string]*UserProperty // key: user name ,value: user's properties 48 | defaultCharset string 49 | defaultCollationID mysql.CollationID 50 | 51 | slowSQLCache *cache.LRUCache 52 | errorSQLCache *cache.LRUCache 53 | backendSlowSQLCache *cache.LRUCache 54 | backendErrorSQLCache *cache.LRUCache 55 | planCache *cache.LRUCache 56 | } 57 | ``` 58 | 59 | ### UserManager配置 60 | 61 | 主要包含auth阶段所需要的结构化信息 62 | 63 | ```golang 64 | // UserManager means user for auth 65 | // username+password是全局唯一的, 而username可以对应多个namespace 66 | type UserManager struct { 67 | users map[string][]string // key: user name, value: user password, same user may have different password, so array of passwords is needed 68 | userNamespaces map[string]string // key: UserName+Password, value: name of namespace 69 | } 70 | ``` 71 | 72 | ### StatisticManager配置 73 | 74 | 作为Manager里的一个子模块，用以打点统计各项指标，并非配置热加载的配置项，在此不进行过多讨论。 75 | 76 | ## 配置初始化 77 | 78 | 通过InitManager，加载所有的namespace，初始化users、namespaces、statistics，构造全局manager(globalManager) 79 | 80 | ## 配置变更接口 81 | 82 | gaea的配置变更没有针对每个配置项分别进行接口封装，而是基于配置完整替换+被替换配置动态资源延迟关闭的策略。无论是某一个还是多个配置项发生变化或者新增、删除namespace，对于gaea来说，都会构建一份新的配置并构造后端的动态资源。当新配置滚动为当前使用的配置之后，旧版本的动态资源在延迟60秒之后主动释放，进行比如关闭套接字、文件描述符等的操作。所以gaea的配置变更可以理解为，只要配置项实现了构造、延迟关闭功能，都可以纳入配置热加载模块内，进行统一管理。 83 | 84 | ## 滚动数组实现无锁化 85 | 86 | 滚动数组是配置热加载过程中经常使用的技巧，通过用空间换时间的方式，规避配置获取和配置变更之间的竟态条件。Manager中的switchIndex即为当前生效配置的下标，switchIndex的类型为BoolIndex，其对应的Get、Set方法均为原子操作，这样就保证了二元数组对应配置的切换为原子过程，而配置复制和赋值永远都是变更当前未在使用的另一元素，即通过写时复制+原子切换实现了配置的无锁化。为了防止多次滚动，导致丢失配置，在进行配置更改时，需要持有全局锁，保证同一时间，只有一个namespace进行配置变更。 87 | 88 | ## 延迟关闭回收动态资源 89 | 90 | 配置在提交之后，新配置生效，老配置需要进行资源回收。通过一个单独的goroutine，在sleep一段时间之后(尽最大努力保证请求得到应答)，调用各个单独项的Close，回收资源。 91 | 92 | ## 两阶段提交保证一致性 93 | 94 | 一个集群会包含多台gaea-proxy，为了保证多台gaea-proxy快速生效相同的配置，故而引入了两阶段提交的配置变更方式，其中协调者为gaea-cc。第一阶段: gaea-cc调用各个gaea-proxy的prepare接口，gaea-proxy在prepare阶段首先复制一份当前的全量配置，然后从etcd加载对应namespace的最新的配置，最后更新对应的全量配置；第二阶段: gaea-cc如果在prepare阶段发生错误(任何一个gaea-proxy报错)则直接报错，prepare成功后则调用gaea-proxy的commit接口，gaea-proxy在commit接口只进行一次简单的配置切换，这样prepare工作重、commit工作非常轻量，可以很大程度上提升配置变更成功的几率。如果commit失败，则gaea-cc也是直接报错，对应的web平台上看到错误后可以决定是否停止变更或者重新发起一次变更(多次发送相同配置幂等)。 95 | 96 | ## 集群配置一致性校验 97 | 98 | 通过两阶段提交配置后，当前所有gaea-proxy的生效配置是相同的。为了方便验证: 1.配置是否发生变化 2.是否所有gaea-proxy的最新配置已经生效，gaea-proxy提供了获取当前配置签名的接口。通过该接口，DBA可以直接通过管理平台查看到各个gaea-proxy前后及当前配置的md5签名，保证配置变更的执行效果符合预期。 -------------------------------------------------------------------------------- /proxy/router/router.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Gaea Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package router 15 | 16 | import ( 17 | "fmt" 18 | "strings" 19 | 20 | "github.com/XiaoMi/Gaea/models" 21 | ) 22 | 23 | type Router struct { 24 | rules map[string]map[string]Rule // dbname-tablename 25 | defaultRule Rule 26 | } 27 | 28 | //NewRouter build router according to the models of namespace 29 | func NewRouter(namespace *models.Namespace) (*Router, error) { 30 | // copy names of slice list 31 | var sliceNames []string 32 | for _, slice := range namespace.Slices { 33 | sliceNames = append(sliceNames, slice.Name) 34 | } 35 | 36 | // check default slice 37 | if !includeSlice(sliceNames, namespace.DefaultSlice) { 38 | return nil, fmt.Errorf("default slice[%s] not in the slice list", 39 | namespace.DefaultSlice) 40 | } 41 | 42 | // create router of special namespace 43 | rt := new(Router) 44 | rt.rules = make(map[string]map[string]Rule) 45 | rt.defaultRule = NewDefaultRule(namespace.DefaultSlice) 46 | 47 | linkedRuleIndexes := make([]int, 0) 48 | 49 | for i, shard := range namespace.ShardRules { 50 | for _, slice := range shard.Slices { 51 | if !includeSlice(sliceNames, slice) { 52 | return nil, fmt.Errorf("shard table[%s] slice[%s] not in the namespace.slices list:[%s]", 53 | shard.Table, slice, strings.Join(shard.Slices, ",")) 54 | } 55 | } 56 | 57 | // get index of linked table source and handle it later 58 | if shard.Type == LinkedTableRuleType { 59 | linkedRuleIndexes = append(linkedRuleIndexes, i) 60 | continue 61 | } 62 | 63 | rule, err := parseRule(shard) 64 | if err != nil { 65 | return nil, err 66 | } 67 | 68 | // if global table rule, use the namespace slice names 69 | // TODO: refactor 70 | if rule.ruleType == GlobalTableRuleType { 71 | rule.slices = sliceNames 72 | } 73 | 74 | if rule.ruleType == DefaultRuleType { 75 | return nil, fmt.Errorf("[default-rule] duplicate, must only one") 76 | } 77 | //if the database exist in rules 78 | if _, ok := rt.rules[rule.db]; ok { 79 | if _, ok := rt.rules[rule.db][rule.table]; ok { 80 | return nil, fmt.Errorf("table %s rule in %s duplicate", rule.table, rule.db) 81 | } else { 82 | rt.rules[rule.db][rule.table] = rule 83 | } 84 | } else { 85 | m := make(map[string]Rule) 86 | rt.rules[rule.db] = m 87 | rt.rules[rule.db][rule.table] = rule 88 | } 89 | } 90 | 91 | // create linked rule 92 | for _, i := range linkedRuleIndexes { 93 | rule, err := createLinkedRule(rt.rules, namespace.ShardRules[i]) 94 | if err != nil { 95 | return nil, fmt.Errorf("create linked rule error: %v", err) 96 | } 97 | rt.rules[rule.db][rule.table] = rule 98 | } 99 | 100 | return rt, nil 101 | } 102 | 103 | func (r *Router) GetShardRule(db, table string) (Rule, bool) { 104 | arry := strings.Split(table, ".") 105 | if len(arry) == 2 { 106 | table = strings.Trim(arry[1], "`") 107 | db = strings.Trim(arry[0], "`") 108 | } 109 | rule, ok := r.rules[db][table] 110 | return rule, ok 111 | } 112 | 113 | func (r *Router) GetRule(db, table string) Rule { 114 | arry := strings.Split(table, ".") 115 | if len(arry) == 2 { 116 | table = strings.Trim(arry[1], "`") 117 | db = strings.Trim(arry[0], "`") 118 | } 119 | rule := r.rules[db][table] 120 | if rule == nil { 121 | //set the database of default rule 122 | r.defaultRule.(*BaseRule).db = db 123 | return r.defaultRule 124 | } else { 125 | return rule 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /backend/pooled_connection.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Gaea Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package backend 16 | 17 | import ( 18 | "github.com/XiaoMi/Gaea/mysql" 19 | ) 20 | 21 | // PooledConnect app use this object to exec parser 22 | type pooledConnectImpl struct { 23 | directConnection *DirectConnection 24 | pool *connectionPoolImpl 25 | } 26 | 27 | // Recycle return PooledConnect to the pool 28 | func (pc *pooledConnectImpl) Recycle() { 29 | if pc.IsClosed() { 30 | pc.pool.Put(nil) 31 | } else { 32 | pc.pool.Put(pc) 33 | } 34 | } 35 | 36 | // Reconnect replaces the existing underlying connection with a new one. 37 | // If we get "MySQL server has gone away (errno 2006)", then call Reconnect 38 | func (pc *pooledConnectImpl) Reconnect() error { 39 | pc.directConnection.Close() 40 | newConn, err := NewDirectConnection(pc.pool.addr, pc.pool.user, pc.pool.password, pc.pool.db, pc.pool.charset, pc.pool.collationID) 41 | if err != nil { 42 | return err 43 | } 44 | pc.directConnection = newConn 45 | return nil 46 | } 47 | 48 | // Close implement util.Resource interface 49 | func (pc *pooledConnectImpl) Close() { 50 | pc.directConnection.Close() 51 | } 52 | 53 | // IsClosed check if pooled connection closed 54 | func (pc *pooledConnectImpl) IsClosed() bool { 55 | if pc.directConnection == nil { 56 | return true 57 | } 58 | return pc.directConnection.IsClosed() 59 | } 60 | 61 | // UseDB wrapper of direct connection, init database 62 | func (pc *pooledConnectImpl) UseDB(db string) error { 63 | return pc.directConnection.UseDB(db) 64 | } 65 | 66 | // Execute wrapper of direct connection, execute parser 67 | func (pc *pooledConnectImpl) Execute(sql string) (*mysql.Result, error) { 68 | return pc.directConnection.Execute(sql) 69 | } 70 | 71 | // SetAutoCommit wrapper of direct connection, set autocommit 72 | func (pc *pooledConnectImpl) SetAutoCommit(v uint8) error { 73 | return pc.directConnection.SetAutoCommit(v) 74 | } 75 | 76 | // Begin wrapper of direct connection, begin transaction 77 | func (pc *pooledConnectImpl) Begin() error { 78 | return pc.directConnection.Begin() 79 | } 80 | 81 | // Commit wrapper of direct connection, commit transaction 82 | func (pc *pooledConnectImpl) Commit() error { 83 | return pc.directConnection.Commit() 84 | } 85 | 86 | // Rollback wrapper of direct connection, rollback transaction 87 | func (pc *pooledConnectImpl) Rollback() error { 88 | return pc.directConnection.Rollback() 89 | } 90 | 91 | // SetCharset wrapper of direct connection, set charset of connection 92 | func (pc *pooledConnectImpl) SetCharset(charset string, collation mysql.CollationID) (bool, error) { 93 | return pc.directConnection.SetCharset(charset, collation) 94 | } 95 | 96 | // FieldList wrapper of direct connection, send field list to mysql 97 | func (pc *pooledConnectImpl) FieldList(table string, wildcard string) ([]*mysql.Field, error) { 98 | return pc.directConnection.FieldList(table, wildcard) 99 | } 100 | 101 | // GetAddr wrapper of return addr of direct connection 102 | func (pc *pooledConnectImpl) GetAddr() string { 103 | return pc.directConnection.GetAddr() 104 | } 105 | 106 | // SetSessionVariables set pc variables according to session 107 | func (pc *pooledConnectImpl) SetSessionVariables(frontend *mysql.SessionVariables) (bool, error) { 108 | return pc.directConnection.SetSessionVariables(frontend) 109 | } 110 | 111 | // WriteSetStatement exec parser 112 | func (pc *pooledConnectImpl) WriteSetStatement() error { 113 | return pc.directConnection.WriteSetStatement() 114 | } 115 | -------------------------------------------------------------------------------- /util/timer/timer.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 Google Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package timer provides various enhanced timer functions. 18 | package timer 19 | 20 | import ( 21 | "sync" 22 | "time" 23 | 24 | "github.com/XiaoMi/Gaea/util/sync2" 25 | ) 26 | 27 | // Out-of-band messages 28 | type typeAction int 29 | 30 | const ( 31 | timerStop typeAction = iota 32 | timerReset 33 | timerTrigger 34 | ) 35 | 36 | /* 37 | Timer provides timer functionality that can be controlled 38 | by the user. You start the timer by providing it a callback function, 39 | which it will call at the specified interval. 40 | 41 | var t = timer.NewTimer(1e9) 42 | t.Start(KeepHouse) 43 | 44 | func KeepHouse() { 45 | // do house keeping work 46 | } 47 | 48 | You can stop the timer by calling t.Stop, which is guaranteed to 49 | wait if KeepHouse is being executed. 50 | 51 | You can create an untimely trigger by calling t.Trigger. You can also 52 | schedule an untimely trigger by calling t.TriggerAfter. 53 | 54 | The timer interval can be changed on the fly by calling t.SetInterval. 55 | A zero value interval will cause the timer to wait indefinitely, and it 56 | will react only to an explicit Trigger or Stop. 57 | */ 58 | type Timer struct { 59 | interval sync2.AtomicDuration 60 | 61 | // state management 62 | mu sync.Mutex 63 | running bool 64 | 65 | // msg is used for out-of-band messages 66 | msg chan typeAction 67 | } 68 | 69 | // NewTimer creates a new Timer object 70 | func NewTimer(interval time.Duration) *Timer { 71 | tm := &Timer{ 72 | msg: make(chan typeAction), 73 | } 74 | tm.interval.Set(interval) 75 | return tm 76 | } 77 | 78 | // Start starts the timer. 79 | func (tm *Timer) Start(keephouse func()) { 80 | tm.mu.Lock() 81 | defer tm.mu.Unlock() 82 | if tm.running { 83 | return 84 | } 85 | tm.running = true 86 | go tm.run(keephouse) 87 | } 88 | 89 | func (tm *Timer) run(keephouse func()) { 90 | for { 91 | var ch <-chan time.Time 92 | interval := tm.interval.Get() 93 | if interval <= 0 { 94 | ch = nil 95 | } else { 96 | ch = time.After(interval) 97 | } 98 | select { 99 | case action := <-tm.msg: 100 | switch action { 101 | case timerStop: 102 | return 103 | case timerReset: 104 | continue 105 | } 106 | case <-ch: 107 | } 108 | keephouse() 109 | } 110 | } 111 | 112 | // SetInterval changes the wait interval. 113 | // It will cause the timer to restart the wait. 114 | func (tm *Timer) SetInterval(ns time.Duration) { 115 | tm.interval.Set(ns) 116 | tm.mu.Lock() 117 | defer tm.mu.Unlock() 118 | if tm.running { 119 | tm.msg <- timerReset 120 | } 121 | } 122 | 123 | // Trigger will cause the timer to immediately execute the keephouse function. 124 | // It will then cause the timer to restart the wait. 125 | func (tm *Timer) Trigger() { 126 | tm.mu.Lock() 127 | defer tm.mu.Unlock() 128 | if tm.running { 129 | tm.msg <- timerTrigger 130 | } 131 | } 132 | 133 | // TriggerAfter waits for the specified duration and triggers the next event. 134 | func (tm *Timer) TriggerAfter(duration time.Duration) { 135 | go func() { 136 | time.Sleep(duration) 137 | tm.Trigger() 138 | }() 139 | } 140 | 141 | // Stop will stop the timer. It guarantees that the timer will not execute 142 | // any more calls to keephouse once it has returned. 143 | func (tm *Timer) Stop() { 144 | tm.mu.Lock() 145 | defer tm.mu.Unlock() 146 | if tm.running { 147 | tm.msg <- timerStop 148 | tm.running = false 149 | } 150 | } 151 | 152 | // Interval returns the current interval. 153 | func (tm *Timer) Interval() time.Duration { 154 | return tm.interval.Get() 155 | } 156 | -------------------------------------------------------------------------------- /mysql/resultset_sort.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The kingshard Authors. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | // not use this file except in compliance with the License. You may obtain 5 | // a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package mysql 16 | 17 | import ( 18 | "bytes" 19 | "fmt" 20 | "sort" 21 | 22 | "github.com/XiaoMi/Gaea/util/hack" 23 | ) 24 | 25 | const ( 26 | // SortAsc asc 27 | SortAsc = "asc" 28 | // SortDesc desc 29 | SortDesc = "desc" 30 | ) 31 | 32 | // SortKey key will sort 33 | type SortKey struct { 34 | //name of the field 35 | Name string 36 | 37 | Direction string 38 | 39 | //column index of the field 40 | Column int 41 | } 42 | 43 | // ResultsetSorter contains resultset will sort 44 | type ResultsetSorter struct { 45 | *Resultset 46 | 47 | sk []SortKey 48 | } 49 | 50 | func newResultsetSorter(r *Resultset, sk []SortKey) (*ResultsetSorter, error) { 51 | s := new(ResultsetSorter) 52 | 53 | s.Resultset = r 54 | 55 | for i, k := range sk { 56 | if column, ok := r.FieldNames[k.Name]; ok { 57 | sk[i].Column = column 58 | } else { 59 | return nil, fmt.Errorf("key %s not in Resultset fields, can not sort", k.Name) 60 | } 61 | } 62 | 63 | s.sk = sk 64 | 65 | return s, nil 66 | } 67 | 68 | func newResultsetSorterWithoutColumnName(r *Resultset, sk []SortKey) *ResultsetSorter { 69 | return &ResultsetSorter{ 70 | Resultset: r, 71 | sk: sk, 72 | } 73 | } 74 | 75 | func (r *ResultsetSorter) Len() int { 76 | return r.RowNumber() 77 | } 78 | 79 | func (r *ResultsetSorter) Less(i, j int) bool { 80 | v1 := r.Values[i] 81 | v2 := r.Values[j] 82 | 83 | for _, k := range r.sk { 84 | v := cmpValue(v1[k.Column], v2[k.Column]) 85 | 86 | if k.Direction == SortDesc { 87 | v = -v 88 | } 89 | 90 | if v < 0 { 91 | return true 92 | } else if v > 0 { 93 | return false 94 | } 95 | 96 | //equal, cmp next key 97 | } 98 | 99 | return false 100 | } 101 | 102 | //compare value using asc 103 | func cmpValue(v1 interface{}, v2 interface{}) int { 104 | if v1 == nil && v2 == nil { 105 | return 0 106 | } else if v1 == nil { 107 | return -1 108 | } else if v2 == nil { 109 | return 1 110 | } 111 | 112 | switch v := v1.(type) { 113 | case string: 114 | s := v2.(string) 115 | return bytes.Compare(hack.Slice(v), hack.Slice(s)) 116 | case []byte: 117 | s := v2.([]byte) 118 | return bytes.Compare(v, s) 119 | case int64: 120 | s := v2.(int64) 121 | if v < s { 122 | return -1 123 | } else if v > s { 124 | return 1 125 | } else { 126 | return 0 127 | } 128 | case uint64: 129 | s := v2.(uint64) 130 | if v < s { 131 | return -1 132 | } else if v > s { 133 | return 1 134 | } else { 135 | return 0 136 | } 137 | case float64: 138 | s := v2.(float64) 139 | if v < s { 140 | return -1 141 | } else if v > s { 142 | return 1 143 | } else { 144 | return 0 145 | } 146 | default: 147 | //can not go here 148 | panic(fmt.Sprintf("invalid type %T", v)) 149 | } 150 | } 151 | 152 | func (r *ResultsetSorter) Swap(i, j int) { 153 | r.Values[i], r.Values[j] = r.Values[j], r.Values[i] 154 | 155 | // 有可能此时RowData已经被清除了, 防止panic 156 | if len(r.RowDatas) != 0 { 157 | r.RowDatas[i], r.RowDatas[j] = r.RowDatas[j], r.RowDatas[i] 158 | } 159 | } 160 | 161 | // Sort sort resultset 162 | func (r *Resultset) Sort(sk []SortKey) error { 163 | s, err := newResultsetSorter(r, sk) 164 | 165 | if err != nil { 166 | return err 167 | } 168 | 169 | sort.Sort(s) 170 | 171 | return nil 172 | } 173 | 174 | // SortWithoutColumnName 只使用SortKey中的column来获取列信息, 不使用Name 175 | func (r *Resultset) SortWithoutColumnName(sk []SortKey) error { 176 | s := newResultsetSorterWithoutColumnName(r, sk) 177 | sort.Sort(s) 178 | return nil 179 | } 180 | --------------------------------------------------------------------------------