3 |
Bucket Info
4 | {{ range $addr, $host:= .bucketinfo}}
5 | {{ range $score, $consistent:= $host}}
6 |
7 |
8 |
9 |
10 | | Host |
11 | Score |
12 | Percentage |
13 |
14 |
15 |
16 | {{ range $arc, $resstats := $consistent}}
17 |
18 | | {{ $addr}} |
19 | {{ $score}} |
20 | {{ $arc}} |
21 |
22 |
23 | | average |
24 | Count |
25 | SumTime |
26 |
27 |
28 | {{ range $_, $response := $resstats }}
29 |
30 | {{ if gt $response.Count 0 }}
31 | | {{divide $response.Sum $response.Count }} |
32 | {{ else }}
33 | 0 |
34 | {{ end }}
35 | {{ $response.Count }} |
36 | {{ $response.Sum }} |
37 |
38 | {{ end }}
39 | {{ end }}
40 |
41 |
42 |
43 | {{ end }}
44 | {{ end }}
45 |
35 | {{ template "body" . }}
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
--------------------------------------------------------------------------------
/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/douban/gobeansproxy
2 |
3 | require (
4 | github.com/acomagu/trie/v2 v2.0.0
5 | github.com/douban/gobeansdb v1.1.3
6 | github.com/gocql/gocql v1.5.2
7 | github.com/sirupsen/logrus v1.9.3
8 | github.com/stretchr/testify v1.8.0
9 | gopkg.in/natefinch/lumberjack.v2 v2.2.1
10 | gopkg.in/yaml.v2 v2.4.0
11 | gopkg.in/yaml.v3 v3.0.1
12 | )
13 |
14 | require (
15 | github.com/beorn7/perks v1.0.1 // indirect
16 | github.com/cespare/xxhash/v2 v2.2.0 // indirect
17 | github.com/davecgh/go-spew v1.1.1 // indirect
18 | github.com/golang/protobuf v1.5.3 // indirect
19 | github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
20 | github.com/kr/text v0.2.0 // indirect
21 | github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
22 | github.com/pmezard/go-difflib v1.0.0 // indirect
23 | github.com/prometheus/client_model v0.3.0 // indirect
24 | github.com/prometheus/common v0.42.0 // indirect
25 | github.com/prometheus/procfs v0.10.1 // indirect
26 | github.com/rogpeppe/go-internal v1.11.0 // indirect
27 | github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da // indirect
28 | github.com/spaolacci/murmur3 v1.1.0 // indirect
29 | golang.org/x/exp v0.0.0-20230310171629-522b1b587ee0 // indirect
30 | golang.org/x/sys v0.10.0 // indirect
31 | google.golang.org/protobuf v1.30.0 // indirect
32 | gopkg.in/inf.v0 v0.9.1 // indirect
33 | )
34 |
35 | require (
36 | github.com/golang/snappy v0.0.4 // indirect
37 | github.com/prometheus/client_golang v1.16.0
38 | golang.org/x/sync v0.3.0
39 | )
40 |
41 | go 1.20
42 |
43 | // for lcoal dev
44 | // replace github.com/douban/gobeansdb => ../gobeansdb
45 |
46 | // replace github.com/douban/gobeansproxy => ../gobeansproxy
47 |
--------------------------------------------------------------------------------
/cassandra/logger.go:
--------------------------------------------------------------------------------
1 | package cassandra
2 |
3 | import (
4 | "fmt"
5 | "os"
6 | "path/filepath"
7 |
8 | "github.com/douban/gobeansproxy/config"
9 | logrus "github.com/sirupsen/logrus"
10 | rotateLogger "gopkg.in/natefinch/lumberjack.v2"
11 | )
12 |
13 | var (
14 | log = logrus.New()
15 | dumpLogger = logrus.New()
16 | )
17 |
18 | func setLogLevel(logLevel string) {
19 | l, err := logrus.ParseLevel(logLevel)
20 | if err != nil {
21 | log.Warnf("log level no supported will use info level (passed %s)", logLevel)
22 | }
23 | log.SetLevel(l)
24 | log.SetFormatter(&logrus.TextFormatter{
25 | DisableColors: false,
26 | FullTimestamp: true,
27 | })
28 | }
29 |
30 | type DualWriteErrorMgr struct {
31 | EFile string
32 | ELogger *logrus.Logger
33 | }
34 |
35 | func NewDualWErrMgr(ecfg *config.DualWErrCfg, logger *logrus.Logger) (*DualWriteErrorMgr, error) {
36 | if logger == nil {
37 | logger = dumpLogger
38 | }
39 |
40 | setLogLevel(ecfg.LoggerLevel)
41 |
42 | // check if target folder exists
43 | if stat, err := os.Stat(ecfg.DumpToDir); err != nil || !stat.IsDir() {
44 | return nil, fmt.Errorf("%s is not a dir or not exists", ecfg.DumpToDir)
45 | }
46 |
47 | // set dump Logger
48 | logger.SetFormatter(&logrus.JSONFormatter{})
49 | dumpFile := filepath.Join(ecfg.DumpToDir, ecfg.FName)
50 | logger.SetOutput(&rotateLogger.Logger{
51 | Filename: dumpFile,
52 | MaxSize: ecfg.RotateSize,
53 | Compress: ecfg.Compress,
54 | MaxAge: ecfg.MaxAges,
55 | MaxBackups: ecfg.MaxBackups,
56 | })
57 |
58 | return &DualWriteErrorMgr{
59 | EFile: dumpFile,
60 | ELogger: logger,
61 | }, nil
62 | }
63 |
64 | func (e *DualWriteErrorMgr) HandleErr(key, op string, err error) {
65 | e.ELogger.WithFields(logrus.Fields{
66 | "key": key,
67 | "op": op,
68 | }).Error(err)
69 | }
70 |
--------------------------------------------------------------------------------
/dstore/consistent_test.go:
--------------------------------------------------------------------------------
1 | package dstore
2 |
3 | import (
4 | "fmt"
5 | "strconv"
6 | "testing"
7 |
8 | "github.com/stretchr/testify/assert"
9 | )
10 |
11 | // 基本功能。
12 | func TestConsistent(t *testing.T) {
13 | assert := assert.New(t)
14 |
15 | hashs := NewPartition(100, 3)
16 |
17 | // 计数器
18 | counter := make(map[int]int)
19 | loop := 1000000
20 | threshold := loop / 3 / 10 * 4
21 |
22 | // 统计顺序 key 分布。
23 | for i := 0; i < loop; i++ {
24 | s := hashs.offsetGet(fmt.Sprintf("key%d", i))
25 | counter[s] += 1
26 | }
27 |
28 | // 检查计数是否超过最低阈值。
29 | for _, v := range counter {
30 | assert.True(v > threshold)
31 | }
32 | }
33 |
34 | // 阶段故障
35 | func TestConsistentRemove(t *testing.T) {
36 | assert := assert.New(t)
37 | hashs := NewPartition(100, 3)
38 | hashs.remove(0)
39 | loop := 1000000
40 | for i := 0; i < loop; i++ {
41 | s := hashs.offsetGet(fmt.Sprintf("key%d", i))
42 | assert.NotEqual(s, 0)
43 | }
44 | t.Log(hashs)
45 | hashs.remove(0)
46 | t.Log(hashs)
47 | }
48 |
49 | func TestConsistentBalance(t *testing.T) {
50 | assert := assert.New(t)
51 | hashs := NewPartition(100, 3)
52 | hashs.reBalance(0, 2, 100)
53 | assert.Equal(hashs.offsets[2], 99)
54 | }
55 |
56 | func TestConsistentMulti(t *testing.T) {
57 | assert := assert.New(t)
58 |
59 | hashs1 := NewPartition(100, 3)
60 |
61 | hashs2 := NewPartition(100, 3)
62 |
63 | // 多次访问结果一致。
64 | s1 := hashs1.offsetGet("abc")
65 | s2 := hashs1.offsetGet("abc")
66 | assert.Equal(s1, s2)
67 |
68 | s1 = hashs1.offsetGet("abc")
69 | s2 = hashs2.offsetGet("abc")
70 | assert.Equal(s1, s2)
71 |
72 | }
73 |
74 | // 哈希函数性能。
75 | func BenchmarkConsistentHash(b *testing.B) {
76 | h := NewPartition(100, 3)
77 |
78 | b.ResetTimer()
79 | for i := 0; i < b.N; i++ {
80 | h.hash("a" + strconv.Itoa(i))
81 | }
82 | b.StopTimer()
83 | }
84 |
--------------------------------------------------------------------------------
/cassandra/udt.go:
--------------------------------------------------------------------------------
1 | package cassandra
2 |
3 | import (
4 | "fmt"
5 | "time"
6 |
7 | mc "github.com/douban/gobeansdb/memcache"
8 | "github.com/gocql/gocql"
9 | )
10 |
11 | type BDBValue struct {
12 | ReceiveTime time.Time `cql:"rtime"`
13 | Flag int `cql:"flag"`
14 | Exptime int `cql:"exptime"`
15 | Cas int `cql:"cas"`
16 | Body []byte `cql:"body"`
17 | }
18 |
19 | func NewBDBValue(item *mc.Item) *BDBValue {
20 | return &BDBValue{
21 | ReceiveTime: item.ReceiveTime,
22 | Flag: item.Flag,
23 | Exptime: item.Exptime,
24 | Cas: item.Cas,
25 | Body: item.CArray.Body,
26 | }
27 | }
28 |
29 | func (b *BDBValue) ToMCItem() (*mc.Item, error) {
30 | item := &mc.Item{
31 | ReceiveTime: b.ReceiveTime,
32 | Flag: b.Flag,
33 | Exptime: b.Exptime,
34 | Cas: b.Cas,
35 | }
36 | ok := item.Alloc(len(b.Body))
37 | if !ok {
38 | logger.Errorf("Alloc mem err for len %d", len(b.Body))
39 | return nil, fmt.Errorf("alloc mem error")
40 | }
41 | copy(item.CArray.Body, b.Body)
42 | return item, nil
43 | }
44 |
45 | func (b BDBValue) MarshalUDT(name string, info gocql.TypeInfo) ([]byte, error) {
46 | switch name {
47 | case "rtime":
48 | return gocql.Marshal(info, b.ReceiveTime)
49 | case "flag":
50 | return gocql.Marshal(info, b.Flag)
51 | case "exptime":
52 | return gocql.Marshal(info, b.Exptime)
53 | case "cas":
54 | return gocql.Marshal(info, b.Cas)
55 | case "body":
56 | return gocql.Marshal(info, b.Body)
57 | default:
58 | return nil, fmt.Errorf("unknown column for position: %q", name)
59 | }
60 | }
61 |
62 | func (b *BDBValue) UnmarshalUDT(name string, info gocql.TypeInfo, data []byte) error {
63 | switch name {
64 | case "rtime":
65 | return gocql.Unmarshal(info, data, &b.ReceiveTime)
66 | case "flag":
67 | return gocql.Unmarshal(info, data, &b.Flag)
68 | case "exptime":
69 | return gocql.Unmarshal(info, data, &b.Exptime)
70 | case "cas":
71 | return gocql.Unmarshal(info, data, &b.Cas)
72 | case "body":
73 | return gocql.Unmarshal(info, data, &b.Body)
74 | default:
75 | return fmt.Errorf("unknown column for position: %q", name)
76 | }
77 |
78 | }
79 |
--------------------------------------------------------------------------------
/dstore/ringqueue.go:
--------------------------------------------------------------------------------
1 | package dstore
2 |
3 | import (
4 | "errors"
5 | "sync"
6 | "time"
7 | )
8 |
9 | const (
10 | TIMEINTERVAL = 30 * 1000 * 1000 * 1000 // proxy 链接 后端 超时时间 为 3 秒，清空 30s 之前的数据，30 * 1000 * 1000 * 1000
11 | QUEUECAP = 60
12 | )
13 |
14 | const (
15 | latencyDataType = iota
16 | errorDataType
17 | )
18 |
19 | type Response struct {
20 | ReqTime time.Time
21 | Count int
22 | Sum float64
23 | }
24 |
25 | type RingQueue struct {
26 | resData *[QUEUECAP]Response
27 | errData *[QUEUECAP]Response
28 | sync.RWMutex
29 | }
30 |
31 | var (
32 | ErrQueueFull = errors.New("queue full")
33 | ErrQueueEmpty = errors.New("queue empty")
34 | )
35 |
36 | func NewRingQueue() *RingQueue {
37 | return &RingQueue{
38 | resData: &[QUEUECAP]Response{},
39 | errData: &[QUEUECAP]Response{},
40 | }
41 | }
42 |
43 | func (q *RingQueue) Push(start time.Time, ResTime float64, dataType int) error {
44 | second := start.Second()
45 | var data *[QUEUECAP]Response
46 | switch dataType {
47 | case latencyDataType:
48 | data = q.resData
49 | case errorDataType:
50 | data = q.errData
51 | }
52 |
53 | // TODO errData/ resData 锁分开:
54 | q.Lock()
55 | defer q.Unlock()
56 | if start.Sub(data[second].ReqTime) > TIMEINTERVAL {
57 | data[second].Sum = ResTime
58 | data[second].Count = 1
59 | data[second].ReqTime = start
60 | }
61 | data[second].Sum += ResTime
62 | data[second].ReqTime = start
63 | data[second].Count++
64 |
65 | return nil
66 | }
67 |
68 | func (q *RingQueue) Get(num, dataType int) (responses []Response) {
69 | now := time.Now()
70 | second := now.Second()
71 | offset := second - num
72 |
73 | var data *[QUEUECAP]Response
74 | switch dataType {
75 | case latencyDataType:
76 | data = q.resData
77 | case errorDataType:
78 | data = q.errData
79 | }
80 | q.RLock()
81 | defer q.RUnlock()
82 | if offset > 0 {
83 | return data[offset:second]
84 | } else {
85 | return append(data[len(q.resData)+offset:], data[0:second]...)
86 | }
87 | }
88 |
89 | func (q *RingQueue) clear() {
90 | q.Lock()
91 | q.Unlock()
92 | for i := 0; i < QUEUECAP; i++ {
93 | q.errData[i] = Response{}
94 | q.resData[i] = Response{}
95 | }
96 | }
97 |
--------------------------------------------------------------------------------
/gobeansproxy/gobeansproxy.go:
--------------------------------------------------------------------------------
1 | package gobeansproxy
2 |
3 | import (
4 | "flag"
5 | "fmt"
6 | "log"
7 | "runtime"
8 |
9 | dbcfg "github.com/douban/gobeansdb/config"
10 | "github.com/douban/gobeansdb/loghub"
11 | mc "github.com/douban/gobeansdb/memcache"
12 |
13 | "github.com/douban/gobeansproxy/config"
14 | "github.com/douban/gobeansproxy/dstore"
15 | )
16 |
17 | var (
18 | server *mc.Server
19 | proxyConf = &config.Proxy
20 | logger = loghub.ErrorLogger
21 | accessLogger = loghub.AccessLogger
22 | )
23 |
24 | func Main() {
25 | var version = flag.Bool("version", false, "print vresion of beansproxy")
26 | var confdir = flag.String("confdir", "", "path of proxy config dir")
27 | var dumpconf = flag.Bool("dumpconf", false, "print configuration")
28 | flag.Parse()
29 |
30 | if *version {
31 | fmt.Println("gobeansproxy version", config.Version)
32 | return
33 | } else {
34 | log.Printf("gobeansproxy version %s", config.Version)
35 | }
36 |
37 | proxyConf.InitDefault()
38 | if *confdir != "" {
39 | log.Printf("use confdir %s", *confdir)
40 | proxyConf.Load(*confdir)
41 | }
42 | log.Printf("server port: %d, web port: %d", proxyConf.Port, proxyConf.WebPort)
43 |
44 | if *dumpconf {
45 | config.DumpConfig(proxyConf)
46 | return
47 | }
48 |
49 | runtime.GOMAXPROCS(proxyConf.Threads)
50 |
51 | loghub.InitLogger(proxyConf.ErrorLog, proxyConf.AccessLog, proxyConf.AnalysisLog)
52 | logger.Infof("start gobeansproxy")
53 | logger.Infof("gobeansproxy version %s starting at %d, config: %#v",
54 | config.Version, proxyConf.Port, proxyConf)
55 | logger.Infof("route table: %#v", config.Route)
56 |
57 | if proxyConf.DStoreConfig.Enable {
58 | dstore.InitGlobalManualScheduler(config.Route, proxyConf.N, proxyConf.Scheduler)
59 | }
60 | storage := new(dstore.Storage)
61 | err := storage.InitStorageEngine(proxyConf)
62 | if err != nil {
63 | log.Fatalf("Init storage engine err: %s", err)
64 | }
65 | addr := fmt.Sprintf("%s:%d", proxyConf.Listen, proxyConf.Port)
66 | server = mc.NewServer(storage)
67 | server.Listen(addr)
68 |
69 | logger.Infof("ready")
70 | log.Printf("ready")
71 |
72 | server.HandleSignals(proxyConf.ErrorLog, proxyConf.AccessLog, proxyConf.AnalysisLog)
73 | dbcfg.AllowReload = true
74 | startWeb()
75 | server.Serve()
76 | }
77 |
--------------------------------------------------------------------------------
/tests/utils.py:
--------------------------------------------------------------------------------
1 | import os
2 | import errno
3 | import yaml
4 | try:
5 | from yaml import Cloader as Loader
6 | except ImportError:
7 | from yaml import Loader
8 | import string
9 | import socket
10 | import random
11 | import urllib.request, urllib.error, urllib.parse
12 | import time
13 | import shlex
14 | import subprocess
15 |
16 |
17 | def mkdir_p(path):
18 | "like `mkdir -p`"
19 | try:
20 | os.makedirs(path)
21 | except OSError as exc:
22 | if exc.errno == errno.EEXIST and os.path.isdir(path):
23 | pass
24 | else:
25 | raise
26 |
27 |
28 | def random_string(n):
29 | s = string.ascii_letters
30 | result = ""
31 | for _ in range(n):
32 | result += random.choice(s)
33 | return result
34 |
35 |
36 | def gethttp(addr, path):
37 | url = "http://%s/%s" % (addr, path)
38 | response = urllib.request.urlopen(url)
39 | return response.read()
40 |
41 |
42 | def start_cmd(cmd):
43 | print("start", cmd)
44 | log_file = '/tmp/beansdb/log.txt'
45 | mkdir_p(os.path.dirname(log_file))
46 | with open(log_file, 'a') as f:
47 | p = subprocess.Popen(
48 | cmd if isinstance(cmd, (tuple, list)) else shlex.split(cmd),
49 | stderr=f,
50 | )
51 | time.sleep(0.2)
52 | if p.poll() is not None:
53 | raise Exception("cannot start %s" % (cmd))
54 | return p
55 |
56 | def stop_cmd(popen):
57 | if popen.poll() is not None:
58 | return
59 | popen.terminate()
60 | popen.wait()
61 |
62 |
63 | def get_server_addr(conf_dir, server_name):
64 | if server_name == 'gobeansdb':
65 | conf_file = 'global.yaml'
66 | key = 'server'
67 | elif server_name == 'gobeansproxy':
68 | conf_file = 'proxy.yaml'
69 | key = 'proxy'
70 | else:
71 | raise ValueError(server_name)
72 | conf = load_yaml(os.path.join(conf_dir, conf_file))
73 | port = conf[key]['port']
74 | webport = conf[key]['webport']
75 | host = conf[key]['hostname'] or socket.gethostname()
76 |
77 | def port_to_addr(port):
78 | return '%s:%s' % (host, port)
79 |
80 | return list(map(port_to_addr, [port, webport]))
81 |
82 |
83 | def load_yaml(filepath):
84 | with open(filepath) as f:
85 | return yaml.load(f, Loader=Loader)
86 |
--------------------------------------------------------------------------------
/cassandra/prefix_cfg.go:
--------------------------------------------------------------------------------
1 | package cassandra
2 |
3 | import (
4 | "fmt"
5 |
6 | "github.com/douban/gobeansproxy/config"
7 | )
8 |
9 | type PrefixDisPatcher interface {
10 | LoadStaticCfg(string) (*config.CassandraStoreCfg, error)
11 | LoadCfg(*config.CassandraStoreCfg, *CassandraStore) error
12 | Upsert(*config.CassandraStoreCfg, map[string][]string, *CassandraStore) error
13 | DeletePrefix(*config.CassandraStoreCfg, string, *CassandraStore) error
14 | GetCurrentMap() map[string]string
15 | }
16 |
17 | type DisPatcherCfg config.PrefixDisPatcherCfg
18 |
19 | func (config *DisPatcherCfg) LoadFromDB(
20 | cqlStore *CassandraStore) (prefixKeys [][]rune, vstatus []string, err error) {
21 | r := cqlStore.session.Query(
22 | fmt.Sprintf(
23 | "select prefix, value from %s.%s",
24 | config.CfgFromCstarKeySpace,
25 | config.CfgFromCstarTable,
26 | ),
27 | ).Iter().Scanner()
28 |
29 | for r.Next() {
30 | var (
31 | prefix string
32 | value string
33 | )
34 |
35 | err := r.Scan(&prefix, &value)
36 | if err != nil {
37 | return nil, nil, fmt.Errorf("load cfg from c* table err: %s", err)
38 | }
39 |
40 | prefixKeys = append(prefixKeys, []rune(prefix))
41 | vstatus = append(vstatus, value)
42 | }
43 |
44 | if err := r.Err(); err != nil {
45 | return nil, nil, fmt.Errorf("load cfg from c* iter err: %s", err)
46 | }
47 |
48 | return prefixKeys, vstatus, err
49 | }
50 |
51 | func (c *DisPatcherCfg) SaveToDB(m map[string][]string, cqlStore *CassandraStore) error {
52 | for value, prefix := range m {
53 | for _, p := range prefix {
54 | err := cqlStore.session.Query(
55 | fmt.Sprintf(
56 | "insert into %s.%s (prefix, value) values (?, ?)",
57 | c.CfgFromCstarKeySpace, c.CfgFromCstarTable,
58 | ), p, value,
59 | ).Exec()
60 |
61 | if err != nil {
62 | return fmt.Errorf("insert %s -> %s err: %s", p, value)
63 | }
64 | }
65 | }
66 | return nil
67 | }
68 |
69 | func (c *DisPatcherCfg) DeletePrefixCfg(prefix string, cqlStore *CassandraStore) error {
70 | err := cqlStore.session.Query(
71 | fmt.Sprintf(
72 | "delete from %s.%s where prefix = ?",
73 | c.CfgFromCstarKeySpace, c.CfgFromCstarTable,
74 | ), prefix,
75 | ).Exec()
76 |
77 | if err != nil {
78 | return fmt.Errorf("delete cfg prefix %s err: %s", prefix, err)
79 | }
80 | return nil
81 | }
82 |
--------------------------------------------------------------------------------
/.doubanpde/scripts/bdb/gobeansproxy/dstore-only/conf/proxy.yaml:
--------------------------------------------------------------------------------
1 | cassandra:
2 | default_key_space: doubandb
3 | default_table: kvstore
4 | enable: false
5 | hosts:
6 | - 127.0.0.1:9042
7 | timeout_ms: 1000
8 | connect_timeout_ms: 3000
9 | write_timeout_ms: 1000
10 | retry_num: 3
11 | reconnect_interval_sec: 180
12 | max_conn_for_getm: 10
13 | num_conns: 10
14 | username: "doubandb_test"
15 | password: "doubandb_test"
16 | consistency: "local_one"
17 | prefix_table_dispatcher_cfg:
18 | # if not enable will use default keyspace and table
19 | enable: true
20 | static:
21 | # dispatch prefix1 key to table table_name1
22 | kvstore_prefix_a:
23 | - "/prefix_a"
24 | cfg_table: bdb_prefix_table_finder
25 | cfg_keyspace: doubandb
26 | prefix_rw_dispatcher_cfg:
27 | enable: true
28 | static:
29 | # dispatch prefix /test_prefix_c/ to dual write
30 | br1w1cr0w1:
31 | - "/test_prefix_c/"
32 | - "/test_prefix_d/"
33 | br0w0cr1w1:
34 | - "test_"
35 | cfg_table: bdb_prefix_rw_switcher
36 | cfg_keyspace: doubandb
37 | default_storage: "br1w1cr0w0"
38 | dual_write_err_cfg:
39 | dump_to_dir: /tmp/log/gobeansproxy/proxy/
40 | log_file_name: dual_write_err.log
41 | logger_level: "INFO"
42 | rotate_size_mb: 100
43 | compress: true
44 | max_ages: 7
45 | max_backups: 100
46 | dstore:
47 | enable: true
48 | connect_timeout_ms: 300
49 | dial_fail_silence_ms: 5000
50 | error_seconds: 10
51 | item_size_stats: 4096
52 | max_connect_errors: 3
53 | max_free_conns_per_host: 20
54 | n: 3
55 | r: 1
56 | read_timeout_ms: 2000
57 | response_time_min: 4000
58 | response_time_seconds: 10
59 | score_deviation: 10
60 | w: 2
61 | write_timeout_ms: 2000
62 | mc:
63 | body_big_str: 5M
64 | body_c_str: 0K
65 | body_max_str: 50M
66 | max_key_len: 250
67 | max_req: 16
68 | proxy:
69 | accesslog: /tmp/log/gobeansproxy/proxy/access.log
70 | errorlog: /tmp/log/gobeansproxy/proxy/error.log
71 | hostname: 127.0.0.1
72 | listen: 0.0.0.0
73 | port: 47907
74 | staticdir: /var/lib/gobeansproxy
75 | threads: 8
76 | webport: 47910
77 | zkpath: /gobeansproxy/test
78 | zkservers: []
79 |
--------------------------------------------------------------------------------
/.doubanpde/scripts/bdb/rivenbeansproxy/conf/proxy.yaml:
--------------------------------------------------------------------------------
1 | cassandra:
2 | default_key_space: doubandb
3 | default_table: kvstore
4 | enable: true
5 | hosts:
6 | - 127.0.0.1:9042
7 | timeout_ms: 1000
8 | connect_timeout_ms: 3000
9 | write_timeout_ms: 1000
10 | retry_num: 3
11 | reconnect_interval_sec: 180
12 | max_conn_for_getm: 10
13 | num_conns: 10
14 | username: "doubandb_test"
15 | password: "doubandb_test"
16 | consistency: "local_one"
17 | prefix_table_dispatcher_cfg:
18 | # if not enable will use default keyspace and table
19 | enable: true
20 | static:
21 | # dispatch prefix1 key to table table_name1
22 | kvstore_ark:
23 | - "/ark"
24 | cfg_table: bdb_prefix_table_finder
25 | cfg_keyspace: doubandb
26 | prefix_rw_dispatcher_cfg:
27 | enable: false
28 | static:
29 | # dispatch prefix /test_prefix_c/ to dual write
30 | br1w1cr0w1:
31 | - "/test_prefix_c/"
32 | - "/test_prefix_d/"
33 | - "/arkark/"
34 | br0w0cr1w1:
35 | - "test_"
36 | cfg_table: bdb_prefix_rw_switcher
37 | cfg_keyspace: doubandb
38 | default_storage: "br0w0cr1w0"
39 | dual_write_err_cfg:
40 | dump_to_dir: /tmp/gobeansproxy_prefix/proxy/
41 | log_file_name: dual_write_err.log
42 | logger_level: "INFO"
43 | rotate_size_mb: 100
44 | compress: true
45 | max_ages: 7
46 | max_backups: 100
47 | dstore:
48 | enable: false
49 | connect_timeout_ms: 300
50 | dial_fail_silence_ms: 5000
51 | error_seconds: 10
52 | item_size_stats: 4096
53 | max_connect_errors: 3
54 | max_free_conns_per_host: 20
55 | n: 3
56 | r: 1
57 | read_timeout_ms: 2000
58 | response_time_min: 4000
59 | response_time_seconds: 10
60 | score_deviation: 10
61 | w: 2
62 | write_timeout_ms: 2000
63 | mc:
64 | body_big_str: 5M
65 | body_c_str: 0K
66 | body_max_str: 50M
67 | max_key_len: 250
68 | max_req: 16
69 | proxy:
70 | accesslog: /tmp/gobeansproxy_prefix/proxy/access.log
71 | errorlog: /tmp/gobeansproxy_prefix/proxy/error.log
72 | hostname: 127.0.0.1
73 | listen: 0.0.0.0
74 | port: 47907
75 | staticdir: /var/lib/gobeansproxy
76 | threads: 8
77 | webport: 47910
78 | zkpath: /gobeansproxy/test
79 | zkservers:
80 | - zk1:2181
81 |
--------------------------------------------------------------------------------
/.doubanpde/scripts/bdb/gobeansproxy/prefix-switch-cfg/conf/proxy.yaml:
--------------------------------------------------------------------------------
1 | cassandra:
2 | default_key_space: doubandb
3 | default_table: kvstore
4 | enable: true
5 | hosts:
6 | - 127.0.0.1:9042
7 | timeout_ms: 1000
8 | connect_timeout_ms: 3000
9 | write_timeout_ms: 1000
10 | retry_num: 3
11 | reconnect_interval_sec: 180
12 | max_conn_for_getm: 10
13 | num_conns: 10
14 | username: "doubandb_test"
15 | password: "doubandb_test"
16 | consistency: "local_one"
17 | prefix_table_dispatcher_cfg:
18 | # if not enable will use default keyspace and table
19 | enable: true
20 | static:
21 | # dispatch prefix1 key to table table_name1
22 | kvstore_ark:
23 | - "/ark"
24 | cfg_table: bdb_prefix_table_finder
25 | cfg_keyspace: doubandb
26 | prefix_rw_dispatcher_cfg:
27 | enable: true
28 | static:
29 | # dispatch prefix /test_prefix_c/ to dual write
30 | br1w1cr0w1:
31 | - "/test_prefix_c/"
32 | - "/test_prefix_d/"
33 | - "/arkark/"
34 | br0w0cr1w1:
35 | - "test_"
36 | cfg_table: bdb_prefix_rw_switcher
37 | cfg_keyspace: doubandb
38 | default_storage: "br0w1cr1w1"
39 | dual_write_err_cfg:
40 | dump_to_dir: /tmp/gobeansproxy_prefix/proxy/
41 | log_file_name: dual_write_err.log
42 | logger_level: "INFO"
43 | rotate_size_mb: 100
44 | compress: true
45 | max_ages: 7
46 | max_backups: 100
47 | dstore:
48 | enable: true
49 | connect_timeout_ms: 300
50 | dial_fail_silence_ms: 5000
51 | error_seconds: 10
52 | item_size_stats: 4096
53 | max_connect_errors: 3
54 | max_free_conns_per_host: 20
55 | n: 3
56 | r: 1
57 | read_timeout_ms: 2000
58 | response_time_min: 4000
59 | response_time_seconds: 10
60 | score_deviation: 10
61 | w: 2
62 | write_timeout_ms: 2000
63 | mc:
64 | body_big_str: 5M
65 | body_c_str: 0K
66 | body_max_str: 50M
67 | max_key_len: 250
68 | max_req: 16
69 | proxy:
70 | accesslog: /tmp/gobeansproxy_prefix/proxy/access.log
71 | errorlog: /tmp/gobeansproxy_prefix/proxy/error.log
72 | hostname: 127.0.0.1
73 | listen: 0.0.0.0
74 | port: 47907
75 | staticdir: /var/lib/gobeansproxy
76 | threads: 8
77 | webport: 47910
78 | zkpath: /gobeansproxy/test
79 | zkservers:
80 | - zk1:2181
81 |
--------------------------------------------------------------------------------
/dstore/metrics.go:
--------------------------------------------------------------------------------
1 | package dstore
2 |
3 | import (
4 | "github.com/prometheus/client_golang/prometheus"
5 | )
6 |
7 | var (
8 | totalReqs *prometheus.CounterVec
9 | errorReqs *prometheus.CounterVec
10 | rrrStoreReqs *prometheus.CounterVec
11 | rrrStoreErr *prometheus.CounterVec
12 | rrrStoreLag *prometheus.GaugeVec
13 | cmdReqDurationSeconds *prometheus.HistogramVec
14 | cmdE2EDurationSeconds *prometheus.HistogramVec
15 | BdbProxyPromRegistry *prometheus.Registry
16 | )
17 |
18 | func init() {
19 | BdbProxyPromRegistry = prometheus.NewRegistry()
20 | totalReqs = prometheus.NewCounterVec(
21 | prometheus.CounterOpts{
22 | Namespace: "gobeansproxy",
23 | Name: "total_reqs",
24 | Help: "total requests counter",
25 | },
26 |
27 | []string{"cmd", "store"},
28 | )
29 | BdbProxyPromRegistry.MustRegister(totalReqs)
30 |
31 | errorReqs = prometheus.NewCounterVec(
32 | prometheus.CounterOpts{
33 | Namespace: "gobeansproxy",
34 | Name: "error_reqs",
35 | Help: "error requests counter",
36 | },
37 |
38 | []string{"cmd", "store"},
39 | )
40 | BdbProxyPromRegistry.MustRegister(errorReqs)
41 |
42 | cmdE2EDurationSeconds = prometheus.NewHistogramVec(
43 | prometheus.HistogramOpts{
44 | Namespace: "gobeansproxy",
45 | Name: "cmd_e2e_duration_seconds",
46 | Help: "cmd e2e duration",
47 | Buckets: []float64{
48 | 0.001, 0.003, 0.005,
49 | 0.01, 0.03, 0.05, 0.07,
50 | 0.1, 0.3, 0.5, 0.7,
51 | 1, 2, 5,
52 | },
53 | },
54 |
55 | []string{"cmd"},
56 | )
57 | BdbProxyPromRegistry.MustRegister(cmdE2EDurationSeconds)
58 |
59 | rrrStoreReqs = prometheus.NewCounterVec(
60 | prometheus.CounterOpts{
61 | Namespace: "gobeansproxy",
62 | Name: "rrr_store_reqs",
63 | Help: "read only rr backends req counter",
64 | },
65 | []string{"host"},
66 | )
67 | BdbProxyPromRegistry.MustRegister(rrrStoreReqs)
68 |
69 | rrrStoreErr = prometheus.NewCounterVec(
70 | prometheus.CounterOpts{
71 | Namespace: "gobeansproxy",
72 | Name: "rrr_store_conn_err",
73 | Help: "store connection error counter",
74 | },
75 | []string{"host", "conn"},
76 | )
77 | BdbProxyPromRegistry.MustRegister(rrrStoreErr)
78 |
79 | rrrStoreLag = prometheus.NewGaugeVec(
80 | prometheus.GaugeOpts{
81 | Namespace: "gobeansproxy",
82 | Name: "rrr_store_lag_ms",
83 | Help: "round robin read only sch store lag",
84 | },
85 | []string{"host"},
86 | )
87 | BdbProxyPromRegistry.MustRegister(rrrStoreLag)
88 | }
89 |
--------------------------------------------------------------------------------
/tests/abnormal_cmd_test.py:
--------------------------------------------------------------------------------
1 | # coding: utf-8
2 | import telnetlib
3 | from tests.base import BaseTest
4 |
5 |
6 | class AbnormalCmdTest(BaseTest):
7 | def setUp(self):
8 | BaseTest.setUp(self)
9 | self.invalid_key = '/this/is/a/bad/key/%s' % chr(15)
10 |
11 | def run_cmd_by_telnet(self, cmd, expected, timeout=2):
12 | addr, port = self.proxy.addr.split(':')
13 | t = telnetlib.Telnet(addr, port)
14 | t.write(f'{cmd}\r\n'.encode())
15 | out = t.read_until(b'\n', timeout=timeout)
16 | t.write(b'quit\n')
17 | t.close()
18 | r = out.strip(b'\r\n')
19 | self.assertEqual(r, expected.encode())
20 |
21 | def test_get(self):
22 | # get not exist key
23 | cmd = 'get /test/get'
24 | self.run_cmd_by_telnet(cmd, 'END')
25 |
26 | # invalid key
27 | cmd = 'get %s' % self.invalid_key
28 | self.run_cmd_by_telnet(cmd, 'END')
29 | self.checkCounterZero()
30 |
31 | def test_set(self):
32 | # invalid key
33 | cmd = 'set %s 0 0 3\r\naaa' % self.invalid_key
34 | if self.bdb_write_enable:
35 | self.run_cmd_by_telnet(cmd, 'SERVER_ERROR write failed')
36 | elif self.cstar_write_enable:
37 | self.run_cmd_by_telnet(cmd, 'STORED')
38 |
39 | cmd = 'set /test/set 0 0 3\r\naaaa'
40 | self.run_cmd_by_telnet(cmd, 'CLIENT_ERROR bad data chunk')
41 |
42 | self.checkCounterZero()
43 |
44 | def test_delete(self):
45 | key = '/delete/not/exist/key'
46 | cmd = 'delete %s' % key
47 | if self.bdb_write_enable and not self.cstar_write_enable:
48 | self.run_cmd_by_telnet(cmd, 'NOT FOUND')
49 |
50 | if self.cstar_write_enable:
51 | # cstar will delete a key event if not exists
52 | # it just write a tombestone to mem
53 | self.run_cmd_by_telnet(cmd, 'DELETED')
54 |
55 | cmd = 'delete %s' % self.invalid_key
56 | if self.bdb_write_enable and not self.cstar_write_enable:
57 | self.run_cmd_by_telnet(cmd, 'NOT FOUND')
58 |
59 | if self.cstar_write_enable:
60 | # cstar will delete a key event if not exists
61 | # it just write a tombestone to mem
62 | self.run_cmd_by_telnet(cmd, 'DELETED')
63 | self.checkCounterZero()
64 |
65 | def test_get_meta_by_key(self):
66 | key = '/get_meta_by_key/not/exist/key'
67 | cmd = 'get ?%s' % key
68 | self.run_cmd_by_telnet(cmd, 'END')
69 |
70 | cmd = 'get ?%s' % self.invalid_key
71 | self.run_cmd_by_telnet(cmd, 'END')
72 | self.checkCounterZero()
73 |
--------------------------------------------------------------------------------
/conf/proxy.yaml:
--------------------------------------------------------------------------------
1 | # for doubandb proxy
2 | proxy:
3 | listen: 0.0.0.0
4 | port: 7905
5 | webport: 7908
6 | threads: 8
7 | errorlog: "./proxy-error.log"
8 | accesslog: "./proxy-access.log"
9 | hostname: 127.0.0.1
10 | staticdir: /var/lib/gobeansproxy
11 | zkserves: []
12 | zkpath: "/gobeansproxy/test"
13 | mc:
14 | max_key_len: 250
15 | max_req: 16
16 | body_max_str: 50M
17 | body_big_str: 5M
18 | body_c_str: 0K
19 | dstore:
20 | n: 3
21 | w: 2
22 | r: 1
23 | max_free_conns_per_host: 20
24 | connect_timeout_ms: 300
25 | write_timeout_ms: 2000
26 | read_timeout_ms: 2000
27 | dial_fail_silence_ms: 5000
28 | response_time_seconds: 10
29 | error_seconds: 10
30 | max_connect_errors: 10
31 | score_deviation: 10000
32 | item_size_stats: 4096
33 | response_time_min: 4000
34 | enable: true
35 | cassandra:
36 | enable: true
37 | default_key_space: dbname
38 | default_table: tablename
39 | hosts:
40 | - cassandra:9042
41 | timeout_ms: 1000
42 | connect_timeout_ms: 3000
43 | write_timeout_ms: 1000
44 | retry_num: 3
45 | reconnect_interval_sec: 180
46 | max_conn_for_getm: 10
47 | num_conns: 10
48 | username: ""
49 | # plaintext password only for test usage
50 | # please use password_file in prod env
51 | password: ""
52 | password_file: ""
53 | # local_one only for test usage
54 | # default: quorum
55 | # consistency: "local_one"
56 | prefix_table_dispatcher_cfg:
57 | # if not enable will use default keyspace and table
58 | enable: false
59 | static:
60 | # dispatch prefix1 key to table table_name1
61 | table_name1:
62 | - "prefix1"
63 | cfg_table: cassandra_cfg_table_name
64 | cfg_keyspace: cassandra_cfg_keyspace
65 | prefix_rw_dispatcher_cfg:
66 | enable: true
67 | static:
68 | # dispatch prefix /test_prefix_c/ to dual write
69 | br1w1cr0w1:
70 | - "/test_prefix_c/"
71 | - "/test_prefix_d/"
72 | br0w0cr1w1:
73 | - "test_"
74 | cfg_table: cassandra_cfg_table_name
75 | cfg_keyspace: cassandra_cfg_keyspace
76 | # if not match rw dispatcher config
77 | # will fallback to this default storage rw cfg
78 | # br1w1cr0w0: only use beansdb as backend
79 | # br1w1cr0w1: dual write and read from beansdb
80 | # br0w1cr1w1: dual write and read from c*
81 | # br0w0cr1w1: only use c* for rw backend
82 | default_storage: "br1w1cr0w0"
83 | # dual write error log config
84 | dual_write_err_cfg:
85 | dump_to_dir: /var/gobeansproxy/log/
86 | log_file_name: dual_write_err.log
87 | logger_level: "INFO"
88 | rotate_size_mb: 100
89 | compress: true
90 | max_ages: 7
91 | max_backups: 100
92 |
--------------------------------------------------------------------------------
/tests/dbclient.py:
--------------------------------------------------------------------------------
1 | # coding: utf-8
2 | import libmc
3 |
4 |
5 | def connect(server, **kwargs):
6 | comp_threshold = kwargs.pop('comp_threshold', 0)
7 | prefix = kwargs.pop('prefix', None)
8 |
9 | c = libmc.Client([server],
10 | do_split=0,
11 | comp_threshold=comp_threshold,
12 | prefix=prefix)
13 | c.config(libmc.MC_CONNECT_TIMEOUT, 300) # 0.3s
14 | c.config(libmc.MC_POLL_TIMEOUT, 10000) # 3s
15 | c.config(libmc.MC_RETRY_TIMEOUT, 5) # 5s
16 | return c
17 |
18 |
19 | class MCStore(object):
20 |
21 | IGNORED_LIBMC_RET = frozenset([
22 | libmc.MC_RETURN_OK,
23 | libmc.MC_RETURN_INVALID_KEY_ERR
24 | ])
25 |
26 | def __init__(self, addr, **kwargs):
27 | self.addr = addr
28 | self.mc = connect(addr, **kwargs)
29 |
30 | def __repr__(self):
31 | return '