├── .gitignore ├── README.md ├── directory ├── conf │ └── config.go ├── directory.go ├── directory.toml ├── directory_test.go ├── dispatcher.go ├── dispatcher_test.go ├── hbase │ ├── file.go │ ├── file_test.go │ ├── hbase_client.go │ ├── hbase_client_test.go │ ├── needle.go │ └── needle_test.go ├── http.go ├── http_api.go ├── http_api_test.go ├── http_perf.go ├── main.go ├── signal.go ├── snowflake │ ├── generate_key.go │ ├── generate_key_test.go │ └── gosnowflake.go └── zk │ ├── zk.go │ └── zk_test.go ├── doc ├── bfs.graffle ├── directory.md ├── pitchfork.md └── store.md ├── libs ├── encoding │ └── binary │ │ └── endian.go ├── errors │ ├── common.go │ ├── directory.go │ ├── errors.go │ ├── proxy.go │ └── store.go ├── gohbase │ ├── .travis.yml │ ├── AUTHORS │ ├── COPYING │ ├── Makefile │ ├── README.md │ ├── b_test.go │ ├── c_test.go │ ├── check_line_len.awk │ ├── client.go │ ├── conf │ │ └── conf.go │ ├── discovery_test.go │ ├── filter │ │ ├── comparator.go │ │ └── filter.go │ ├── hbase │ │ └── hbase.go │ ├── hrpc │ │ ├── call.go │ │ ├── create.go │ │ ├── delete.go │ │ ├── disable.go │ │ ├── enable.go │ │ ├── get.go │ │ ├── hrpc_test.go │ │ ├── mutate.go │ │ ├── scan.go │ │ └── tableop.go │ ├── integration_test.go │ ├── metacache_test.go │ ├── pb │ │ ├── Cell.pb.go │ │ ├── Cell.proto │ │ ├── Client.pb.go │ │ ├── Client.proto │ │ ├── ClusterId.pb.go │ │ ├── ClusterId.proto │ │ ├── ClusterStatus.pb.go │ │ ├── ClusterStatus.proto │ │ ├── Comparator.pb.go │ │ ├── Comparator.proto │ │ ├── ErrorHandling.pb.go │ │ ├── ErrorHandling.proto │ │ ├── FS.pb.go │ │ ├── FS.proto │ │ ├── Filter.pb.go │ │ ├── Filter.proto │ │ ├── HBase.pb.go │ │ ├── HBase.proto │ │ ├── Master.pb.go │ │ ├── Master.proto │ │ ├── MultiRowMutation.pb.go │ │ ├── MultiRowMutation.proto │ │ ├── Quota.pb.go │ │ ├── Quota.proto │ │ ├── README.txt │ │ ├── RPC.pb.go │ │ ├── RPC.proto │ │ ├── Tracing.pb.go │ │ ├── Tracing.proto │ │ ├── ZooKeeper.pb.go │ │ ├── ZooKeeper.proto │ │ ├── generate.go │ │ └── marshal.go │ ├── region │ │ ├── client.go │ │ └── client_test.go │ ├── regioninfo │ │ ├── info.go │ │ └── info_test.go │ ├── test │ │ └── test.go │ ├── tools │ │ ├── test_get │ │ │ └── main.go │ │ └── test_get_dapper │ │ │ └── main.go │ └── zk │ │ └── client.go ├── memcache │ ├── gomemcache │ │ └── memcache │ │ │ ├── conn.go │ │ │ ├── errors.go │ │ │ ├── memcache.go │ │ │ └── pool.go │ └── memcache.go ├── meta │ ├── const.go │ ├── directory.go │ ├── file.go │ ├── needle.go │ ├── store.go │ ├── super_block.go │ └── volume.go ├── stat │ ├── stat.go │ └── stat_test.go ├── time │ └── time.go └── uuid │ ├── uuid.go │ └── uuid_test.go ├── ops ├── README.md ├── commons │ ├── __init__.py │ ├── __init__.pyc │ ├── blogging.py │ ├── blogging.pyc │ ├── config.py │ ├── config.pyc │ ├── global_var.py │ ├── global_var.pyc │ ├── store_client.py │ ├── store_client.pyc │ ├── zk_client.py │ └── zk_client.pyc ├── runserver.py ├── setup.py ├── test │ └── ops_initialization.py └── views │ ├── __init__.py │ ├── __init__.pyc │ ├── ops.py │ └── ops.pyc ├── pitchfork ├── conf │ └── config.go ├── main.go ├── pitchfork.go ├── pitchfork.toml ├── pitchfork_test.go ├── signal.go └── zk │ ├── zk.go │ └── zk_test.go ├── proxy ├── auth │ └── auth.go ├── bfs │ ├── bfs.go │ └── bfs_test.go ├── bucket │ └── bucket.go ├── cache │ └── cache.go ├── conf │ └── config.go ├── http_api.go ├── http_api_test.go ├── http_perf.go ├── main.go ├── proxy.toml └── service.go └── store ├── block ├── supper_block.go └── supper_block_test.go ├── conf ├── config.go ├── config.go.bak └── config_test.go ├── http.go ├── http_admin.go ├── http_api.go ├── http_perf.go ├── http_stat.go ├── index ├── index.go ├── index_test.go ├── ring.go └── ring_test.go ├── main.go ├── needle ├── cache.go ├── cache_test.go ├── needle.go ├── needle_test.go ├── needles.go ├── needles_test.go ├── test_test.go └── utils.go ├── os ├── fadvise_darwin.go ├── fadvise_linux.go ├── falloc_darwin.go ├── falloc_linux.go ├── fdatasync_darwin.go ├── fdatasync_linux.go ├── file.go ├── io_darwin.go ├── io_linux.go ├── sync_file_range_darwin.go └── sync_file_range_linux.go ├── signal.go ├── store.go ├── store.toml ├── store_test.go ├── test ├── 1.jpg ├── 10.jpg ├── 2.jpg ├── 3.jpg ├── 4.jpg ├── 5.jpg ├── 6.jpg ├── 7.jpg ├── 8.jpg ├── 9.jpg └── test.sh ├── test_test.go ├── ver.go ├── volume ├── volume.go └── volume_test.go └── zk ├── zk.go └── zk_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | *.gitattributes 26 | 27 | directory/directory 28 | pitchfork/pitchfork 29 | proxy/proxy 30 | store/store 31 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | bfs 2 | ============== 3 | `bfs` 是基于facebook haystack 用golang实现的小文件存储系统。 4 | 5 | --------------------------------------- 6 | * [特性](#特性) 7 | * [安装](#安装) 8 | * [集群](#集群) 9 | * [API](#API) 10 | * [更多](#更多) 11 | 12 | --------------------------------------- 13 | 14 | ## 特性 15 | * 高吞吐量和低延迟 16 | * 容错性 17 | * 高效 18 | * 维护简单 19 | 20 | ## 安装 21 | 22 | ### 一、安装hbase、zookeeper 23 | 24 | * 参考hbase官网. 安装、启动请查看[这里](https://hbase.apache.org/). 25 | * 参考zookeeper官网. 安装、启动请查看[这里](http://zookeeper.apache.org/). 26 | 27 | ### 二、搭建golang、python环境 28 | 29 | * 参考golang官网. 安装请查看[这里](https://golang.org/doc/install). 30 | * 参考python官网. 安装请查看[这里] 31 | (https://www.python.org/) 32 | 33 | ### 三、安装gosnowflake 34 | 35 | * 参考[这里](https://github.com/Terry-Mao/gosnowflake) 36 | 37 | ### 四、部署 38 | 1.下载bfs及依赖包 39 | ```sh 40 | $ go get -u github.com/Terry-Mao/bfs 41 | $ cd /data/apps/go/src/github.com/Terry-Mao/bfs 42 | $ go get ./... 43 | ``` 44 | 45 | 2.安装directory、store、pitchfork、proxy模块(配置文件请依据实际机器环境配置) 46 | ```sh 47 | $ cd $GOPATH/src/github.com/Terry-Mao/bfs/directory 48 | $ go install 49 | $ cp directory.toml $GOPATH/bin/directory.toml 50 | $ cd ../store/ 51 | $ go install 52 | $ cp store.toml $GOPATH/bin/store.toml 53 | $ cd ../pitchfork/ 54 | $ go install 55 | $ cp pitchfork.toml $GOPATH/bin/pitchfork.toml 56 | $ cd ../proxy 57 | $ go install 58 | $ cp proxy.toml $GOPATH/bin/proxy.toml 59 | 60 | ``` 61 | 到此所有的环境都搭建完成！ 62 | 63 | ### 五、启动 64 | ```sh 65 | $ cd /$GOPATH/bin 66 | $ nohup $GOPATH/bin/directory -c $GOPATH/bin/directory.toml & 67 | $ nohup $GOPATH/bin/store -c $GOPATH/bin/store.toml & 68 | $ nohup $GOPATH/bin/pitchfork -c $GOPATH/bin/pitchfork.toml & 69 | $ nohup $GOPATH/bin/proxy -c $GOPATH/bin/proxy.toml & 70 | $ cd $GOPATH/github.com/Terry-Mao/bfs/ops 71 | $ nohup python runserver.py & 72 | ``` 73 | 74 | ### 六、测试 75 | * bfs初始化，分配存储空间，请查看[这里](https://github.com/Terry-Mao/bfs/doc/ops.md) 76 | * 请求bfs，请查看[这里](https://github.com/Terry-Mao/bfs/doc/proxy.md) 77 | 78 | ## 集群 79 | 80 | ![Aaron Swartz](http://i0.hdslb.com/bfs/active/bfs_server.png) 81 | 82 | ### directory 83 | 84 | * directory主要负责请求的均匀调度和元数据管理，元数据存放在hbase，由gosnowflake产生文件key 85 | 86 | ### store 87 | 88 | * store主要负责文件的物理存储 89 | 90 | ### pitchfork 91 | 92 | * pitchfork负责监控store的服务状态、可用性和磁盘状态 93 | 94 | ### proxy 95 | 96 | * proxy作为bfs存储的代理以及维护bucket相关 97 | 98 | ### ops 99 | 100 | * ops作为bfs的后台管理界面，负责分配存储、扩容、压缩等维护工作 101 | 102 | ## API 103 | [api文档](https://github.com/Terry-Mao/bfs/blob/master/doc/api.md) 104 | ## 更多 -------------------------------------------------------------------------------- /directory/conf/config.go: -------------------------------------------------------------------------------- 1 | package conf 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "time" 7 | 8 | xtime "bfs/libs/time" 9 | 10 | "github.com/BurntSushi/toml" 11 | ) 12 | 13 | type Config struct { 14 | Snowflake *Snowflake 15 | Zookeeper *Zookeeper 16 | HBase *HBase 17 | 18 | MaxNum int 19 | ApiListen string 20 | PprofEnable bool 21 | PprofListen string 22 | } 23 | 24 | type Snowflake struct { 25 | ZkAddrs []string 26 | ZkTimeout duration 27 | ZkPath string 28 | WorkId int64 29 | } 30 | 31 | type Zookeeper struct { 32 | Addrs []string 33 | Timeout duration 34 | PullInterval duration 35 | VolumeRoot string 36 | StoreRoot string 37 | GroupRoot string 38 | } 39 | 40 | // HBase config. 41 | type HBase struct { 42 | ZookeeperHbase *ZookeeperHbase 43 | // default "" means use default hbase zk path. It should correspond to server config 44 | Master string 45 | Meta string 46 | TestRowKey string 47 | DialTimeout xtime.Duration // 0 means no dial timeout 48 | ReadTimeout xtime.Duration 49 | ReadsTimeout xtime.Duration 50 | WriteTimeout xtime.Duration 51 | WritesTimeout xtime.Duration 52 | } 53 | 54 | type ZookeeperHbase struct { 55 | Root string 56 | Addrs []string 57 | Timeout xtime.Duration 58 | } 59 | 60 | // Code to implement the TextUnmarshaler interface for `duration`: 61 | type duration struct { 62 | time.Duration 63 | } 64 | 65 | func (d *duration) UnmarshalText(text []byte) error { 66 | var err error 67 | d.Duration, err = time.ParseDuration(string(text)) 68 | return err 69 | } 70 | 71 | // NewConfig new a config. 72 | func NewConfig(conf string) (c *Config, err error) { 73 | var ( 74 | file *os.File 75 | blob []byte 76 | ) 77 | c = new(Config) 78 | if file, err = os.Open(conf); err != nil { 79 | return 80 | } 81 | if blob, err = ioutil.ReadAll(file); err != nil { 82 | return 83 | } 84 | err = toml.Unmarshal(blob, c) 85 | return 86 | } 87 | -------------------------------------------------------------------------------- /directory/directory.toml: -------------------------------------------------------------------------------- 1 | # listen 2 | ApiListen = "localhost:6065" 3 | 4 | #batchUpload max num of keys once upload 5 | MaxNum = 16 6 | 7 | # enable golang pprof 8 | PprofEnable = true 9 | 10 | # pprof http addr 11 | PprofListen = "localhost:6066" 12 | 13 | [snowflake] 14 | # zookeeper cluster addrs, multiple addrs split by ",". 15 | ZkAddrs = [ 16 | "localhost:2181" 17 | ] 18 | 19 | # path 20 | ZkPath = "/gosnowflake-servers" 21 | 22 | # timeout 23 | ZkTimeout = "15s" 24 | 25 | # workid 26 | WorkId = 1 27 | 28 | [zookeeper] 29 | # zookeeper cluster addrs, multiple addrs split by ",". 30 | Addrs = [ 31 | "localhost:2181" 32 | ] 33 | 34 | # zookeeper heartbeat timeout. 35 | Timeout = "15s" 36 | 37 | # zookeeper storeroot path. 38 | StoreRoot = "/rack" 39 | 40 | # zookeeper volumeroot path 41 | VolumeRoot = "/volume" 42 | 43 | # zookeeper grouproot path 44 | GroupRoot = "/group" 45 | 46 | # zookeeper pullinterval 47 | PullInterval = "10s" 48 | 49 | [hbase] 50 | master = "" 51 | meta = "" 52 | dialTimeout = "1s" 53 | readTimeout = "10s" 54 | readsTimeout = "10s" 55 | writeTimeout = "10s" 56 | writesTimeout = "10s" 57 | [hbase.zookeeperHbase] 58 | root = "" 59 | addrs = ["localhost:2181"] 60 | timeout = "30s" -------------------------------------------------------------------------------- /directory/directory_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "testing" 5 | // "time" 6 | 7 | "bfs/directory/conf" 8 | dzk "bfs/directory/zk" 9 | ) 10 | 11 | func TestDirectory(t *testing.T) { 12 | var ( 13 | err error 14 | config *conf.Config 15 | zk *dzk.Zookeeper 16 | d *Directory 17 | ) 18 | if config, err = conf.NewConfig("./directory.toml"); err != nil { 19 | t.Errorf("NewConfig() error(%v)", err) 20 | t.FailNow() 21 | } 22 | 23 | if zk, err = dzk.NewZookeeper(config); err != nil { 24 | t.Errorf("NewZookeeper() error(%v)", err) 25 | t.FailNow() 26 | } 27 | defer zk.Close() 28 | if d, err = NewDirectory(config); err != nil { 29 | t.Errorf("NewDirectory() error(%v)", err) 30 | t.FailNow() 31 | } 32 | if _, err = d.syncStores(); err != nil { 33 | t.Errorf("syncStores() error(%v)", err) 34 | t.FailNow() 35 | } 36 | if err = d.syncGroups(); err != nil { 37 | t.Errorf("syncGroups() error(%v)", err) 38 | t.FailNow() 39 | } 40 | if err = d.syncVolumes(); err != nil { 41 | t.Errorf("syncVolumes() error(%v)", err) 42 | t.FailNow() 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /directory/dispatcher_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bfs/directory/conf" 5 | dzk "bfs/directory/zk" 6 | "testing" 7 | "time" 8 | // "fmt" 9 | ) 10 | 11 | func TestDispatcher(t *testing.T) { 12 | var ( 13 | err error 14 | config *conf.Config 15 | zk *dzk.Zookeeper 16 | d *Directory 17 | ds *Dispatcher 18 | vid int32 19 | ) 20 | if config, err = conf.NewConfig("./directory.toml"); err != nil { 21 | t.Errorf("NewConfig() error(%v)", err) 22 | return 23 | } 24 | 25 | if zk, err = dzk.NewZookeeper(config); err != nil { 26 | t.Errorf("NewZookeeper() error(%v)", err) 27 | t.FailNow() 28 | } 29 | defer zk.Close() 30 | if d, err = NewDirectory(config); err != nil { 31 | t.Errorf("NewDirectory() error(%v)", err) 32 | t.FailNow() 33 | } 34 | time.Sleep(5 * time.Second) 35 | ds = NewDispatcher() 36 | if err = ds.Update(d.group, d.store, d.volume, d.storeVolume); err != nil { 37 | t.Errorf("Update() error(%v)", err) 38 | t.FailNow() 39 | } 40 | if vid, err = ds.VolumeID(d.group, d.storeVolume); err != nil { 41 | t.Errorf("Update() error(%v)", err) 42 | t.FailNow() 43 | } 44 | t.Logf("vid:%v", vid) 45 | } 46 | -------------------------------------------------------------------------------- /directory/hbase/file_test.go: -------------------------------------------------------------------------------- 1 | package hbase 2 | 3 | import ( 4 | "bfs/libs/errors" 5 | "bfs/libs/meta" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | func TestGetFile(t *testing.T) { 11 | c := getClient() 12 | f, err := c.getFile("test", "guhaotest111.jpg") 13 | if err != nil && err != errors.ErrNeedleNotExist { 14 | t.Fatalf("err:%v", err.Error()) 15 | } 16 | t.Logf("f: %v", f) 17 | } 18 | 19 | func TestPutFile(t *testing.T) { 20 | c := getClient() 21 | err := c.putFile("test", &meta.File{ 22 | Filename: "guhaotest111.jpg", 23 | Key: 1234567, 24 | Sha1: "12312312312312312", 25 | Mine: "image/jpg", 26 | Status: 123, 27 | MTime: time.Now().Unix(), 28 | }) 29 | if err != nil { 30 | t.Fatalf("err:%v", err.Error()) 31 | } 32 | } 33 | 34 | func TestDelFile(t *testing.T) { 35 | c := getClient() 36 | if err := c.delFile("test", "guhaotest111.jpg"); err != nil { 37 | t.Fatalf("err:%v", err.Error()) 38 | } 39 | } 40 | 41 | func TestExistFile(t *testing.T) { 42 | c := getClient() 43 | exist, err := c.existFile("test", "guhaotest111.jpg") 44 | if err != errors.ErrNeedleExist { 45 | t.Fatalf("err:%v", err.Error()) 46 | } 47 | t.Logf("pass:err:%v", exist) 48 | } 49 | -------------------------------------------------------------------------------- /directory/hbase/hbase_client.go: -------------------------------------------------------------------------------- 1 | package hbase 2 | 3 | import ( 4 | "strings" 5 | "time" 6 | 7 | "bfs/directory/conf" 8 | "bfs/libs/errors" 9 | "bfs/libs/gohbase" 10 | hconf "bfs/libs/gohbase/conf" 11 | "bfs/libs/gohbase/hbase" 12 | "bfs/libs/meta" 13 | 14 | log "github.com/golang/glog" 15 | ) 16 | 17 | const ( 18 | _family = "hbase_client" 19 | _prefix = "bucket_" 20 | ) 21 | 22 | // Client hbase client. 23 | type Client struct { 24 | c gohbase.Client 25 | testCell *hbase.HBaseCell 26 | addr string 27 | } 28 | 29 | // NewClient new a hbase client. 30 | func NewClient(c *conf.HBase, options ...gohbase.Option) *Client { 31 | var testRowKey string 32 | if c.TestRowKey != "" { 33 | testRowKey = c.TestRowKey 34 | } else { 35 | testRowKey = "test" 36 | } 37 | return &Client{ 38 | c: gohbase.NewClient(hconf.NewConf( 39 | c.ZookeeperHbase.Addrs, 40 | c.ZookeeperHbase.Root, 41 | c.Master, 42 | c.Meta, 43 | time.Duration(c.ZookeeperHbase.Timeout), 44 | 0, 45 | 0, 46 | time.Duration(c.DialTimeout), 47 | ), options...), 48 | testCell: &hbase.HBaseCell{ 49 | "test", 50 | testRowKey, 51 | "test", 52 | "test", 53 | "test", 54 | }, 55 | addr: strings.Join(c.ZookeeperHbase.Addrs, ","), 56 | } 57 | } 58 | 59 | // Put put file and needle into hbase 60 | func (c *Client) Put(bucket string, f *meta.File, n *meta.Needle) (err error) { 61 | if err = c.putFile(bucket, f); err != nil { 62 | return 63 | } 64 | if err = c.putNeedle(n); err != nil && err != errors.ErrNeedleExist { 65 | log.Warningf("table not match: bucket: %s filename: %s", bucket, f.Filename) 66 | c.delFile(bucket, f.Filename) 67 | } 68 | return 69 | } 70 | 71 | // Get get needle from hbase 72 | func (c *Client) Get(bucket, filename string) (n *meta.Needle, f *meta.File, err error) { 73 | if f, err = c.getFile(bucket, filename); err != nil { 74 | return 75 | } 76 | if n, err = c.getNeedle(f.Key); err == errors.ErrNeedleNotExist { 77 | log.Warningf("table not match: bucket: %s filename: %s", bucket, filename) 78 | c.delFile(bucket, filename) 79 | } 80 | return 81 | } 82 | 83 | // Del del file and needle from hbase 84 | func (c *Client) Del(bucket, filename string) (err error) { 85 | var ( 86 | f *meta.File 87 | ) 88 | if f, err = c.getFile(bucket, filename); err != nil { 89 | return 90 | } 91 | if err = c.delFile(bucket, filename); err != nil { 92 | return 93 | } 94 | err = c.delNeedle(f.Key) 95 | return 96 | } 97 | -------------------------------------------------------------------------------- /directory/hbase/hbase_client_test.go: -------------------------------------------------------------------------------- 1 | package hbase 2 | 3 | import ( 4 | "bfs/directory/conf" 5 | "bfs/libs/errors" 6 | "bfs/libs/meta" 7 | xtime "bfs/libs/time" 8 | "fmt" 9 | "testing" 10 | "time" 11 | ) 12 | 13 | func getClient() *Client { 14 | d, err := time.ParseDuration("1s") 15 | if err != nil { 16 | panic(err) 17 | } 18 | return NewClient(&conf.HBase{ 19 | Master: "", 20 | Meta: "", 21 | TestRowKey: "", 22 | DialTimeout: xtime.Duration(d), 23 | ReadTimeout: xtime.Duration(d), 24 | ReadsTimeout: xtime.Duration(d), 25 | WriteTimeout: xtime.Duration(d), 26 | WritesTimeout: xtime.Duration(d), 27 | ZookeeperHbase: &conf.ZookeeperHbase{ 28 | Root: "", 29 | Addrs: []string{"localhost:2181"}, 30 | Timeout: xtime.Duration(d), 31 | }, 32 | }) 33 | } 34 | 35 | func TestGet(t *testing.T) { 36 | c := getClient() 37 | fmt.Println(c.c) 38 | n, f, err := c.Get("test", "guhaotest123.jpg") 39 | if err != nil && err != errors.ErrNeedleNotExist { 40 | t.Fatalf("err:%v", err.Error()) 41 | } 42 | t.Logf("vid:%v,key:%v,cookie:%v,fkey:%v", n.Vid, n.Key, n.Cookie, f.Key) 43 | } 44 | 45 | func TestPut(t *testing.T) { 46 | c := getClient() 47 | mf := &meta.File{ 48 | Filename: "guhaotest111.jpg", 49 | Key: 1234567, 50 | Sha1: "12312312312312312", 51 | Mine: "image/jpg", 52 | Status: 123, 53 | MTime: time.Now().Unix(), 54 | } 55 | mn := &meta.Needle{ 56 | Key: 1234567, 57 | Cookie: 123333, 58 | Vid: 1, 59 | MTime: time.Now().Unix(), 60 | } 61 | if err := c.Put("test", mf, mn); err != nil { 62 | t.Fatalf("err:%v", err.Error()) 63 | } 64 | t.Logf("pass:%v", mf) 65 | } 66 | 67 | func TestDelete(t *testing.T) { 68 | c := getClient() 69 | if err := c.Del("test", "guhaotest.jpg"); err != nil { 70 | t.Fatalf("err:%v", err.Error()) 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /directory/hbase/needle.go: -------------------------------------------------------------------------------- 1 | package hbase 2 | 3 | import ( 4 | "bfs/libs/errors" 5 | "bfs/libs/gohbase/hrpc" 6 | "bfs/libs/meta" 7 | "bytes" 8 | "context" 9 | "crypto/sha1" 10 | "encoding/binary" 11 | 12 | log "github.com/golang/glog" 13 | ) 14 | 15 | var ( 16 | _table = []byte("bfsmeta") 17 | 18 | _familyBasic = "basic" // basic store info column family 19 | _columnVid = "vid" 20 | _columnCookie = "cookie" 21 | _columnUpdateTime = "update_time" 22 | ) 23 | 24 | func (c *Client) delNeedle(key int64) (err error) { 25 | var ( 26 | mutate *hrpc.Mutate 27 | ) 28 | if mutate, err = hrpc.NewDel(context.Background(), _table, c.key(key), nil); err != nil { 29 | log.Errorf("Client.delNeedle.NewDel(%v) error:%v", key, err.Error()) 30 | return 31 | } 32 | if _, err = c.c.Delete(mutate); err != nil { 33 | log.Errorf("Client.delNeedle.Delete(%v) error:%v", key, err.Error()) 34 | } 35 | return 36 | } 37 | 38 | func (c *Client) putNeedle(n *meta.Needle) (err error) { 39 | var ( 40 | mutate *hrpc.Mutate 41 | vbuf = make([]byte, 4) 42 | cbuf = make([]byte, 4) 43 | ubuf = make([]byte, 8) 44 | exist bool 45 | ) 46 | if exist, err = c.existNeedle(n.Key); err != nil { 47 | return 48 | } 49 | if exist { 50 | err = errors.ErrNeedleExist 51 | return 52 | } 53 | binary.BigEndian.PutUint32(vbuf, uint32(n.Vid)) 54 | binary.BigEndian.PutUint32(cbuf, uint32(n.Cookie)) 55 | binary.BigEndian.PutUint64(ubuf, uint64(n.MTime)) 56 | values := map[string]map[string][]byte{ 57 | _familyBasic: map[string][]byte{ 58 | _columnVid: vbuf, 59 | _columnCookie: cbuf, 60 | _columnUpdateTime: ubuf, 61 | }, 62 | } 63 | if mutate, err = hrpc.NewPut(context.Background(), _table, c.key(n.Key), values); err != nil { 64 | log.Errorf("Client.putNeedle.NewPut(%v) error:%v", n.Key, err.Error()) 65 | return 66 | } 67 | if _, err = c.c.Put(mutate); err != nil { 68 | log.Errorf("Client.putNeedle.Put(%v) error:%v", n.Key, err.Error()) 69 | } 70 | return 71 | } 72 | 73 | func (c *Client) existNeedle(key int64) (exist bool, err error) { 74 | var ( 75 | getter *hrpc.Get 76 | result *hrpc.Result 77 | ) 78 | if getter, err = hrpc.NewGet(context.Background(), _table, c.key(key)); err != nil { 79 | log.Errorf("Client.existNeedle.NewGet(%v) error:%v", key, err.Error()) 80 | return 81 | } 82 | result, err = c.c.Get(getter) 83 | if err != nil { 84 | log.Errorf("Client.existNeedle.Get(%v) error:%v", key, err.Error()) 85 | return 86 | } 87 | if result == nil || len(result.Cells) == 0 { 88 | return 89 | } 90 | exist = true 91 | return 92 | } 93 | 94 | func (c *Client) getNeedle(key int64) (n *meta.Needle, err error) { 95 | var ( 96 | getter *hrpc.Get 97 | result *hrpc.Result 98 | ) 99 | if getter, err = hrpc.NewGet(context.Background(), _table, c.key(key)); err != nil { 100 | log.Errorf("Client.getNeedle.NewGet(%v) error:%v", key, err.Error()) 101 | return 102 | } 103 | result, err = c.c.Get(getter) 104 | if err != nil { 105 | log.Errorf("Client.getNeedle.Get(%v) error:%v", key, err.Error()) 106 | return 107 | } 108 | if result == nil || len(result.Cells) == 0 { 109 | err = errors.ErrNeedleNotExist 110 | return 111 | } 112 | n = &meta.Needle{ 113 | Key: key, 114 | } 115 | for _, cell := range result.Cells { 116 | if cell == nil { 117 | continue 118 | } 119 | if bytes.Equal(cell.Family, []byte(_familyBasic)) { 120 | if bytes.Equal(cell.Qualifier, []byte(_columnVid)) { 121 | n.Vid = int32(binary.BigEndian.Uint32(cell.Value)) 122 | } else if bytes.Equal(cell.Qualifier, []byte(_columnCookie)) { 123 | n.Cookie = int32(binary.BigEndian.Uint32(cell.Value)) 124 | } else if bytes.Equal(cell.Qualifier, []byte(_columnUpdateTime)) { 125 | n.MTime = int64(binary.BigEndian.Uint64(cell.Value)) 126 | } 127 | } 128 | } 129 | return 130 | } 131 | 132 | func (c *Client) key(key int64) []byte { 133 | var ( 134 | sb [sha1.Size]byte 135 | b []byte 136 | ) 137 | b = make([]byte, 8) 138 | binary.BigEndian.PutUint64(b, uint64(key)) 139 | sb = sha1.Sum(b) 140 | return sb[:] 141 | } 142 | -------------------------------------------------------------------------------- /directory/hbase/needle_test.go: -------------------------------------------------------------------------------- 1 | package hbase 2 | 3 | import ( 4 | "bfs/libs/meta" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func TestPutNeedle(t *testing.T) { 10 | c := getClient() 11 | if err := c.putNeedle(&meta.Needle{ 12 | Key: 1234567, 13 | Cookie: 1111111, 14 | Vid: 1, 15 | MTime: time.Now().Unix(), 16 | }); err != nil { 17 | t.Fatalf("err:%v", err.Error()) 18 | } 19 | 20 | } 21 | 22 | func TestGetNeedle(t *testing.T) { 23 | c := getClient() 24 | mn, err := c.getNeedle(1234567) 25 | if err != nil { 26 | t.Fatalf("err:%v", err.Error()) 27 | } 28 | t.Logf("mn:%v", mn) 29 | } 30 | 31 | func TestDelNeedle(t *testing.T) { 32 | c := getClient() 33 | if err := c.delNeedle(1234567); err != nil { 34 | t.Fatalf("err:%v", err.Error()) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /directory/http.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bfs/libs/meta" 5 | "encoding/json" 6 | log "github.com/golang/glog" 7 | "net/http" 8 | "time" 9 | ) 10 | 11 | // HttpGetWriter 12 | func HttpGetWriter(r *http.Request, wr http.ResponseWriter, start time.Time, res *meta.Response) { 13 | var ( 14 | err error 15 | byteJson []byte 16 | ret = res.Ret 17 | ) 18 | if byteJson, err = json.Marshal(res); err != nil { 19 | log.Errorf("json.Marshal(\"%v\") failed (%v)", res, err) 20 | return 21 | } 22 | wr.Header().Set("Content-Type", "application/json;charset=utf-8") 23 | if _, err = wr.Write(byteJson); err != nil { 24 | log.Errorf("HttpWriter Write error(%v)", err) 25 | return 26 | } 27 | log.Infof("%s path:%s(params:%s,time:%f,ret:%v)", r.Method, 28 | r.URL.Path, r.Form.Encode(), time.Now().Sub(start).Seconds(), ret) 29 | } 30 | 31 | // HttpUploadWriter 32 | func HttpUploadWriter(r *http.Request, wr http.ResponseWriter, start time.Time, res *meta.Response) { 33 | var ( 34 | err error 35 | byteJson []byte 36 | ret = res.Ret 37 | ) 38 | if byteJson, err = json.Marshal(res); err != nil { 39 | log.Errorf("json.Marshal(\"%v\") failed (%v)", res, err) 40 | return 41 | } 42 | wr.Header().Set("Content-Type", "application/json;charset=utf-8") 43 | if _, err = wr.Write(byteJson); err != nil { 44 | log.Errorf("HttpWriter Write error(%v)", err) 45 | return 46 | } 47 | log.Infof("%s path:%s(params:%s,time:%f,ret:%v)", r.Method, 48 | r.URL.Path, r.Form.Encode(), time.Now().Sub(start).Seconds(), ret) 49 | } 50 | 51 | // HttpDelWriter 52 | func HttpDelWriter(r *http.Request, wr http.ResponseWriter, start time.Time, res *meta.Response) { 53 | var ( 54 | err error 55 | byteJson []byte 56 | ret = res.Ret 57 | ) 58 | if byteJson, err = json.Marshal(res); err != nil { 59 | log.Errorf("json.Marshal(\"%v\") failed (%v)", res, err) 60 | return 61 | } 62 | wr.Header().Set("Content-Type", "application/json;charset=utf-8") 63 | if _, err = wr.Write(byteJson); err != nil { 64 | log.Errorf("HttpWriter Write error(%v)", err) 65 | return 66 | } 67 | log.Infof("%s path:%s(params:%s,time:%f,ret:%v)", r.Method, 68 | r.URL.Path, r.Form.Encode(), time.Now().Sub(start).Seconds(), ret) 69 | } 70 | -------------------------------------------------------------------------------- /directory/http_api_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "io/ioutil" 8 | "net/http" 9 | "testing" 10 | "time" 11 | 12 | "bfs/directory/conf" 13 | dzk "bfs/directory/zk" 14 | "bfs/libs/meta" 15 | ) 16 | 17 | func TestHTTPAPI(t *testing.T) { 18 | var ( 19 | err error 20 | config *conf.Config 21 | zk *dzk.Zookeeper 22 | d *Directory 23 | key int64 24 | cookie int32 25 | body []byte 26 | url string 27 | resp *http.Response 28 | res meta.Response 29 | buf = &bytes.Buffer{} 30 | ) 31 | if config, err = conf.NewConfig("./directory.toml"); err != nil { 32 | t.Errorf("NewConfig() error(%v)", err) 33 | t.FailNow() 34 | } 35 | 36 | if zk, err = dzk.NewZookeeper(config); err != nil { 37 | t.Errorf("NewZookeeper() error(%v)", err) 38 | t.FailNow() 39 | } 40 | defer zk.Close() 41 | if d, err = NewDirectory(config); err != nil { 42 | t.Errorf("NewDirectory() error(%v)", err) 43 | t.FailNow() 44 | } 45 | StartApi(config.ApiListen, d) 46 | time.Sleep(1 * time.Second) 47 | buf.Reset() 48 | buf.WriteString("num=1") 49 | if resp, err = http.Post("http://172.16.13.86:6065/upload", "application/x-www-form-urlencoded", buf); err != nil { 50 | t.Errorf("http.Post error(%v)", err) 51 | t.FailNow() 52 | } 53 | defer resp.Body.Close() 54 | if resp.StatusCode != http.StatusOK { 55 | t.Errorf("http ERROR") 56 | t.FailNow() 57 | } 58 | if body, err = ioutil.ReadAll(resp.Body); err != nil { 59 | t.Errorf("ioutil.ReadAll error(%v)", err) 60 | t.FailNow() 61 | } 62 | if err = json.Unmarshal(body, &res); err != nil { 63 | t.Errorf("json.Unmarshal error(%v)", err) 64 | t.FailNow() 65 | } 66 | key = res.Key 67 | cookie = res.Cookie 68 | fmt.Println("put vid:", res.Vid) 69 | buf.Reset() 70 | url = fmt.Sprintf("http://172.16.13.86:6065/get?key=%d&cookie=%d", key, cookie) 71 | if resp, err = http.Get(url); err != nil { 72 | t.Errorf("http ERROR error(%v)", err) 73 | t.FailNow() 74 | } 75 | if resp.StatusCode != http.StatusOK { 76 | t.Errorf("http ERROR") 77 | t.FailNow() 78 | } 79 | if body, err = ioutil.ReadAll(resp.Body); err != nil { 80 | t.Errorf("ioutil.ReadAll error(%v)", err) 81 | t.FailNow() 82 | } 83 | if err = json.Unmarshal(body, &res); err != nil { 84 | t.Errorf("json.Unmarshal error(%v)", err) 85 | t.FailNow() 86 | } 87 | fmt.Println("get vid:", res.Vid) 88 | buf.Reset() 89 | buf.WriteString(fmt.Sprintf("key=%d&cookie=%d", key, cookie)) 90 | if resp, err = http.Post("http://172.16.13.86:6065/del", "application/x-www-form-urlencoded", buf); err != nil { 91 | t.Errorf("http.Post error(%v)", err) 92 | t.FailNow() 93 | } 94 | if resp.StatusCode != http.StatusOK { 95 | t.Errorf("http ERROR") 96 | t.FailNow() 97 | } 98 | if body, err = ioutil.ReadAll(resp.Body); err != nil { 99 | t.Errorf("ioutil.ReadAll error(%v)", err) 100 | t.FailNow() 101 | } 102 | if err = json.Unmarshal(body, &res); err != nil { 103 | t.Errorf("json.Unmarshal error(%v)", err) 104 | t.FailNow() 105 | } 106 | fmt.Println("del vid", res.Vid) 107 | } 108 | -------------------------------------------------------------------------------- /directory/http_perf.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | log "github.com/golang/glog" 5 | "net/http" 6 | _ "net/http/pprof" 7 | ) 8 | 9 | // StartPprof start a golang pprof. 10 | func StartPprof(addr string) { 11 | go func() { 12 | var err error 13 | if err = http.ListenAndServe(addr, nil); err != nil { 14 | log.Errorf("http.ListenAndServe(\"%s\") error(%v)", addr, err) 15 | return 16 | } 17 | }() 18 | } 19 | -------------------------------------------------------------------------------- /directory/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bfs/directory/conf" 5 | "flag" 6 | "runtime" 7 | 8 | log "github.com/golang/glog" 9 | ) 10 | 11 | var ( 12 | configFile string 13 | ) 14 | 15 | func init() { 16 | flag.StringVar(&configFile, "c", "./directory.toml", " set directory config file path") 17 | } 18 | 19 | func main() { 20 | var ( 21 | c *conf.Config 22 | d *Directory 23 | err error 24 | ) 25 | flag.Parse() 26 | defer log.Flush() 27 | runtime.GOMAXPROCS(runtime.NumCPU()) 28 | log.Infof("bfs directory start") 29 | if c, err = conf.NewConfig(configFile); err != nil { 30 | log.Errorf("NewConfig(\"%s\") error(%v)", configFile, err) 31 | return 32 | } 33 | log.Infof("new directory...") 34 | if d, err = NewDirectory(c); err != nil { 35 | log.Errorf("NewDirectory() failed, Quit now error(%v)", err) 36 | return 37 | } 38 | log.Infof("init http api...") 39 | StartApi(c.ApiListen, d) 40 | if c.PprofEnable { 41 | log.Infof("init http pprof...") 42 | StartPprof(c.PprofListen) 43 | } 44 | StartSignal() 45 | return 46 | } 47 | -------------------------------------------------------------------------------- /directory/signal.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | log "github.com/golang/glog" 5 | "os" 6 | "os/signal" 7 | "syscall" 8 | ) 9 | 10 | // StartSignal register signals handler. 11 | func StartSignal() { 12 | var ( 13 | c chan os.Signal 14 | s os.Signal 15 | ) 16 | c = make(chan os.Signal, 1) 17 | signal.Notify(c, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, 18 | syscall.SIGINT, syscall.SIGSTOP) 19 | // Block until a signal is received. 20 | for { 21 | s = <-c 22 | log.Infof("get a signal %s", s.String()) 23 | switch s { 24 | case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGSTOP, syscall.SIGINT: 25 | return 26 | case syscall.SIGHUP: 27 | // TODO reload 28 | //return 29 | default: 30 | return 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /directory/snowflake/generate_key.go: -------------------------------------------------------------------------------- 1 | package snowflake 2 | 3 | import ( 4 | "errors" 5 | log "github.com/golang/glog" 6 | "time" 7 | ) 8 | 9 | const ( 10 | maxSize = 10000 11 | errorSleep = 1 * time.Second 12 | genKeyTimeout = 2 * time.Second 13 | ) 14 | 15 | // Genkey generate key for upload file 16 | type Genkey struct { 17 | client *Client 18 | keys chan int64 19 | } 20 | 21 | // NewGenkey 22 | func NewGenkey(zservers []string, zpath string, ztimeout time.Duration, workerId int64) (g *Genkey, err error) { 23 | if err = Init(zservers, zpath, ztimeout); err != nil { 24 | log.Errorf("NewGenkey Init error(%v)", err) 25 | return nil, err 26 | } 27 | g = &Genkey{} 28 | g.client = NewClient(workerId) 29 | g.keys = make(chan int64, maxSize) 30 | g.preGenerate() 31 | return 32 | } 33 | 34 | // Getkey get key for upload file 35 | func (g *Genkey) Getkey() (key int64, err error) { 36 | select { 37 | case key = <-g.keys: 38 | return 39 | case <-time.After(genKeyTimeout): 40 | err = errors.New("getKey timeout") 41 | return 42 | } 43 | } 44 | 45 | // preGenerate pre generate key until 1000 46 | func (g *Genkey) preGenerate() { 47 | var ( 48 | i int 49 | ) 50 | time.Sleep(errorSleep) 51 | for i = 0; i < 10; i++ { 52 | go func() { 53 | var ( 54 | key int64 55 | keys []int64 56 | err error 57 | ) 58 | for { 59 | if keys, err = g.client.Ids(100); err != nil { 60 | log.Errorf("preGenerate() error(%v) retry", err) 61 | time.Sleep(errorSleep) 62 | continue 63 | } 64 | for _, key = range keys { 65 | g.keys <- key 66 | } 67 | } 68 | }() 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /directory/snowflake/generate_key_test.go: -------------------------------------------------------------------------------- 1 | package snowflake 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func TestSnowflake(t *testing.T) { 10 | var ( 11 | err error 12 | genkey *Genkey 13 | i int 14 | key int64 15 | ) 16 | if genkey, err = NewGenkey([]string{"localhost:2181"}, "/gosnowflake-servers", time.Second*15, 0); err != nil { 17 | t.Errorf("NewGenkey failed error(%v)", err) 18 | t.FailNow() 19 | } 20 | for i = 0; i < 100000; i++ { 21 | if key, err = genkey.Getkey(); err != nil { 22 | t.Errorf("Getkey failed error(%v)", err) 23 | t.FailNow() 24 | } 25 | fmt.Println("key ", i, ":", key) 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /directory/zk/zk.go: -------------------------------------------------------------------------------- 1 | package zk 2 | 3 | import ( 4 | "bfs/directory/conf" 5 | log "github.com/golang/glog" 6 | "github.com/samuel/go-zookeeper/zk" 7 | "path" 8 | ) 9 | 10 | type Zookeeper struct { 11 | c *zk.Conn 12 | config *conf.Config 13 | } 14 | 15 | // NewZookeeper new a connection to zookeeper. 16 | func NewZookeeper(config *conf.Config) (z *Zookeeper, err error) { 17 | var ( 18 | s <-chan zk.Event 19 | ) 20 | z = &Zookeeper{} 21 | z.config = config 22 | if z.c, s, err = zk.Connect(config.Zookeeper.Addrs, config.Zookeeper.Timeout.Duration); err != nil { 23 | log.Errorf("zk.Connect(\"%v\") error(%v)", config.Zookeeper.Addrs, err) 24 | return 25 | } 26 | go func() { 27 | var e zk.Event 28 | for { 29 | if e = <-s; e.Type == 0 { 30 | return 31 | } 32 | log.Infof("zookeeper get a event: %s", e.State.String()) 33 | } 34 | }() 35 | return 36 | } 37 | 38 | // WatchRacks get all racks and watch 39 | func (z *Zookeeper) WatchRacks() (nodes []string, ev <-chan zk.Event, err error) { 40 | if _, _, ev, err = z.c.GetW(z.config.Zookeeper.StoreRoot); err != nil { 41 | log.Errorf("zk.GetW(\"%s\") error(%v)", z.config.Zookeeper.StoreRoot, err) 42 | return 43 | } 44 | if nodes, _, err = z.c.Children(z.config.Zookeeper.StoreRoot); err != nil { 45 | log.Errorf("zk.Children(\"%s\") error(%v)", z.config.Zookeeper.StoreRoot, err) 46 | } 47 | return 48 | } 49 | 50 | // Stores get all stores 51 | func (z *Zookeeper) Stores(rack string) (nodes []string, err error) { 52 | var spath = path.Join(z.config.Zookeeper.StoreRoot, rack) 53 | if nodes, _, err = z.c.Children(spath); err != nil { 54 | log.Errorf("zk.Children(\"%s\") error(%v)", spath, err) 55 | } 56 | return 57 | } 58 | 59 | // Store get store node data 60 | func (z *Zookeeper) Store(rack, store string) (data []byte, err error) { 61 | var spath = path.Join(z.config.Zookeeper.StoreRoot, rack, store) 62 | if data, _, err = z.c.Get(spath); err != nil { 63 | log.Errorf("zk.Get(\"%s\") error(%v)", spath, err) 64 | } 65 | return 66 | } 67 | 68 | // StoreVolumes get volumes of store 69 | func (z *Zookeeper) StoreVolumes(rack, store string) (nodes []string, err error) { 70 | var spath = path.Join(z.config.Zookeeper.StoreRoot, rack, store) 71 | if nodes, _, err = z.c.Children(spath); err != nil { 72 | log.Errorf("zk.Children(\"%s\") error(%v)", spath, err) 73 | } 74 | return 75 | } 76 | 77 | // Volumes get all volumes 78 | func (z *Zookeeper) Volumes() (nodes []string, err error) { 79 | if nodes, _, err = z.c.Children(z.config.Zookeeper.VolumeRoot); err != nil { 80 | log.Errorf("zk.Children(\"%s\") error(%v)", z.config.Zookeeper.VolumeRoot, err) 81 | } 82 | return 83 | } 84 | 85 | // Volume get volume node data 86 | func (z *Zookeeper) Volume(volume string) (data []byte, err error) { 87 | var spath = path.Join(z.config.Zookeeper.VolumeRoot, volume) 88 | if data, _, err = z.c.Get(spath); err != nil { 89 | log.Errorf("zk.Get(\"%s\") error(%v)", spath, err) 90 | } 91 | return 92 | } 93 | 94 | // VolumeStores get stores of volume 95 | func (z *Zookeeper) VolumeStores(volume string) (nodes []string, err error) { 96 | var spath = path.Join(z.config.Zookeeper.VolumeRoot, volume) 97 | if nodes, _, err = z.c.Children(spath); err != nil { 98 | log.Errorf("zk.Get(\"%s\") error(%v)", spath, err) 99 | } 100 | return 101 | } 102 | 103 | // Groups get all groups and watch 104 | func (z *Zookeeper) Groups() (nodes []string, err error) { 105 | if nodes, _, err = z.c.Children(z.config.Zookeeper.GroupRoot); err != nil { 106 | log.Errorf("zk.Children(\"%s\") error(%v)", z.config.Zookeeper.GroupRoot, err) 107 | } 108 | return 109 | } 110 | 111 | // GroupStores get stores of group 112 | func (z *Zookeeper) GroupStores(group string) (nodes []string, err error) { 113 | var spath = path.Join(z.config.Zookeeper.GroupRoot, group) 114 | if nodes, _, err = z.c.Children(spath); err != nil { 115 | log.Errorf("zk.Children(\"%s\") error(%v)", spath, err) 116 | } 117 | return 118 | } 119 | 120 | // Close close the zookeeper connection. 121 | func (z *Zookeeper) Close() { 122 | z.c.Close() 123 | } 124 | -------------------------------------------------------------------------------- /directory/zk/zk_test.go: -------------------------------------------------------------------------------- 1 | package zk 2 | 3 | import ( 4 | "bfs/libs/meta" 5 | "encoding/json" 6 | "fmt" 7 | "testing" 8 | "time" 9 | ) 10 | 11 | func TestZk(t *testing.T) { 12 | 13 | var ( 14 | zk *Zookeeper 15 | rack, store, volume, group string 16 | racks, stores, volumes, groups []string 17 | data []byte 18 | storeMeta *meta.Store 19 | volumeState *meta.VolumeState 20 | err error 21 | ) 22 | 23 | if zk, err = NewZookeeper([]string{"localhost:2181"}, time.Second*1, "/rack", "/volume", "/group"); err != nil { 24 | t.Errorf("NewZookeeper() error(%v)", err) 25 | t.FailNow() 26 | } 27 | 28 | if racks, _, err = zk.WatchRacks(); err != nil { 29 | t.Errorf("WatchRacks() error(%v)", err) 30 | t.FailNow() 31 | } 32 | 33 | for _, rack = range racks { 34 | if stores, err = zk.Stores(rack); err != nil { 35 | t.Errorf("Stores() error(%v)", err) 36 | t.FailNow() 37 | } 38 | for _, store = range stores { 39 | if data, err = zk.Store(rack, store); err != nil { 40 | t.Errorf("Store() error(%v)", err) 41 | t.FailNow() 42 | } 43 | storeMeta = new(meta.Store) 44 | if err = json.Unmarshal(data, storeMeta); err != nil { 45 | t.Errorf("Unmarshal error(%v)", err) 46 | t.FailNow() 47 | } 48 | fmt.Println("store:", storeMeta.Id, storeMeta.Stat) 49 | if volumes, err = zk.StoreVolumes(rack, store); err != nil { 50 | t.Errorf("StoreVolumes() error(%v)", err) 51 | t.FailNow() 52 | } 53 | for _, volume = range volumes { 54 | fmt.Println("store:", store, "volume:", volume) 55 | } 56 | } 57 | } 58 | if volumes, err = zk.Volumes(); err != nil { 59 | t.Errorf("Volumes() error(%v)", err) 60 | t.FailNow() 61 | } 62 | for _, volume = range volumes { 63 | if data, err = zk.Volume(volume); err != nil { 64 | t.Errorf("Volume() error(%v)", err) 65 | t.FailNow() 66 | } 67 | volumeState = new(meta.VolumeState) 68 | if err = json.Unmarshal(data, volumeState); err != nil { 69 | t.Errorf("Unmarshal error(%v)", err) 70 | t.FailNow() 71 | } 72 | fmt.Println("volume:", volumeState.FreeSpace) 73 | if stores, err = zk.VolumeStores(volume); err != nil { 74 | t.Errorf("VolumeStores error(%v)", err) 75 | t.FailNow() 76 | } 77 | for _, store = range stores { 78 | fmt.Println("Volume:", volume, " store:", store) 79 | } 80 | } 81 | if groups, _, err = zk.WatchGroups(); err != nil { 82 | t.Errorf("WatchGroups error(%v)", err) 83 | t.FailNow() 84 | } 85 | for _, group = range groups { 86 | if stores, err = zk.GroupStores(group); err != nil { 87 | t.Errorf("GroupStores error(%v)") 88 | t.FailNow() 89 | } 90 | for _, store = range stores { 91 | fmt.Println("group:", group, " store:", store) 92 | } 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /doc/bfs.graffle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Terry-Mao/bfs/6a964012975171345fcb931ea9183ee58666748e/doc/bfs.graffle -------------------------------------------------------------------------------- /doc/directory.md: -------------------------------------------------------------------------------- 1 | # Directory 2 | Directory is part of bfs, it provides http apis for client 3 | 4 | Table of Contents 5 | ================= 6 | 7 | * [Features](#features) 8 | * [Architechure](#architechure) 9 | * [Directory](#directory) 10 | * [Dispatcher](#dispatcher) 11 | * [API](#api) 12 | * [Get](#get) 13 | * [Upload](#upload) 14 | * [Del](#del) 15 | * [Installation](#installation) 16 | 17 | ## Features 18 | * Scheduling module of bfs, directory provieds http api for client 19 | * High availability and easy extension 20 | 21 | [Back to TOC](#table-of-contents) 22 | 23 | ## API 24 | 25 | ### Get 26 | 27 | get a file 28 | 29 | **URL** 30 | 31 | http://DOMAIN/get 32 | 33 | ***HTTP Method*** 34 | 35 | GET 36 | 37 | ***Query String*** 38 | 39 | | name | required | type | description | 40 | | :----- | :--- | :--- | :--- | 41 | | key | true | int64 | file key | 42 | | cookie | true | int64 | file cookie | 43 | 44 | e.g curl "http://localhost:6065/get?key=679114092262199341&cookie=2937" 45 | 46 | ***Get Response*** 47 | 48 | ```json 49 | {"vid":315,"stores":["192.168.0.1:6062","192.168.0.2:6062","192.168.0.3:6062"]} 50 | ``` 51 | 52 | ### Upload 53 | 54 | upload a file 55 | 56 | **URL** 57 | 58 | http://DOMAIN/upload 59 | 60 | ***HTTP Method*** 61 | 62 | POST multipart/form-data 63 | 64 | ***Form String*** 65 | 66 | | name | required | type | description | 67 | | :----- | :--- | :--- | :--- | 68 | | num | true | int32 | num of files | 69 | 70 | e.g curl -d "num=2" "http://localhost:6065/upload" 71 | 72 | ***Upload Response*** 73 | 74 | ```json 75 | {"keys":[679114092262199341,679114092740349989],"vid":315,"cookie":2937,"stores":["192.168.0.1:6062","192.168.0.2:6062","192.168.0.3:6062"]} 76 | ``` 77 | 78 | ### Delete 79 | 80 | delete a file 81 | 82 | **URL** 83 | 84 | http://DOMAIN/del 85 | 86 | ***HTTP Method*** 87 | 88 | POST application/x-www-form-urlencoded 89 | 90 | ***Query String*** 91 | 92 | | name | required | type | description | 93 | | :----- | :--- | :--- | :--- | 94 | | key | true | int64 | file key | 95 | | cookie | true | int32 | cookie | 96 | 97 | e.g curl -d "key=5&cookie=5" "http://localhost:6065/del" 98 | 99 | ***Del Response*** 100 | 101 | ```json 102 | {"vid":315,"stores":["192.168.0.1:6062","192.168.0.2:6062","192.168.0.3:6062"]} 103 | ``` 104 | 105 | [Back to TOC](#table-of-contents) 106 | 107 | ## Architechure 108 | ### Directory 109 | Directory pull store status from zookeeper and update into memory 110 | 111 | ### Dispatcher 112 | Dispatcher schedule client requests, and guarantee load balancing 113 | 114 | [Back to TOC](#table-of-contents) 115 | 116 | ## Installation 117 | 118 | just pull `Terry-Mao/bfs` from github using `go get`: 119 | 120 | ```sh 121 | $ go get github.com/Terry-Mao/bfs 122 | $ cd $GOPATH/github.com/Terry-Mao/bfs 123 | $ go build 124 | ``` 125 | 126 | [Back to TOC](#table-of-contents) 127 | 128 | Have Fun! 129 | -------------------------------------------------------------------------------- /doc/pitchfork.md: -------------------------------------------------------------------------------- 1 | # Pitchfork 2 | Pitchfork is part of bfs, it's for probe Store and feed back to Directory 3 | 4 | Table of Contents 5 | ================= 6 | 7 | * [Features](#features) 8 | * [Architechure](#architechure) 9 | * [Pitchfork](#pitchfork) 10 | * [Store](#store) 11 | * [Installation](#installation) 12 | 13 | ## Features 14 | * Mostly probe all store nodes and feed back to all directorys 15 | * Adaptive Designs when store nodes change or pitchfork nodes change 16 | * High-low coupling pitchfork feed back to directory through zookeeper 17 | 18 | [Back to TOC](#table-of-contents) 19 | 20 | ## Architechure 21 | ### Pitchfork 22 | Pitchfork contains unique id of pitchfork 23 | 24 | ### Store 25 | Store contains unique id, rack position in zookeeper and accessed host 26 | 27 | [Back to TOC](#table-of-contents) 28 | 29 | ## Installation 30 | 31 | just pull `Terry-Mao/bfs` from github using `go get`: 32 | 33 | ```sh 34 | $ go get github.com/Terry-Mao/bfs 35 | $ cd $GOPATH/github.com/Terry-Mao/bfs/pitchfork 36 | $ go build 37 | ``` 38 | 39 | [Back to TOC](#table-of-contents) 40 | 41 | Have Fun! 42 | -------------------------------------------------------------------------------- /libs/encoding/binary/endian.go: -------------------------------------------------------------------------------- 1 | package binary 2 | 3 | import ( 4 | "bufio" 5 | ) 6 | 7 | var BigEndian bigEndian 8 | 9 | type bigEndian struct{} 10 | 11 | func (bigEndian) Uint16(b []byte) uint16 { return uint16(b[1]) | uint16(b[0])<<8 } 12 | 13 | func (bigEndian) PutUint16(b []byte, v uint16) { 14 | b[0] = byte(v >> 8) 15 | b[1] = byte(v) 16 | } 17 | 18 | func (bigEndian) Int32(b []byte) int32 { 19 | return int32(b[3]) | int32(b[2])<<8 | int32(b[1])<<16 | int32(b[0])<<24 20 | } 21 | 22 | func (bigEndian) Uint32(b []byte) uint32 { 23 | return uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24 24 | } 25 | 26 | func (bigEndian) PutUint32(b []byte, v uint32) { 27 | b[0] = byte(v >> 24) 28 | b[1] = byte(v >> 16) 29 | b[2] = byte(v >> 8) 30 | b[3] = byte(v) 31 | } 32 | 33 | func (bigEndian) WriteUint32(w *bufio.Writer, v uint32) (err error) { 34 | if err = w.WriteByte(byte(v >> 24)); err != nil { 35 | return 36 | } 37 | if err = w.WriteByte(byte(v >> 16)); err != nil { 38 | return 39 | } 40 | if err = w.WriteByte(byte(v >> 8)); err != nil { 41 | return 42 | } 43 | err = w.WriteByte(byte(v)) 44 | return 45 | } 46 | 47 | func (bigEndian) PutInt32(b []byte, v int32) { 48 | b[0] = byte(v >> 24) 49 | b[1] = byte(v >> 16) 50 | b[2] = byte(v >> 8) 51 | b[3] = byte(v) 52 | } 53 | 54 | func (bigEndian) WriteInt32(w *bufio.Writer, v int32) (err error) { 55 | if err = w.WriteByte(byte(v >> 24)); err != nil { 56 | return 57 | } 58 | if err = w.WriteByte(byte(v >> 16)); err != nil { 59 | return 60 | } 61 | if err = w.WriteByte(byte(v >> 8)); err != nil { 62 | return 63 | } 64 | err = w.WriteByte(byte(v)) 65 | return 66 | } 67 | 68 | func (bigEndian) Int64(b []byte) int64 { 69 | return int64(b[7]) | int64(b[6])<<8 | int64(b[5])<<16 | int64(b[4])<<24 | 70 | int64(b[3])<<32 | int64(b[2])<<40 | int64(b[1])<<48 | int64(b[0])<<56 71 | } 72 | 73 | func (bigEndian) Uint64(b []byte) uint64 { 74 | return uint64(b[7]) | uint64(b[6])<<8 | uint64(b[5])<<16 | uint64(b[4])<<24 | 75 | uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56 76 | } 77 | 78 | func (bigEndian) PutInt64(b []byte, v int64) { 79 | b[0] = byte(v >> 56) 80 | b[1] = byte(v >> 48) 81 | b[2] = byte(v >> 40) 82 | b[3] = byte(v >> 32) 83 | b[4] = byte(v >> 24) 84 | b[5] = byte(v >> 16) 85 | b[6] = byte(v >> 8) 86 | b[7] = byte(v) 87 | } 88 | 89 | func (bigEndian) WriteInt64(w *bufio.Writer, v int64) (err error) { 90 | if err = w.WriteByte(byte(v >> 56)); err != nil { 91 | return 92 | } 93 | if err = w.WriteByte(byte(v >> 48)); err != nil { 94 | return 95 | } 96 | if err = w.WriteByte(byte(v >> 40)); err != nil { 97 | return 98 | } 99 | if err = w.WriteByte(byte(v >> 32)); err != nil { 100 | return 101 | } 102 | if err = w.WriteByte(byte(v >> 24)); err != nil { 103 | return 104 | } 105 | if err = w.WriteByte(byte(v >> 16)); err != nil { 106 | return 107 | } 108 | if err = w.WriteByte(byte(v >> 8)); err != nil { 109 | return 110 | } 111 | err = w.WriteByte(byte(v)) 112 | return 113 | } 114 | -------------------------------------------------------------------------------- /libs/errors/common.go: -------------------------------------------------------------------------------- 1 | package errors 2 | 3 | const ( 4 | RetOK = 1 5 | RetServiceUnavailable = 65533 6 | RetParamErr = 65534 7 | RetInternalErr = 65535 8 | 9 | // needle 10 | RetNeedleExist = 5000 11 | ) 12 | 13 | var ( 14 | // common 15 | ErrParam = Error(RetParamErr) 16 | ErrInternal = Error(RetInternalErr) 17 | ErrServiceUnavailable = Error(RetServiceUnavailable) 18 | 19 | ErrNeedleExist = Error(RetNeedleExist) 20 | ) 21 | -------------------------------------------------------------------------------- /libs/errors/directory.go: -------------------------------------------------------------------------------- 1 | package errors 2 | 3 | const ( 4 | // hbase 5 | RetHBase = 30100 6 | // id 7 | RetIdNotAvailable = 30200 8 | // store 9 | RetStoreNotAvailable = 30300 10 | // zookeeper 11 | RetZookeeperDataError = 30400 12 | ) 13 | 14 | var ( 15 | // hbase 16 | ErrHBase = Error(RetHBase) 17 | // id 18 | ErrIdNotAvailable = Error(RetIdNotAvailable) 19 | // store 20 | ErrStoreNotAvailable = Error(RetStoreNotAvailable) 21 | // zookeeper 22 | ErrZookeeperDataError = Error(RetZookeeperDataError) 23 | ) 24 | -------------------------------------------------------------------------------- /libs/errors/errors.go: -------------------------------------------------------------------------------- 1 | package errors 2 | 3 | type Error int 4 | 5 | func (e Error) Error() string { 6 | return errorMsg[int(e)] 7 | } 8 | 9 | var ( 10 | errorMsg = map[int]string{ 11 | /* ========================= Store ========================= */ 12 | // common 13 | RetOK: "ok", 14 | RetParamErr: "store param error", 15 | RetInternalErr: "internal server error", 16 | // api 17 | RetUploadMaxFile: "exceed upload max file num", 18 | // block 19 | RetSuperBlockMagic: "super block magic not match", 20 | RetSuperBlockVer: "super block ver not match", 21 | RetSuperBlockPadding: "super block padding not match", 22 | RetSuperBlockNoSpace: "super block no left free space", 23 | RetSuperBlockRepairSize: "super block repair size must equal original", 24 | RetSuperBlockClosed: "super block closed", 25 | RetSuperBlockOffset: "super block offset not consistency with size", 26 | // index 27 | RetIndexSize: "index size error", 28 | RetIndexClosed: "index closed", 29 | RetIndexOffset: "index offset", 30 | RetIndexEOF: "index eof", 31 | // needle 32 | RetNeedleExist: "needle already exist", 33 | RetNeedleNotExist: "needle not exist", 34 | RetNeedleChecksum: "needle data checksum not match", 35 | RetNeedleFlag: "needle flag not match", 36 | RetNeedleSize: "needle size error", 37 | RetNeedleHeaderMagic: "needle header magic not match", 38 | RetNeedleFooterMagic: "needle footer magic not match", 39 | RetNeedleKey: "needle key not match", 40 | RetNeedlePadding: "needle padding not match", 41 | RetNeedleCookie: "needle cookie not match", 42 | RetNeedleDeleted: "needle deleted", 43 | RetNeedleTooLarge: "needle has no left free space", 44 | RetNeedleHeaderSize: "needle header size", 45 | RetNeedleDataSize: "needle data size", 46 | RetNeedleFooterSize: "needle footer size", 47 | RetNeedlePaddingSize: "needle padding size", 48 | RetNeedleFull: "needle full", 49 | // ring 50 | RetRingEmpty: "index ring buffer empty", 51 | RetRingFull: "index ring buffer full", 52 | // store 53 | RetStoreVolumeIndex: "store volume index", 54 | RetStoreNoFreeVolume: "store no free volume", 55 | RetStoreFileExist: "store rename file exist", 56 | // volume 57 | RetVolumeExist: "volume exist", 58 | RetVolumeNotExist: "volume not exist", 59 | RetVolumeDel: "volume deleted", 60 | RetVolumeInCompact: "volume in compacting", 61 | RetVolumeClosed: "volume closed", 62 | RetVolumeBatch: "volume exceed batch write number", 63 | /* ========================= Store ========================= */ 64 | /* ========================= Directory ========================= */ 65 | // hbase 66 | RetHBase: "hbase failed", 67 | // id 68 | RetIdNotAvailable: "generate id failed", 69 | // store 70 | RetStoreNotAvailable: "store not available", 71 | // zookeeper 72 | RetZookeeperDataError: "zookeeper data error", 73 | /* ========================= Directory ========================= */ 74 | /* ========================= Proxy ========================= */ 75 | // common 76 | RetBucketNotExist: "bucket not exist", 77 | RetAuthFailed: "authorization failed", 78 | RetUrlBad: "bad url", 79 | // upload 80 | RetFileTooLarge: "file too large", 81 | /* ========================= Proxy ========================= */ 82 | } 83 | ) 84 | -------------------------------------------------------------------------------- /libs/errors/proxy.go: -------------------------------------------------------------------------------- 1 | package errors 2 | 3 | const ( 4 | // common 5 | RetUrlBad = 400 6 | RetAuthFailed = 401 7 | RetBucketNotExist = 404 8 | // upload 9 | RetFileTooLarge = 413 10 | ) 11 | 12 | var ( 13 | // common 14 | ErrUrlBad = Error(RetUrlBad) 15 | ErrAuthFailed = Error(RetAuthFailed) 16 | ErrBucketNotExist = Error(RetBucketNotExist) 17 | // upload 18 | ErrFileTooLarge = Error(RetFileTooLarge) 19 | ) 20 | -------------------------------------------------------------------------------- /libs/errors/store.go: -------------------------------------------------------------------------------- 1 | package errors 2 | 3 | const ( 4 | // api 5 | RetUploadMaxFile = 2000 6 | RetDelMaxFile = 2001 7 | // block 8 | RetSuperBlockMagic = 3000 9 | RetSuperBlockVer = 3001 10 | RetSuperBlockPadding = 3002 11 | RetSuperBlockNoSpace = 3003 12 | RetSuperBlockRepairSize = 3004 13 | RetSuperBlockClosed = 3005 14 | RetSuperBlockOffset = 3006 15 | // index 16 | RetIndexSize = 4000 17 | RetIndexClosed = 4001 18 | RetIndexOffset = 4002 19 | RetIndexEOF = 4003 20 | // needle 21 | RetNeedleNotExist = 5001 22 | RetNeedleChecksum = 5002 23 | RetNeedleFlag = 5003 24 | RetNeedleSize = 5004 25 | RetNeedleHeaderMagic = 5005 26 | RetNeedleFooterMagic = 5006 27 | RetNeedleKey = 5007 28 | RetNeedlePadding = 5008 29 | RetNeedleCookie = 5009 30 | RetNeedleDeleted = 5010 31 | RetNeedleTooLarge = 5011 32 | RetNeedleHeaderSize = 5012 33 | RetNeedleDataSize = 5013 34 | RetNeedleFooterSize = 5014 35 | RetNeedlePaddingSize = 5015 36 | RetNeedleFull = 5016 37 | // ring 38 | RetRingEmpty = 6000 39 | RetRingFull = 6001 40 | // store 41 | RetStoreVolumeIndex = 7000 42 | RetStoreNoFreeVolume = 7001 43 | RetStoreFileExist = 7002 44 | // volume 45 | RetVolumeExist = 8000 46 | RetVolumeNotExist = 8001 47 | RetVolumeDel = 8002 48 | RetVolumeInCompact = 8003 49 | RetVolumeClosed = 8004 50 | RetVolumeBatch = 8005 51 | ) 52 | 53 | var ( 54 | ErrUploadMaxFile = Error(RetUploadMaxFile) 55 | ErrDelMaxFile = Error(RetDelMaxFile) 56 | // block 57 | ErrSuperBlockMagic = Error(RetSuperBlockMagic) 58 | ErrSuperBlockVer = Error(RetSuperBlockVer) 59 | ErrSuperBlockPadding = Error(RetSuperBlockPadding) 60 | ErrSuperBlockNoSpace = Error(RetSuperBlockNoSpace) 61 | ErrSuperBlockRepairSize = Error(RetSuperBlockRepairSize) 62 | ErrSuperBlockClosed = Error(RetSuperBlockClosed) 63 | ErrSuperBlockOffset = Error(RetSuperBlockOffset) 64 | // index 65 | ErrIndexSize = Error(RetIndexSize) 66 | ErrIndexClosed = Error(RetIndexClosed) 67 | ErrIndexOffset = Error(RetIndexOffset) 68 | ErrIndexEOF = Error(RetIndexEOF) 69 | // needle 70 | ErrNeedleNotExist = Error(RetNeedleNotExist) 71 | ErrNeedleChecksum = Error(RetNeedleChecksum) 72 | ErrNeedleFlag = Error(RetNeedleFlag) 73 | ErrNeedleSize = Error(RetNeedleSize) 74 | ErrNeedleHeaderMagic = Error(RetNeedleHeaderMagic) 75 | ErrNeedleFooterMagic = Error(RetNeedleFooterMagic) 76 | ErrNeedleKey = Error(RetNeedleKey) 77 | ErrNeedlePadding = Error(RetNeedlePadding) 78 | ErrNeedleCookie = Error(RetNeedleCookie) 79 | ErrNeedleDeleted = Error(RetNeedleDeleted) 80 | ErrNeedleTooLarge = Error(RetNeedleTooLarge) 81 | ErrNeedleHeaderSize = Error(RetNeedleHeaderSize) 82 | ErrNeedleDataSize = Error(RetNeedleDataSize) 83 | ErrNeedleFooterSize = Error(RetNeedleFooterSize) 84 | ErrNeedlePaddingSize = Error(RetNeedlePaddingSize) 85 | ErrNeedleFull = Error(RetNeedleFull) 86 | // ring 87 | ErrRingEmpty = Error(RetRingEmpty) 88 | ErrRingFull = Error(RetRingFull) 89 | // store 90 | ErrStoreVolumeIndex = Error(RetStoreVolumeIndex) 91 | ErrStoreNoFreeVolume = Error(RetStoreNoFreeVolume) 92 | ErrStoreFileExist = Error(RetStoreFileExist) 93 | // volume 94 | ErrVolumeExist = Error(RetVolumeExist) 95 | ErrVolumeNotExist = Error(RetVolumeNotExist) 96 | ErrVolumeDel = Error(RetVolumeDel) 97 | ErrVolumeInCompact = Error(RetVolumeInCompact) 98 | ErrVolumeClosed = Error(RetVolumeClosed) 99 | ErrVolumeBatch = Error(RetVolumeBatch) 100 | ) 101 | -------------------------------------------------------------------------------- /libs/gohbase/.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: 3 | - 1.5.3 4 | before_install: 5 | - go get golang.org/x/tools/cmd/cover github.com/golang/lint/golint 6 | install: 7 | - go get ./... 8 | after_success: 9 | - make coverdata 10 | - bash <(curl -s https://codecov.io/bash) 11 | script: 12 | - make -j4 check GOTEST_FLAGS=-v 13 | -------------------------------------------------------------------------------- /libs/gohbase/AUTHORS: -------------------------------------------------------------------------------- 1 | The GoHBase Authors 2 | ------------------- 3 | 4 | GoHBase ("gohbase") was originally written by Benoit Sigoure. 5 | 6 | All contributors are required to sign a "Contributor License Agreement" at 7 | http://opentsdb.net/contributing.html 8 | 9 | The following organizations and people have contributed at least 0.5% of the 10 | current code of GoHBase. 11 | (Please keep both lists sorted alphabetically.) 12 | 13 | 14 | 15 | 16 | 17 | Benoit Sigoure 18 | 19 | 20 | 21 | This list can be obtained at any time with the following script: 22 | 23 | find src test -type f \ 24 | | while read i; do \ 25 | git blame -t $i 2>/dev/null; \ 26 | done \ 27 | | sed 's/^[0-9a-f]\{8\} [^(]*(\([^)]*\) [-+0-9 ]\{14,\}).*/\1/;s/ *$//' \ 28 | | awk '{a[$0]++; t++} END{for(n in a) if (a[n]*100.0/t > 0.5) print n}' \ 29 | | sort 30 | -------------------------------------------------------------------------------- /libs/gohbase/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2015 The GoHBase Authors. All rights reserved. 2 | # This file is part of GoHBase. 3 | # Use of this source code is governed by the Apache License 2.0 4 | # that can be found in the COPYING file. 5 | 6 | GO := go 7 | TEST_TIMEOUT := 30s 8 | INTEGRATION_TIMEOUT := 120s 9 | GOTEST_FLAGS := 10 | 11 | DEFAULT_GOPATH := $${GOPATH%%:*} 12 | GOPATH_BIN := $(DEFAULT_GOPATH)/bin 13 | GOLINT := $(GOPATH_BIN)/golint 14 | 15 | all: install 16 | 17 | install: 18 | $(GO) install ./... 19 | 20 | check: vet test fmtcheck lint 21 | 22 | COVER_PKGS := `find ./* -name '*_test.go' | xargs -I{} dirname {} | sort -u` 23 | COVER_MODE := count 24 | coverdata: 25 | echo 'mode: $(COVER_MODE)' >coverage.out 26 | for dir in $(COVER_PKGS); do \ 27 | $(GO) test -covermode=$(COVER_MODE) -coverprofile=cov.out-t $$dir || exit; \ 28 | tail -n +2 cov.out-t >> coverage.out && \ 29 | rm cov.out-t; \ 30 | done; 31 | 32 | coverage: coverdata 33 | $(GO) tool cover -html=coverage.out 34 | rm -f coverage.out 35 | 36 | fmtcheck: 37 | errors=`gofmt -l .`; if test -n "$$errors"; then echo Check these files for style errors:; echo "$$errors"; exit 1; fi 38 | find . -name '*.go' ! -path "./pb/*" -exec ./check_line_len.awk {} + 39 | 40 | vet: 41 | $(GO) vet ./... 42 | 43 | lint: 44 | find ./* -type d ! -name pb | xargs -L 1 $(GOLINT) &>lint; : 45 | if test -s lint; then echo Check these packages for golint:; cat lint; rm lint; exit 1; else rm lint; fi 46 | # The above is ugly, but unfortunately golint doesn't exit 1 when it finds 47 | # lint. See https://github.com/golang/lint/issues/65 48 | 49 | test: 50 | $(GO) test $(GOTEST_FLAGS) -race -timeout=$(TEST_TIMEOUT) ./... 51 | 52 | integration: 53 | $(GO) test $(GOTEST_FLAGS) -race -timeout=$(INTEGRATION_TIMEOUT) -v integration_test.go 54 | 55 | .PHONY: all check coverage coverdata fmtcheck install integration lint test vet 56 | -------------------------------------------------------------------------------- /libs/gohbase/README.md: -------------------------------------------------------------------------------- 1 | # Golang HBase client [![Build Status](https://travis-ci.org/tsuna/gohbase.svg?branch=master)](https://travis-ci.org/tsuna/gohbase) [![codecov.io](http://codecov.io/github/tsuna/gohbase/coverage.svg?branch=master)](http://codecov.io/github/tsuna/gohbase?branch=master) [![GoDoc](https://godoc.org/github.com/tsuna/gohbase?status.png)](https://godoc.org/github.com/tsuna/gohbase) 2 | 3 | This is a pure-[Go](http://golang.org/) client for [HBase](http://hbase.org). 4 | 5 | Current status: prototype. 6 | 7 | ## Supported Versions 8 | 9 | HBase >= 1.0 10 | 11 | ## Installation 12 | 13 | go get github.com/tsuna/gohbase 14 | 15 | ## Example Usage 16 | 17 | #### Create a client 18 | ```go 19 | client := gohbase.NewClient("localhost") 20 | ``` 21 | #### Insert a cell 22 | ```go 23 | // Values maps a ColumnFamily -> Qualifiers -> Values. 24 | values := map[string]map[string][]byte{"cf": map[string][]byte{"a": []byte{0}}} 25 | putRequest, err := hrpc.NewPutStr(context.Background(), "table", "key", values) 26 | rsp, err := client.Put(putRequest) 27 | ``` 28 | 29 | #### Get an entire row 30 | ```go 31 | getRequest, err := hrpc.NewGetStr(context.Background(), "table", "row") 32 | getRsp, err := client.Get(getRequest) 33 | ``` 34 | 35 | #### Get a specific cell 36 | ```go 37 | // Perform a get for the cell with key "15", column family "cf" and qualifier "a" 38 | family := map[string][]string{"cf": []string{"a"}} 39 | getRequest, err := hrpc.NewGetStr(context.Background(), "table", "15", 40 | hrpc.Families(family)) 41 | getRsp, err := client.Get(getRequest) 42 | ``` 43 | 44 | #### Get a specific cell with a filter 45 | ```go 46 | pFilter := filter.NewKeyOnlyFilter(true) 47 | family := map[string][]string{"cf": []string{"a"}} 48 | getRequest, err := hrpc.NewGetStr(context.Background(), "table", "15", 49 | hrpc.Families(family), hrpc.Filters(pFilter)) 50 | getRsp, err := client.Get(getRequest) 51 | ``` 52 | 53 | #### Scan with a filter 54 | ```go 55 | pFilter := filter.NewPrefixFilter([]byte("7")) 56 | scanRequest, err := hrpc.NewScanStr(context.Background(), "table", 57 | hrpc.Filters(pFilter)) 58 | scanRsp, err := client.Scan(scanRequest) 59 | ``` 60 | 61 | ## Contributing 62 | 63 | Any help would be appreciated. Please use 64 | [GerritHub](https://review.gerrithub.io/#/admin/projects/tsuna/gohbase) to 65 | send changes for review, instead of GitHub pull requests. Please sign the 66 | [Contributor License Agreement](https://docs.google.com/spreadsheet/viewform?formkey=dFNiOFROLXJBbFBmMkQtb1hNMWhUUnc6MQ) 67 | when you send your first change for review. 68 | 69 | ## License 70 | 71 | Copyright © 2015 The GoHBase Authors. All rights reserved. Use of this source code is governed by the Apache License 2.0 that can be found in the [COPYING](COPYING) file. 72 | -------------------------------------------------------------------------------- /libs/gohbase/c_test.go: -------------------------------------------------------------------------------- 1 | package gohbase 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "time" 7 | 8 | "bfs/libs/gohbase/conf" 9 | "bfs/libs/gohbase/hrpc" 10 | 11 | log "github.com/golang/glog" 12 | ) 13 | 14 | func getStopRow(s []byte) []byte { 15 | res := make([]byte, len(s)+20) 16 | copy(res, s) 17 | return res 18 | } 19 | 20 | func TestMGet1(t *testing.T) { 21 | c := newClient(standardClient, conf.NewConf([]string{"172.16.33.45:2181"}, "", "", "", 30*time.Second, 0, 0, 0)) 22 | keys := []string{ 23 | "row_001", "row_003", "row_007", 24 | } 25 | gets := make([]*hrpc.Get, len(keys)) 26 | for i, key := range keys { 27 | get, err := hrpc.NewGetStr(context.Background(), "fuckclient", key) 28 | if err != nil { 29 | log.Error("NewGetStr error for key %s, err is %v", key, err) 30 | continue 31 | } 32 | gets[i] = get 33 | } 34 | //time.Sleep(15 * time.Second) 35 | for _, get := range gets { 36 | st := time.Now() 37 | res, err := c.Get(get) 38 | if err != nil { 39 | log.Error("get meet error, err is %v", err) 40 | continue 41 | } 42 | et := time.Now() 43 | for _, cell := range res.Cells { 44 | log.Info("%s-%s-%s-%s, st: %v, et: %v, cost: %d", string(cell.Row), string(cell.Family), string(cell.Qualifier), string(cell.Value), st, et, (et.Nanosecond()-st.Nanosecond())/1000000) 45 | } 46 | time.Sleep(3 * time.Second) 47 | } 48 | 49 | c.clearAllRegions() 50 | log.Info("%v: clearAllRegions", time.Now()) 51 | time.Sleep(3 * time.Second) 52 | //time.Sleep(30 * time.Second) 53 | log.Info("%v: do second scan", time.Now()) 54 | 55 | gets = make([]*hrpc.Get, len(keys)) 56 | for i, key := range keys { 57 | get, err := hrpc.NewGetStr(context.Background(), "fuckclient", key) 58 | if err != nil { 59 | log.Error("NewGetStr error for key %s, err is %v", key, err) 60 | continue 61 | } 62 | gets[i] = get 63 | } 64 | for _, get := range gets { 65 | st := time.Now() 66 | res, err := c.Get(get) 67 | if err != nil { 68 | log.Error("get meet error, err is %v", err) 69 | continue 70 | } 71 | et := time.Now() 72 | for _, cell := range res.Cells { 73 | log.Info("%s-%s-%s-%s, st: %v, et: %v, cost: %d", string(cell.Row), string(cell.Family), string(cell.Qualifier), string(cell.Value), st, et, (et.Nanosecond()-st.Nanosecond())/1000000) 74 | } 75 | } 76 | time.Sleep(1 * time.Second) 77 | } 78 | -------------------------------------------------------------------------------- /libs/gohbase/check_line_len.awk: -------------------------------------------------------------------------------- 1 | #!/usr/bin/awk -f 2 | 3 | BEGIN { 4 | max = 100; 5 | } 6 | 7 | # Expand tabs to 4 spaces. 8 | { 9 | gsub(/\t/, " "); 10 | } 11 | 12 | length() > max { 13 | errors++; 14 | print FILENAME ":" FNR ": Line too long (" length() "/" max ")"; 15 | } 16 | 17 | END { 18 | if (errors >= 125) { 19 | errors = 125; 20 | } 21 | exit errors; 22 | } 23 | -------------------------------------------------------------------------------- /libs/gohbase/conf/conf.go: -------------------------------------------------------------------------------- 1 | package conf 2 | 3 | import "time" 4 | 5 | // Conf RpcQueueSize <= 0 && FlushInterval <= 0 is not allowed 6 | type Conf struct { 7 | ZkRoot string 8 | Zkquorum []string 9 | Master, Meta string 10 | RpcQueueSize int 11 | ZkTimeout, FlushInterval, DialTimeout time.Duration 12 | } 13 | 14 | func NewConf(zkquorum []string, zkRoot, master, meta string, zkTimeout time.Duration, rpcQueueSize int, flushInterval, dialTimeout time.Duration) (res *Conf) { 15 | // set default value 16 | if rpcQueueSize <= 0 && flushInterval <= 0 { 17 | rpcQueueSize = 1 18 | } 19 | if dialTimeout == 0 { 20 | dialTimeout = 10 * time.Second 21 | } 22 | res = &Conf{ 23 | ZkRoot: zkRoot, 24 | Zkquorum: zkquorum, 25 | Master: master, 26 | Meta: meta, 27 | ZkTimeout: zkTimeout, 28 | RpcQueueSize: rpcQueueSize, 29 | FlushInterval: flushInterval, 30 | DialTimeout: dialTimeout, 31 | } 32 | 33 | return 34 | } 35 | -------------------------------------------------------------------------------- /libs/gohbase/discovery_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 The GoHBase Authors. All rights reserved. 2 | // This file is part of GoHBase. 3 | // Use of this source code is governed by the Apache License 2.0 4 | // that can be found in the COPYING file. 5 | 6 | package gohbase 7 | 8 | import ( 9 | "bytes" 10 | "testing" 11 | 12 | "bfs/libs/gohbase/conf" 13 | "bfs/libs/gohbase/pb" 14 | "bfs/libs/gohbase/regioninfo" 15 | "time" 16 | ) 17 | 18 | func TestRegionDiscovery(t *testing.T) { 19 | 20 | client := newClient(standardClient, conf.NewConf([]string{"~invalid.quorum"}, "", "", "", 30*time.Second, 0, 0, 0)) 21 | 22 | reg := client.getRegionFromCache([]byte("test"), []byte("theKey")) 23 | if reg != nil { 24 | t.Errorf("Found region %#v even though the cache was empty?!", reg) 25 | } 26 | 27 | // Inject a "test" table with a single region that covers the entire key 28 | // space (both the start and stop keys are empty). 29 | family := []byte("info") 30 | metaRow := &pb.GetResponse{ 31 | Result: &pb.Result{Cell: []*pb.Cell{ 32 | &pb.Cell{ 33 | Row: []byte("test,,1234567890042.56f833d5569a27c7a43fbf547b4924a4."), 34 | Family: family, 35 | Qualifier: []byte("regioninfo"), 36 | Value: []byte("PBUF\b\xc4\xcd\xe9\x99\xe0)\x12\x0f\n\adefault\x12\x04test" + 37 | "\x1a\x00\"\x00(\x000\x008\x00"), 38 | }, 39 | &pb.Cell{ 40 | Row: []byte("test,,1234567890042.56f833d5569a27c7a43fbf547b4924a4."), 41 | Family: family, 42 | Qualifier: []byte("seqnumDuringOpen"), 43 | Value: []byte("\x00\x00\x00\x00\x00\x00\x00\x02"), 44 | }, 45 | &pb.Cell{ 46 | Row: []byte("test,,1234567890042.56f833d5569a27c7a43fbf547b4924a4."), 47 | Family: family, 48 | Qualifier: []byte("server"), 49 | Value: []byte("localhost:50966"), 50 | }, 51 | &pb.Cell{ 52 | Row: []byte("test,,1234567890042.56f833d5569a27c7a43fbf547b4924a4."), 53 | Family: family, 54 | Qualifier: []byte("serverstartcode"), 55 | Value: []byte("\x00\x00\x01N\x02\x92R\xb1"), 56 | }, 57 | }}} 58 | 59 | reg, _, _, err := client.parseMetaTableResponse(metaRow) 60 | if err != nil { 61 | t.Fatalf("Failed to discover region: %s", err) 62 | } 63 | client.regions.put(reg.RegionName, reg) 64 | 65 | reg = client.getRegionFromCache([]byte("test"), []byte("theKey")) 66 | if reg == nil { 67 | t.Fatal("Region not found even though we injected it in the cache.") 68 | } 69 | expected := ®ioninfo.Info{ 70 | Table: []byte("test"), 71 | RegionName: []byte("test,,1234567890042.56f833d5569a27c7a43fbf547b4924a4."), 72 | StartKey: []byte(""), 73 | StopKey: []byte(""), 74 | } 75 | if !bytes.Equal(reg.Table, expected.Table) || 76 | !bytes.Equal(reg.RegionName, expected.RegionName) || 77 | !bytes.Equal(reg.StartKey, expected.StartKey) || 78 | !bytes.Equal(reg.StopKey, expected.StopKey) { 79 | t.Errorf("Found region %#v \nbut expected %#v", reg, expected) 80 | } 81 | 82 | reg = client.getRegionFromCache([]byte("notfound"), []byte("theKey")) 83 | if reg != nil { 84 | t.Errorf("Found region %#v even though this table doesn't exist", reg) 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /libs/gohbase/hbase/hbase.go: -------------------------------------------------------------------------------- 1 | package hbase 2 | 3 | type HBaseCell struct { 4 | Table string 5 | RowKey string 6 | Family string 7 | Qualifier string 8 | Value string 9 | } 10 | 11 | func (c *HBaseCell) Valid() bool { 12 | return c != nil && c.Table != "" && c.Family != "" && c.Qualifier != "" && c.Value != "" 13 | } 14 | -------------------------------------------------------------------------------- /libs/gohbase/hrpc/create.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 The GoHBase Authors. All rights reserved. 2 | // This file is part of GoHBase. 3 | // Use of this source code is governed by the Apache License 2.0 4 | // that can be found in the COPYING file. 5 | 6 | package hrpc 7 | 8 | import ( 9 | "context" 10 | 11 | "bfs/libs/gohbase/pb" 12 | 13 | "github.com/golang/protobuf/proto" 14 | ) 15 | 16 | // CreateTable represents a CreateTable HBase call 17 | type CreateTable struct { 18 | tableOp 19 | 20 | columns []string 21 | } 22 | 23 | // NewCreateTable creates a new CreateTable request that will create the given 24 | // table in HBase. For use by the admin client. 25 | func NewCreateTable(ctx context.Context, table []byte, columns []string) *CreateTable { 26 | ct := &CreateTable{ 27 | tableOp: tableOp{base{ 28 | table: table, 29 | ctx: ctx, 30 | }}, 31 | columns: columns, 32 | } 33 | return ct 34 | } 35 | 36 | // GetName returns the name of this RPC call. 37 | func (ct *CreateTable) GetName() string { 38 | return "CreateTable" 39 | } 40 | 41 | // Serialize will convert this HBase call into a slice of bytes to be written to 42 | // the network 43 | func (ct *CreateTable) Serialize() ([]byte, error) { 44 | pbcols := make([]*pb.ColumnFamilySchema, len(ct.columns)) 45 | for i, col := range ct.columns { 46 | pbcols[i] = &pb.ColumnFamilySchema{ 47 | Name: []byte(col), 48 | } 49 | } 50 | ctable := &pb.CreateTableRequest{ 51 | TableSchema: &pb.TableSchema{ 52 | TableName: &pb.TableName{ 53 | Namespace: []byte("default"), 54 | Qualifier: ct.table, 55 | }, 56 | ColumnFamilies: pbcols, 57 | }, 58 | } 59 | return proto.Marshal(ctable) 60 | } 61 | 62 | // NewResponse creates an empty protobuf message to read the response of this 63 | // RPC. 64 | func (ct *CreateTable) NewResponse() proto.Message { 65 | return &pb.CreateTableResponse{} 66 | } 67 | -------------------------------------------------------------------------------- /libs/gohbase/hrpc/delete.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 The GoHBase Authors. All rights reserved. 2 | // This file is part of GoHBase. 3 | // Use of this source code is governed by the Apache License 2.0 4 | // that can be found in the COPYING file. 5 | 6 | package hrpc 7 | 8 | import ( 9 | "context" 10 | 11 | "bfs/libs/gohbase/pb" 12 | 13 | "github.com/golang/protobuf/proto" 14 | ) 15 | 16 | // DeleteTable represents a DeleteTable HBase call 17 | type DeleteTable struct { 18 | tableOp 19 | } 20 | 21 | // NewDeleteTable creates a new DeleteTable request that will delete the 22 | // given table in HBase. For use by the admin client. 23 | func NewDeleteTable(ctx context.Context, table []byte) *DeleteTable { 24 | dt := &DeleteTable{ 25 | tableOp{base{ 26 | table: table, 27 | ctx: ctx, 28 | }}, 29 | } 30 | return dt 31 | } 32 | 33 | // GetName returns the name of this RPC call. 34 | func (dt *DeleteTable) GetName() string { 35 | return "DeleteTable" 36 | } 37 | 38 | // Serialize will convert this HBase call into a slice of bytes to be written to 39 | // the network 40 | func (dt *DeleteTable) Serialize() ([]byte, error) { 41 | dtreq := &pb.DeleteTableRequest{ 42 | TableName: &pb.TableName{ 43 | Namespace: []byte("default"), 44 | Qualifier: dt.table, 45 | }, 46 | } 47 | return proto.Marshal(dtreq) 48 | } 49 | 50 | // NewResponse creates an empty protobuf message to read the response of this 51 | // RPC. 52 | func (dt *DeleteTable) NewResponse() proto.Message { 53 | return &pb.DeleteTableResponse{} 54 | } 55 | -------------------------------------------------------------------------------- /libs/gohbase/hrpc/disable.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 The GoHBase Authors. All rights reserved. 2 | // This file is part of GoHBase. 3 | // Use of this source code is governed by the Apache License 2.0 4 | // that can be found in the COPYING file. 5 | 6 | package hrpc 7 | 8 | import ( 9 | "context" 10 | 11 | "bfs/libs/gohbase/pb" 12 | 13 | "github.com/golang/protobuf/proto" 14 | ) 15 | 16 | // DisableTable represents a DisableTable HBase call 17 | type DisableTable struct { 18 | tableOp 19 | } 20 | 21 | // NewDisableTable creates a new DisableTable request that will disable the 22 | // given table in HBase. For use by the admin client. 23 | func NewDisableTable(ctx context.Context, table []byte) *DisableTable { 24 | dt := &DisableTable{ 25 | tableOp{base{ 26 | table: table, 27 | ctx: ctx, 28 | }}, 29 | } 30 | return dt 31 | } 32 | 33 | // GetName returns the name of this RPC call. 34 | func (dt *DisableTable) GetName() string { 35 | return "DisableTable" 36 | } 37 | 38 | // Serialize will convert this HBase call into a slice of bytes to be written to 39 | // the network 40 | func (dt *DisableTable) Serialize() ([]byte, error) { 41 | dtreq := &pb.DisableTableRequest{ 42 | TableName: &pb.TableName{ 43 | Namespace: []byte("default"), 44 | Qualifier: dt.table, 45 | }, 46 | } 47 | return proto.Marshal(dtreq) 48 | } 49 | 50 | // NewResponse creates an empty protobuf message to read the response of this 51 | // RPC. 52 | func (dt *DisableTable) NewResponse() proto.Message { 53 | return &pb.DisableTableResponse{} 54 | } 55 | -------------------------------------------------------------------------------- /libs/gohbase/hrpc/enable.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 The GoHBase Authors. All rights reserved. 2 | // This file is part of GoHBase. 3 | // Use of this source code is governed by the Apache License 2.0 4 | // that can be found in the COPYING file. 5 | 6 | package hrpc 7 | 8 | import ( 9 | "context" 10 | 11 | "bfs/libs/gohbase/pb" 12 | 13 | "github.com/golang/protobuf/proto" 14 | ) 15 | 16 | // EnableTable represents a EnableTable HBase call 17 | type EnableTable struct { 18 | tableOp 19 | } 20 | 21 | // NewEnableTable creates a new EnableTable request that will enable the 22 | // given table in HBase. For use by the admin client. 23 | func NewEnableTable(ctx context.Context, table []byte) *EnableTable { 24 | et := &EnableTable{ 25 | tableOp{base{ 26 | table: table, 27 | ctx: ctx, 28 | }}, 29 | } 30 | return et 31 | } 32 | 33 | // GetName returns the name of this RPC call. 34 | func (et *EnableTable) GetName() string { 35 | return "EnableTable" 36 | } 37 | 38 | // Serialize will convert this HBase call into a slice of bytes to be written to 39 | // the network 40 | func (et *EnableTable) Serialize() ([]byte, error) { 41 | dtreq := &pb.EnableTableRequest{ 42 | TableName: &pb.TableName{ 43 | Namespace: []byte("default"), 44 | Qualifier: et.table, 45 | }, 46 | } 47 | return proto.Marshal(dtreq) 48 | } 49 | 50 | // NewResponse creates an empty protobuf message to read the response of this 51 | // RPC. 52 | func (et *EnableTable) NewResponse() proto.Message { 53 | return &pb.EnableTableResponse{} 54 | } 55 | -------------------------------------------------------------------------------- /libs/gohbase/hrpc/tableop.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 The GoHBase Authors. All rights reserved. 2 | // This file is part of GoHBase. 3 | // Use of this source code is governed by the Apache License 2.0 4 | // that can be found in the COPYING file. 5 | 6 | package hrpc 7 | 8 | import ( 9 | "errors" 10 | 11 | "bfs/libs/gohbase/filter" 12 | ) 13 | 14 | // tableOp represents an administrative operation on a table. 15 | type tableOp struct { 16 | base 17 | } 18 | 19 | // SetFilter always returns an error. 20 | func (to *tableOp) SetFilter(filter.Filter) error { 21 | // Doesn't make sense on this kind of RPC. 22 | return errors.New("Cannot set filter on admin operations.") 23 | } 24 | 25 | // SetFamilies always returns an error. 26 | func (to *tableOp) SetFamilies(map[string][]string) error { 27 | // Doesn't make sense on this kind of RPC. 28 | return errors.New("Cannot set families on admin operations.") 29 | } 30 | -------------------------------------------------------------------------------- /libs/gohbase/metacache_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 The GoHBase Authors. All rights reserved. 2 | // This file is part of GoHBase. 3 | // Use of this source code is governed by the Apache License 2.0 4 | // that can be found in the COPYING file. 5 | 6 | package gohbase 7 | 8 | import ( 9 | "reflect" 10 | "testing" 11 | 12 | "bfs/libs/gohbase/conf" 13 | "bfs/libs/gohbase/region" 14 | "bfs/libs/gohbase/regioninfo" 15 | "time" 16 | ) 17 | 18 | func TestMetaCache(t *testing.T) { 19 | client := newClient(standardClient, conf.NewConf([]string{"~invalid.quorum~"}, "", "", "", 30*time.Second, 0, 0, 0)) // We shouldn't connect to ZK. 20 | 21 | reg := client.getRegionFromCache([]byte("test"), []byte("theKey")) 22 | if reg != nil { 23 | t.Errorf("Found region %#v even though the cache was empty?!", reg) 24 | } 25 | 26 | // Inject an entry in the cache. This entry covers the entire key range. 27 | wholeTable := ®ioninfo.Info{ 28 | Table: []byte("test"), 29 | RegionName: []byte("test,,1234567890042.56f833d5569a27c7a43fbf547b4924a4."), 30 | StopKey: []byte(""), 31 | } 32 | regClient := ®ion.Client{} 33 | client.regions.put(wholeTable.RegionName, wholeTable) 34 | client.clients.put(wholeTable, regClient) 35 | 36 | reg = client.getRegionFromCache([]byte("test"), []byte("theKey")) 37 | if !reflect.DeepEqual(reg, wholeTable) { 38 | t.Errorf("Found region %#v but expected %#v", reg, wholeTable) 39 | } 40 | reg = client.getRegionFromCache([]byte("test"), []byte("")) // edge case. 41 | if !reflect.DeepEqual(reg, wholeTable) { 42 | t.Errorf("Found region %#v but expected %#v", reg, wholeTable) 43 | } 44 | 45 | // Clear our client. 46 | client = newClient(standardClient, conf.NewConf([]string{"~invalid.quorum~"}, "", "", "", 30*time.Second, 0, 0, 0)) 47 | 48 | // Inject 3 entries in the cache. 49 | region1 := ®ioninfo.Info{ 50 | Table: []byte("test"), 51 | RegionName: []byte("test,,1234567890042.56f833d5569a27c7a43fbf547b4924a4."), 52 | StopKey: []byte("foo"), 53 | } 54 | client.regions.put(region1.RegionName, region1) 55 | client.clients.put(region1, regClient) 56 | 57 | region2 := ®ioninfo.Info{ 58 | Table: []byte("test"), 59 | RegionName: []byte("test,foo,1234567890042.56f833d5569a27c7a43fbf547b4924a4."), 60 | StopKey: []byte("gohbase"), 61 | } 62 | client.regions.put(region2.RegionName, region2) 63 | client.clients.put(region2, regClient) 64 | 65 | region3 := ®ioninfo.Info{ 66 | Table: []byte("test"), 67 | RegionName: []byte("test,gohbase,1234567890042.56f833d5569a27c7a43fbf547b4924a4."), 68 | StopKey: []byte(""), 69 | } 70 | client.regions.put(region3.RegionName, region3) 71 | client.clients.put(region3, regClient) 72 | 73 | testcases := []struct { 74 | key string 75 | reg *regioninfo.Info 76 | }{ 77 | {key: "theKey", reg: region3}, 78 | {key: "", reg: region1}, 79 | {key: "bar", reg: region1}, 80 | {key: "fon\xFF", reg: region1}, 81 | {key: "foo", reg: region2}, 82 | {key: "foo\x00", reg: region2}, 83 | {key: "gohbase", reg: region3}, 84 | } 85 | for i, testcase := range testcases { 86 | reg = client.getRegionFromCache([]byte("test"), []byte(testcase.key)) 87 | if !reflect.DeepEqual(reg, testcase.reg) { 88 | t.Errorf("[#%d] Found region %#v but expected %#v", i, reg, testcase.reg) 89 | } 90 | } 91 | 92 | // Change the last region (maybe it got split). 93 | region3 = ®ioninfo.Info{ 94 | Table: []byte("test"), 95 | RegionName: []byte("test,gohbase,1234567890042.56f833d5569a27c7a43fbf547b4924a4."), 96 | StopKey: []byte("zab"), 97 | } 98 | client.regions.put(region3.RegionName, region3) 99 | client.clients.put(region3, regClient) 100 | 101 | reg = client.getRegionFromCache([]byte("test"), []byte("theKey")) 102 | if !reflect.DeepEqual(reg, region3) { 103 | t.Errorf("Found region %#v but expected %#v", reg, region3) 104 | } 105 | reg = client.getRegionFromCache([]byte("test"), []byte("zoo")) 106 | if reg != nil { 107 | t.Errorf("Shouldn't have found any region yet found %#v", reg) 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /libs/gohbase/pb/Cell.proto: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | // Cell and KeyValue protos 20 | 21 | package pb; 22 | option java_package = "org.apache.hadoop.hbase.protobuf.generated"; 23 | option java_outer_classname = "CellProtos"; 24 | option java_generate_equals_and_hash = true; 25 | option optimize_for = SPEED; 26 | 27 | /** 28 | * The type of the key in a Cell 29 | */ 30 | enum CellType { 31 | MINIMUM = 0; 32 | PUT = 4; 33 | 34 | DELETE = 8; 35 | DELETE_COLUMN = 12; 36 | DELETE_FAMILY = 14; 37 | 38 | // MAXIMUM is used when searching; you look from maximum on down. 39 | MAXIMUM = 255; 40 | } 41 | 42 | /** 43 | * Protocol buffer version of Cell. 44 | */ 45 | message Cell { 46 | optional bytes row = 1; 47 | optional bytes family = 2; 48 | optional bytes qualifier = 3; 49 | optional uint64 timestamp = 4; 50 | optional CellType cell_type = 5; 51 | optional bytes value = 6; 52 | optional bytes tags = 7; 53 | } 54 | 55 | /** 56 | * Protocol buffer version of KeyValue. 57 | * It doesn't have those transient parameters 58 | */ 59 | message KeyValue { 60 | required bytes row = 1; 61 | required bytes family = 2; 62 | required bytes qualifier = 3; 63 | optional uint64 timestamp = 4; 64 | optional CellType key_type = 5; 65 | optional bytes value = 6; 66 | optional bytes tags = 7; 67 | } 68 | -------------------------------------------------------------------------------- /libs/gohbase/pb/ClusterId.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. 2 | // source: ClusterId.proto 3 | // DO NOT EDIT! 4 | 5 | package pb 6 | 7 | import proto "github.com/golang/protobuf/proto" 8 | import math "math" 9 | 10 | // Reference imports to suppress errors if they are not otherwise used. 11 | var _ = proto.Marshal 12 | var _ = math.Inf 13 | 14 | // * 15 | // Content of the '/hbase/hbaseid', cluster id, znode. 16 | // Also cluster of the ${HBASE_ROOTDIR}/hbase.id file. 17 | type ClusterId struct { 18 | // This is the cluster id, a uuid as a String 19 | ClusterId *string `protobuf:"bytes,1,req,name=cluster_id" json:"cluster_id,omitempty"` 20 | XXX_unrecognized []byte `json:"-"` 21 | } 22 | 23 | func (m *ClusterId) Reset() { *m = ClusterId{} } 24 | func (m *ClusterId) String() string { return proto.CompactTextString(m) } 25 | func (*ClusterId) ProtoMessage() {} 26 | 27 | func (m *ClusterId) GetClusterId() string { 28 | if m != nil && m.ClusterId != nil { 29 | return *m.ClusterId 30 | } 31 | return "" 32 | } 33 | 34 | func init() { 35 | } 36 | -------------------------------------------------------------------------------- /libs/gohbase/pb/ClusterId.proto: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | // This file contains protocol buffers that are shared throughout HBase 20 | 21 | package pb; 22 | option java_package = "org.apache.hadoop.hbase.protobuf.generated"; 23 | option java_outer_classname = "ClusterIdProtos"; 24 | option java_generate_equals_and_hash = true; 25 | option optimize_for = SPEED; 26 | 27 | /** 28 | * Content of the '/hbase/hbaseid', cluster id, znode. 29 | * Also cluster of the ${HBASE_ROOTDIR}/hbase.id file. 30 | */ 31 | message ClusterId { 32 | // This is the cluster id, a uuid as a String 33 | required string cluster_id = 1; 34 | } 35 | -------------------------------------------------------------------------------- /libs/gohbase/pb/Comparator.proto: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | // This file contains protocol buffers that are used for filters 20 | 21 | package pb; 22 | option java_package = "org.apache.hadoop.hbase.protobuf.generated"; 23 | option java_outer_classname = "ComparatorProtos"; 24 | option java_generic_services = true; 25 | option java_generate_equals_and_hash = true; 26 | option optimize_for = SPEED; 27 | 28 | // This file contains protocol buffers that are used for comparators (e.g. in filters) 29 | 30 | message Comparator { 31 | required string name = 1; 32 | optional bytes serialized_comparator = 2; 33 | } 34 | 35 | message ByteArrayComparable { 36 | optional bytes value = 1; 37 | } 38 | 39 | message BinaryComparator { 40 | required ByteArrayComparable comparable = 1; 41 | } 42 | 43 | message LongComparator { 44 | required ByteArrayComparable comparable = 1; 45 | } 46 | 47 | message BinaryPrefixComparator { 48 | required ByteArrayComparable comparable = 1; 49 | } 50 | 51 | message BitComparator { 52 | required ByteArrayComparable comparable = 1; 53 | required BitwiseOp bitwise_op = 2; 54 | 55 | enum BitwiseOp { 56 | AND = 1; 57 | OR = 2; 58 | XOR = 3; 59 | } 60 | } 61 | 62 | message NullComparator { 63 | } 64 | 65 | message RegexStringComparator { 66 | required string pattern = 1; 67 | required int32 pattern_flags = 2; 68 | required string charset = 3; 69 | optional string engine = 4; 70 | } 71 | 72 | message SubstringComparator { 73 | required string substr = 1; 74 | } 75 | -------------------------------------------------------------------------------- /libs/gohbase/pb/ErrorHandling.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. 2 | // source: ErrorHandling.proto 3 | // DO NOT EDIT! 4 | 5 | package pb 6 | 7 | import proto "github.com/golang/protobuf/proto" 8 | import math "math" 9 | 10 | // Reference imports to suppress errors if they are not otherwise used. 11 | var _ = proto.Marshal 12 | var _ = math.Inf 13 | 14 | // * 15 | // Protobuf version of a java.lang.StackTraceElement 16 | // so we can serialize exceptions. 17 | type StackTraceElementMessage struct { 18 | DeclaringClass *string `protobuf:"bytes,1,opt,name=declaring_class" json:"declaring_class,omitempty"` 19 | MethodName *string `protobuf:"bytes,2,opt,name=method_name" json:"method_name,omitempty"` 20 | FileName *string `protobuf:"bytes,3,opt,name=file_name" json:"file_name,omitempty"` 21 | LineNumber *int32 `protobuf:"varint,4,opt,name=line_number" json:"line_number,omitempty"` 22 | XXX_unrecognized []byte `json:"-"` 23 | } 24 | 25 | func (m *StackTraceElementMessage) Reset() { *m = StackTraceElementMessage{} } 26 | func (m *StackTraceElementMessage) String() string { return proto.CompactTextString(m) } 27 | func (*StackTraceElementMessage) ProtoMessage() {} 28 | 29 | func (m *StackTraceElementMessage) GetDeclaringClass() string { 30 | if m != nil && m.DeclaringClass != nil { 31 | return *m.DeclaringClass 32 | } 33 | return "" 34 | } 35 | 36 | func (m *StackTraceElementMessage) GetMethodName() string { 37 | if m != nil && m.MethodName != nil { 38 | return *m.MethodName 39 | } 40 | return "" 41 | } 42 | 43 | func (m *StackTraceElementMessage) GetFileName() string { 44 | if m != nil && m.FileName != nil { 45 | return *m.FileName 46 | } 47 | return "" 48 | } 49 | 50 | func (m *StackTraceElementMessage) GetLineNumber() int32 { 51 | if m != nil && m.LineNumber != nil { 52 | return *m.LineNumber 53 | } 54 | return 0 55 | } 56 | 57 | // * 58 | // Cause of a remote failure for a generic exception. Contains 59 | // all the information for a generic exception as well as 60 | // optional info about the error for generic info passing 61 | // (which should be another protobuffed class). 62 | type GenericExceptionMessage struct { 63 | ClassName *string `protobuf:"bytes,1,opt,name=class_name" json:"class_name,omitempty"` 64 | Message *string `protobuf:"bytes,2,opt,name=message" json:"message,omitempty"` 65 | ErrorInfo []byte `protobuf:"bytes,3,opt,name=error_info" json:"error_info,omitempty"` 66 | Trace []*StackTraceElementMessage `protobuf:"bytes,4,rep,name=trace" json:"trace,omitempty"` 67 | XXX_unrecognized []byte `json:"-"` 68 | } 69 | 70 | func (m *GenericExceptionMessage) Reset() { *m = GenericExceptionMessage{} } 71 | func (m *GenericExceptionMessage) String() string { return proto.CompactTextString(m) } 72 | func (*GenericExceptionMessage) ProtoMessage() {} 73 | 74 | func (m *GenericExceptionMessage) GetClassName() string { 75 | if m != nil && m.ClassName != nil { 76 | return *m.ClassName 77 | } 78 | return "" 79 | } 80 | 81 | func (m *GenericExceptionMessage) GetMessage() string { 82 | if m != nil && m.Message != nil { 83 | return *m.Message 84 | } 85 | return "" 86 | } 87 | 88 | func (m *GenericExceptionMessage) GetErrorInfo() []byte { 89 | if m != nil { 90 | return m.ErrorInfo 91 | } 92 | return nil 93 | } 94 | 95 | func (m *GenericExceptionMessage) GetTrace() []*StackTraceElementMessage { 96 | if m != nil { 97 | return m.Trace 98 | } 99 | return nil 100 | } 101 | 102 | // * 103 | // Exception sent across the wire when a remote task needs 104 | // to notify other tasks that it failed and why 105 | type ForeignExceptionMessage struct { 106 | Source *string `protobuf:"bytes,1,opt,name=source" json:"source,omitempty"` 107 | GenericException *GenericExceptionMessage `protobuf:"bytes,2,opt,name=generic_exception" json:"generic_exception,omitempty"` 108 | XXX_unrecognized []byte `json:"-"` 109 | } 110 | 111 | func (m *ForeignExceptionMessage) Reset() { *m = ForeignExceptionMessage{} } 112 | func (m *ForeignExceptionMessage) String() string { return proto.CompactTextString(m) } 113 | func (*ForeignExceptionMessage) ProtoMessage() {} 114 | 115 | func (m *ForeignExceptionMessage) GetSource() string { 116 | if m != nil && m.Source != nil { 117 | return *m.Source 118 | } 119 | return "" 120 | } 121 | 122 | func (m *ForeignExceptionMessage) GetGenericException() *GenericExceptionMessage { 123 | if m != nil { 124 | return m.GenericException 125 | } 126 | return nil 127 | } 128 | 129 | func init() { 130 | } 131 | -------------------------------------------------------------------------------- /libs/gohbase/pb/ErrorHandling.proto: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | // This file contains protocol buffers that are used for error handling 20 | 21 | package pb; 22 | option java_package = "org.apache.hadoop.hbase.protobuf.generated"; 23 | option java_outer_classname = "ErrorHandlingProtos"; 24 | option java_generate_equals_and_hash = true; 25 | option optimize_for = SPEED; 26 | 27 | /** 28 | * Protobuf version of a java.lang.StackTraceElement 29 | * so we can serialize exceptions. 30 | */ 31 | message StackTraceElementMessage { 32 | optional string declaring_class = 1; 33 | optional string method_name = 2; 34 | optional string file_name = 3; 35 | optional int32 line_number = 4; 36 | } 37 | 38 | /** 39 | * Cause of a remote failure for a generic exception. Contains 40 | * all the information for a generic exception as well as 41 | * optional info about the error for generic info passing 42 | * (which should be another protobuffed class). 43 | */ 44 | message GenericExceptionMessage { 45 | optional string class_name = 1; 46 | optional string message = 2; 47 | optional bytes error_info = 3; 48 | repeated StackTraceElementMessage trace = 4; 49 | } 50 | 51 | /** 52 | * Exception sent across the wire when a remote task needs 53 | * to notify other tasks that it failed and why 54 | */ 55 | message ForeignExceptionMessage { 56 | optional string source = 1; 57 | optional GenericExceptionMessage generic_exception = 2; 58 | } 59 | -------------------------------------------------------------------------------- /libs/gohbase/pb/FS.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. 2 | // source: FS.proto 3 | // DO NOT EDIT! 4 | 5 | package pb 6 | 7 | import proto "github.com/golang/protobuf/proto" 8 | import math "math" 9 | 10 | // Reference imports to suppress errors if they are not otherwise used. 11 | var _ = proto.Marshal 12 | var _ = math.Inf 13 | 14 | type Reference_Range int32 15 | 16 | const ( 17 | Reference_TOP Reference_Range = 0 18 | Reference_BOTTOM Reference_Range = 1 19 | ) 20 | 21 | var Reference_Range_name = map[int32]string{ 22 | 0: "TOP", 23 | 1: "BOTTOM", 24 | } 25 | var Reference_Range_value = map[string]int32{ 26 | "TOP": 0, 27 | "BOTTOM": 1, 28 | } 29 | 30 | func (x Reference_Range) Enum() *Reference_Range { 31 | p := new(Reference_Range) 32 | *p = x 33 | return p 34 | } 35 | func (x Reference_Range) String() string { 36 | return proto.EnumName(Reference_Range_name, int32(x)) 37 | } 38 | func (x *Reference_Range) UnmarshalJSON(data []byte) error { 39 | value, err := proto.UnmarshalJSONEnum(Reference_Range_value, data, "Reference_Range") 40 | if err != nil { 41 | return err 42 | } 43 | *x = Reference_Range(value) 44 | return nil 45 | } 46 | 47 | // * 48 | // The ${HBASE_ROOTDIR}/hbase.version file content 49 | type HBaseVersionFileContent struct { 50 | Version *string `protobuf:"bytes,1,req,name=version" json:"version,omitempty"` 51 | XXX_unrecognized []byte `json:"-"` 52 | } 53 | 54 | func (m *HBaseVersionFileContent) Reset() { *m = HBaseVersionFileContent{} } 55 | func (m *HBaseVersionFileContent) String() string { return proto.CompactTextString(m) } 56 | func (*HBaseVersionFileContent) ProtoMessage() {} 57 | 58 | func (m *HBaseVersionFileContent) GetVersion() string { 59 | if m != nil && m.Version != nil { 60 | return *m.Version 61 | } 62 | return "" 63 | } 64 | 65 | // * 66 | // Reference file content used when we split an hfile under a region. 67 | type Reference struct { 68 | Splitkey []byte `protobuf:"bytes,1,req,name=splitkey" json:"splitkey,omitempty"` 69 | Range *Reference_Range `protobuf:"varint,2,req,name=range,enum=pb.Reference_Range" json:"range,omitempty"` 70 | XXX_unrecognized []byte `json:"-"` 71 | } 72 | 73 | func (m *Reference) Reset() { *m = Reference{} } 74 | func (m *Reference) String() string { return proto.CompactTextString(m) } 75 | func (*Reference) ProtoMessage() {} 76 | 77 | func (m *Reference) GetSplitkey() []byte { 78 | if m != nil { 79 | return m.Splitkey 80 | } 81 | return nil 82 | } 83 | 84 | func (m *Reference) GetRange() Reference_Range { 85 | if m != nil && m.Range != nil { 86 | return *m.Range 87 | } 88 | return Reference_TOP 89 | } 90 | 91 | func init() { 92 | proto.RegisterEnum("pb.Reference_Range", Reference_Range_name, Reference_Range_value) 93 | } 94 | -------------------------------------------------------------------------------- /libs/gohbase/pb/FS.proto: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | // This file contains protocol buffers that are written into the filesystem 20 | 21 | package pb; 22 | option java_package = "org.apache.hadoop.hbase.protobuf.generated"; 23 | option java_outer_classname = "FSProtos"; 24 | option java_generate_equals_and_hash = true; 25 | option optimize_for = SPEED; 26 | 27 | /** 28 | * The ${HBASE_ROOTDIR}/hbase.version file content 29 | */ 30 | message HBaseVersionFileContent { 31 | required string version = 1; 32 | } 33 | 34 | /** 35 | * Reference file content used when we split an hfile under a region. 36 | */ 37 | message Reference { 38 | required bytes splitkey = 1; 39 | enum Range { 40 | TOP = 0; 41 | BOTTOM = 1; 42 | } 43 | required Range range = 2; 44 | } 45 | 46 | -------------------------------------------------------------------------------- /libs/gohbase/pb/Filter.proto: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | // This file contains protocol buffers that are used for filters 20 | 21 | package pb; 22 | option java_package = "org.apache.hadoop.hbase.protobuf.generated"; 23 | option java_outer_classname = "FilterProtos"; 24 | option java_generic_services = true; 25 | option java_generate_equals_and_hash = true; 26 | option optimize_for = SPEED; 27 | 28 | import "HBase.proto"; 29 | import "Comparator.proto"; 30 | 31 | message Filter { 32 | required string name = 1; 33 | optional bytes serialized_filter = 2; 34 | } 35 | 36 | message ColumnCountGetFilter { 37 | required int32 limit = 1; 38 | } 39 | 40 | message ColumnPaginationFilter { 41 | required int32 limit = 1; 42 | optional int32 offset = 2; 43 | optional bytes column_offset = 3; 44 | } 45 | 46 | message ColumnPrefixFilter { 47 | required bytes prefix = 1; 48 | } 49 | 50 | message ColumnRangeFilter { 51 | optional bytes min_column = 1; 52 | optional bool min_column_inclusive = 2; 53 | optional bytes max_column = 3; 54 | optional bool max_column_inclusive = 4; 55 | } 56 | 57 | message CompareFilter { 58 | required CompareType compare_op = 1; 59 | optional Comparator comparator = 2; 60 | } 61 | 62 | message DependentColumnFilter { 63 | required CompareFilter compare_filter = 1; 64 | optional bytes column_family = 2; 65 | optional bytes column_qualifier = 3; 66 | optional bool drop_dependent_column = 4; 67 | } 68 | 69 | message FamilyFilter { 70 | required CompareFilter compare_filter = 1; 71 | } 72 | 73 | message FilterList { 74 | required Operator operator = 1; 75 | repeated Filter filters = 2; 76 | 77 | enum Operator { 78 | MUST_PASS_ALL = 1; 79 | MUST_PASS_ONE = 2; 80 | } 81 | } 82 | 83 | message FilterWrapper { 84 | required Filter filter = 1; 85 | } 86 | 87 | message FirstKeyOnlyFilter { 88 | } 89 | 90 | message FirstKeyValueMatchingQualifiersFilter { 91 | repeated bytes qualifiers = 1; 92 | } 93 | 94 | message FuzzyRowFilter { 95 | repeated BytesBytesPair fuzzy_keys_data = 1; 96 | } 97 | 98 | message InclusiveStopFilter { 99 | optional bytes stop_row_key = 1; 100 | } 101 | 102 | message KeyOnlyFilter { 103 | required bool len_as_val = 1; 104 | } 105 | 106 | message MultipleColumnPrefixFilter { 107 | repeated bytes sorted_prefixes = 1; 108 | } 109 | 110 | message PageFilter { 111 | required int64 page_size = 1; 112 | } 113 | 114 | message PrefixFilter { 115 | optional bytes prefix = 1; 116 | } 117 | 118 | message QualifierFilter { 119 | required CompareFilter compare_filter = 1; 120 | } 121 | 122 | message RandomRowFilter { 123 | required float chance = 1; 124 | } 125 | 126 | message RowFilter { 127 | required CompareFilter compare_filter = 1; 128 | } 129 | 130 | message SingleColumnValueExcludeFilter { 131 | required SingleColumnValueFilter single_column_value_filter = 1; 132 | } 133 | 134 | message SingleColumnValueFilter { 135 | optional bytes column_family = 1; 136 | optional bytes column_qualifier = 2; 137 | required CompareType compare_op = 3; 138 | required Comparator comparator = 4; 139 | optional bool filter_if_missing = 5; 140 | optional bool latest_version_only = 6; 141 | } 142 | 143 | message SkipFilter { 144 | required Filter filter = 1; 145 | } 146 | 147 | message TimestampsFilter { 148 | repeated int64 timestamps = 1 [packed=true]; 149 | } 150 | 151 | message ValueFilter { 152 | required CompareFilter compare_filter = 1; 153 | } 154 | 155 | message WhileMatchFilter { 156 | required Filter filter = 1; 157 | } 158 | message FilterAllFilter { 159 | } 160 | 161 | message RowRange { 162 | optional bytes start_row = 1; 163 | optional bool start_row_inclusive = 2; 164 | optional bytes stop_row = 3; 165 | optional bool stop_row_inclusive =4; 166 | } 167 | 168 | message MultiRowRangeFilter { 169 | repeated RowRange row_range_list = 1; 170 | } -------------------------------------------------------------------------------- /libs/gohbase/pb/MultiRowMutation.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. 2 | // source: MultiRowMutation.proto 3 | // DO NOT EDIT! 4 | 5 | package pb 6 | 7 | import proto "github.com/golang/protobuf/proto" 8 | import math "math" 9 | 10 | // Reference imports to suppress errors if they are not otherwise used. 11 | var _ = proto.Marshal 12 | var _ = math.Inf 13 | 14 | type MultiRowMutationProcessorRequest struct { 15 | XXX_unrecognized []byte `json:"-"` 16 | } 17 | 18 | func (m *MultiRowMutationProcessorRequest) Reset() { *m = MultiRowMutationProcessorRequest{} } 19 | func (m *MultiRowMutationProcessorRequest) String() string { return proto.CompactTextString(m) } 20 | func (*MultiRowMutationProcessorRequest) ProtoMessage() {} 21 | 22 | type MultiRowMutationProcessorResponse struct { 23 | XXX_unrecognized []byte `json:"-"` 24 | } 25 | 26 | func (m *MultiRowMutationProcessorResponse) Reset() { *m = MultiRowMutationProcessorResponse{} } 27 | func (m *MultiRowMutationProcessorResponse) String() string { return proto.CompactTextString(m) } 28 | func (*MultiRowMutationProcessorResponse) ProtoMessage() {} 29 | 30 | type MutateRowsRequest struct { 31 | MutationRequest []*MutationProto `protobuf:"bytes,1,rep,name=mutation_request" json:"mutation_request,omitempty"` 32 | NonceGroup *uint64 `protobuf:"varint,2,opt,name=nonce_group" json:"nonce_group,omitempty"` 33 | Nonce *uint64 `protobuf:"varint,3,opt,name=nonce" json:"nonce,omitempty"` 34 | XXX_unrecognized []byte `json:"-"` 35 | } 36 | 37 | func (m *MutateRowsRequest) Reset() { *m = MutateRowsRequest{} } 38 | func (m *MutateRowsRequest) String() string { return proto.CompactTextString(m) } 39 | func (*MutateRowsRequest) ProtoMessage() {} 40 | 41 | func (m *MutateRowsRequest) GetMutationRequest() []*MutationProto { 42 | if m != nil { 43 | return m.MutationRequest 44 | } 45 | return nil 46 | } 47 | 48 | func (m *MutateRowsRequest) GetNonceGroup() uint64 { 49 | if m != nil && m.NonceGroup != nil { 50 | return *m.NonceGroup 51 | } 52 | return 0 53 | } 54 | 55 | func (m *MutateRowsRequest) GetNonce() uint64 { 56 | if m != nil && m.Nonce != nil { 57 | return *m.Nonce 58 | } 59 | return 0 60 | } 61 | 62 | type MutateRowsResponse struct { 63 | XXX_unrecognized []byte `json:"-"` 64 | } 65 | 66 | func (m *MutateRowsResponse) Reset() { *m = MutateRowsResponse{} } 67 | func (m *MutateRowsResponse) String() string { return proto.CompactTextString(m) } 68 | func (*MutateRowsResponse) ProtoMessage() {} 69 | 70 | func init() { 71 | } 72 | -------------------------------------------------------------------------------- /libs/gohbase/pb/MultiRowMutation.proto: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | import "Client.proto"; 19 | package pb; 20 | option java_package = "org.apache.hadoop.hbase.protobuf.generated"; 21 | option java_outer_classname = "MultiRowMutationProtos"; 22 | option java_generate_equals_and_hash = true; 23 | option java_generic_services = true; 24 | option optimize_for = SPEED; 25 | 26 | message MultiRowMutationProcessorRequest{ 27 | } 28 | 29 | message MultiRowMutationProcessorResponse{ 30 | } 31 | 32 | message MutateRowsRequest { 33 | repeated MutationProto mutation_request = 1; 34 | optional uint64 nonce_group = 2; 35 | optional uint64 nonce = 3; 36 | } 37 | 38 | message MutateRowsResponse { 39 | } 40 | 41 | service MultiRowMutationService { 42 | rpc MutateRows(MutateRowsRequest) 43 | returns(MutateRowsResponse); 44 | } -------------------------------------------------------------------------------- /libs/gohbase/pb/Quota.proto: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package pb; 19 | option java_package = "org.apache.hadoop.hbase.protobuf.generated"; 20 | option java_outer_classname = "QuotaProtos"; 21 | option java_generic_services = true; 22 | option java_generate_equals_and_hash = true; 23 | option optimize_for = SPEED; 24 | 25 | import "HBase.proto"; 26 | 27 | enum QuotaScope { 28 | CLUSTER = 1; 29 | MACHINE = 2; 30 | } 31 | 32 | message TimedQuota { 33 | required TimeUnit time_unit = 1; 34 | optional uint64 soft_limit = 2; 35 | optional float share = 3; 36 | optional QuotaScope scope = 4 [default = MACHINE]; 37 | } 38 | 39 | enum ThrottleType { 40 | REQUEST_NUMBER = 1; 41 | REQUEST_SIZE = 2; 42 | WRITE_NUMBER = 3; 43 | WRITE_SIZE = 4; 44 | READ_NUMBER = 5; 45 | READ_SIZE = 6; 46 | } 47 | 48 | message Throttle { 49 | optional TimedQuota req_num = 1; 50 | optional TimedQuota req_size = 2; 51 | 52 | optional TimedQuota write_num = 3; 53 | optional TimedQuota write_size = 4; 54 | 55 | optional TimedQuota read_num = 5; 56 | optional TimedQuota read_size = 6; 57 | } 58 | 59 | message ThrottleRequest { 60 | optional ThrottleType type = 1; 61 | optional TimedQuota timed_quota = 2; 62 | } 63 | 64 | enum QuotaType { 65 | THROTTLE = 1; 66 | } 67 | 68 | message Quotas { 69 | optional bool bypass_globals = 1 [default = false]; 70 | optional Throttle throttle = 2; 71 | } 72 | 73 | message QuotaUsage { 74 | } 75 | -------------------------------------------------------------------------------- /libs/gohbase/pb/README.txt: -------------------------------------------------------------------------------- 1 | These are the protobuf definition files used by GoHBase. 2 | They were copied from HBase (see under hbase-protocol/src/main/protobuf). 3 | 4 | The following changes were made to those files: 5 | - the package name was changed to "pb". 6 | 7 | The files in this directory are also subject to the Apache License 2.0 and 8 | are copyright of the Apache Software Foundation. 9 | -------------------------------------------------------------------------------- /libs/gohbase/pb/Tracing.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. 2 | // source: Tracing.proto 3 | // DO NOT EDIT! 4 | 5 | package pb 6 | 7 | import proto "github.com/golang/protobuf/proto" 8 | import math "math" 9 | 10 | // Reference imports to suppress errors if they are not otherwise used. 11 | var _ = proto.Marshal 12 | var _ = math.Inf 13 | 14 | // Used to pass through the information necessary to continue 15 | // a trace after an RPC is made. All we need is the traceid 16 | // (so we know the overarching trace this message is a part of), and 17 | // the id of the current span when this message was sent, so we know 18 | // what span caused the new span we will create when this message is received. 19 | type RPCTInfo struct { 20 | TraceId *int64 `protobuf:"varint,1,opt,name=trace_id" json:"trace_id,omitempty"` 21 | ParentId *int64 `protobuf:"varint,2,opt,name=parent_id" json:"parent_id,omitempty"` 22 | XXX_unrecognized []byte `json:"-"` 23 | } 24 | 25 | func (m *RPCTInfo) Reset() { *m = RPCTInfo{} } 26 | func (m *RPCTInfo) String() string { return proto.CompactTextString(m) } 27 | func (*RPCTInfo) ProtoMessage() {} 28 | 29 | func (m *RPCTInfo) GetTraceId() int64 { 30 | if m != nil && m.TraceId != nil { 31 | return *m.TraceId 32 | } 33 | return 0 34 | } 35 | 36 | func (m *RPCTInfo) GetParentId() int64 { 37 | if m != nil && m.ParentId != nil { 38 | return *m.ParentId 39 | } 40 | return 0 41 | } 42 | 43 | func init() { 44 | } 45 | -------------------------------------------------------------------------------- /libs/gohbase/pb/Tracing.proto: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package pb; 20 | option java_package = "org.apache.hadoop.hbase.protobuf.generated"; 21 | option java_outer_classname = "TracingProtos"; 22 | option java_generate_equals_and_hash = true; 23 | option optimize_for = SPEED; 24 | 25 | //Used to pass through the information necessary to continue 26 | //a trace after an RPC is made. All we need is the traceid 27 | //(so we know the overarching trace this message is a part of), and 28 | //the id of the current span when this message was sent, so we know 29 | //what span caused the new span we will create when this message is received. 30 | message RPCTInfo { 31 | optional int64 trace_id = 1; 32 | optional int64 parent_id = 2; 33 | } 34 | -------------------------------------------------------------------------------- /libs/gohbase/pb/generate.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 The GoHBase Authors. All rights reserved. 2 | // This file is part of GoHBase. 3 | // Use of this source code is governed by the Apache License 2.0 4 | // that can be found in the COPYING file. 5 | 6 | package pb 7 | 8 | //go:generate sh -c "protoc --go_out=. *.proto" 9 | -------------------------------------------------------------------------------- /libs/gohbase/pb/marshal.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 The GoHBase Authors. All rights reserved. 2 | // This file is part of GoHBase. 3 | // Use of this source code is governed by the Apache License 2.0 4 | // that can be found in the COPYING file. 5 | 6 | package pb 7 | 8 | import ( 9 | "github.com/golang/protobuf/proto" 10 | ) 11 | 12 | // MustMarshal is like proto.Marshal except it panic()'s if the protobuf 13 | // couldn't be serialized. 14 | func MustMarshal(pb proto.Message) []byte { 15 | b, err := proto.Marshal(pb) 16 | if err != nil { 17 | panic(err) 18 | } 19 | return b 20 | } 21 | -------------------------------------------------------------------------------- /libs/gohbase/region/client_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 The GoHBase Authors. All rights reserved. 2 | // This file is part of GoHBase. 3 | // Use of this source code is governed by the Apache License 2.0 4 | // that can be found in the COPYING file. 5 | 6 | package region 7 | 8 | import ( 9 | "fmt" 10 | "testing" 11 | ) 12 | 13 | func TestErrors(t *testing.T) { 14 | ue := UnrecoverableError{fmt.Errorf("oops")} 15 | if ue.Error() != "oops" { 16 | t.Errorf("Wrong error message. Got %q, wanted %q", ue, "oops") 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /libs/gohbase/test/test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2015 The GoHBase Authors. All rights reserved. 2 | // This file is part of GoHBase. 3 | // Use of this source code is governed by the Apache License 2.0 4 | // that can be found in the COPYING file. 5 | 6 | package test 7 | 8 | import ( 9 | "bytes" 10 | "context" 11 | "errors" 12 | "io" 13 | "os" 14 | "os/exec" 15 | "path" 16 | "strings" 17 | 18 | "bfs/libs/gohbase" 19 | "bfs/libs/gohbase/hrpc" 20 | ) 21 | 22 | // This error is returned when the HBASE_HOME environment variable is unset 23 | var errHomeUnset = errors.New("Environment variable HBASE_HOME is not set") 24 | 25 | // getShellCmd returns a new shell subprocess (already started) along with its 26 | // stdin 27 | func getShellCmd() (*exec.Cmd, io.WriteCloser, error) { 28 | hbaseHome := os.Getenv("HBASE_HOME") 29 | if len(hbaseHome) == 0 { 30 | return nil, nil, errHomeUnset 31 | } 32 | hbaseShell := path.Join(hbaseHome, "bin", "hbase") 33 | cmd := exec.Command(hbaseShell, "shell") 34 | stdin, err := cmd.StdinPipe() 35 | if err != nil { 36 | return nil, nil, err 37 | } 38 | 39 | err = cmd.Start() 40 | if err != nil { 41 | stdin.Close() 42 | return nil, nil, err 43 | } 44 | return cmd, stdin, nil 45 | } 46 | 47 | // CreateTable finds the HBase shell via the HBASE_HOME environment variable, 48 | // and creates the given table with the given families 49 | func CreateTable(host, table string, cFamilies []string) error { 50 | // If the table exists, delete it 51 | DeleteTable(host, table) 52 | // Don't check the error, since one will be returned if the table doesn't 53 | // exist 54 | 55 | cmd, stdin, err := getShellCmd() 56 | if err != nil { 57 | return err 58 | } 59 | 60 | var buf bytes.Buffer 61 | buf.WriteString("create '" + table + "'") 62 | 63 | for _, f := range cFamilies { 64 | buf.WriteString(", '") 65 | buf.WriteString(f) 66 | buf.WriteString("'") 67 | } 68 | buf.WriteString("\n") 69 | 70 | stdin.Write(buf.Bytes()) 71 | stdin.Write([]byte("exit\n")) 72 | 73 | return cmd.Wait() 74 | } 75 | 76 | // DeleteTable finds the HBase shell via the HBASE_HOME environment variable, 77 | // and disables and drops the given table 78 | func DeleteTable(host, table string) error { 79 | // TODO: We leak this client. 80 | ac := gohbase.NewAdminClient(host) 81 | dit := hrpc.NewDisableTable(context.Background(), []byte(table)) 82 | _, err := ac.DisableTable(dit) 83 | if err != nil { 84 | if !strings.Contains(err.Error(), "TableNotEnabledException") { 85 | return err 86 | } 87 | } 88 | 89 | det := hrpc.NewDeleteTable(context.Background(), []byte(table)) 90 | _, err = ac.DeleteTable(det) 91 | if err != nil { 92 | return err 93 | } 94 | return nil 95 | } 96 | 97 | // LaunchRegionServers uses the script local-regionservers.sh to create new 98 | // RegionServers. Fails silently if server already exists. 99 | // Ex. LaunchRegions([]string{"2", "3"}) launches two servers with id=2,3 100 | func LaunchRegionServers(servers []string) { 101 | hh := os.Getenv("HBASE_HOME") 102 | servers = append([]string{"start"}, servers...) 103 | exec.Command(hh+"/bin/local-regionservers.sh", servers...).Run() 104 | } 105 | 106 | // StopRegionServers uses the script local-regionservers.sh to stop existing 107 | // RegionServers. Fails silently if server isn't running. 108 | func StopRegionServers(servers []string) { 109 | hh := os.Getenv("HBASE_HOME") 110 | servers = append([]string{"stop"}, servers...) 111 | exec.Command(hh+"/bin/local-regionservers.sh", servers...).Run() 112 | } 113 | -------------------------------------------------------------------------------- /libs/gohbase/tools/test_get/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "strings" 8 | "time" 9 | 10 | "bfs/libs/gohbase" 11 | "bfs/libs/gohbase/conf" 12 | "bfs/libs/gohbase/hrpc" 13 | ) 14 | 15 | var ( 16 | zkStr string 17 | table string 18 | key string 19 | ) 20 | 21 | func init() { 22 | flag.StringVar(&zkStr, "zk", "", ", joined zk hosts") 23 | flag.StringVar(&table, "table", "", "table name") 24 | flag.StringVar(&key, "key", "", "key to get") 25 | } 26 | 27 | func main() { 28 | flag.Parse() 29 | fmt.Printf("do test_get for zk(%s), table(%s), key:(%s)\n", zkStr, table, key) 30 | var ( 31 | get *hrpc.Get 32 | err error 33 | res *hrpc.Result 34 | st, et int64 35 | ) 36 | zks := strings.Split(zkStr, ",") 37 | c := gohbase.NewClient(conf.NewConf(zks, "", "", "", 30*time.Second, 0, 0, 0)) 38 | if c == nil { 39 | fmt.Printf("new client get nil client for zks: (%v)\n", zks) 40 | return 41 | } 42 | if get, err = hrpc.NewGetStr(context.Background(), table, key); err != nil { 43 | fmt.Printf("new get met error: (%v)\n", err) 44 | return 45 | } 46 | st = time.Now().UnixNano() 47 | if res, err = c.Get(get); err != nil { 48 | fmt.Printf("get met error: (%v)\n", err) 49 | return 50 | } else { 51 | for _, cell := range res.Cells { 52 | fmt.Printf("%s-%s-%s: %s;", string(cell.Row), string(cell.Family), string(cell.Qualifier), string(cell.Value)) 53 | fmt.Println("") 54 | } 55 | } 56 | et = time.Now().UnixNano() 57 | fmt.Printf("get (%s) from (%s) cost %d ns (%d ms)\n", table, key, et-st, (et-st)/1000000) 58 | 59 | for _, n := range []int{10, 1000} { 60 | st = time.Now().UnixNano() 61 | for i := 0; i < n; i++ { 62 | if get, err = hrpc.NewGetStr(context.Background(), table, key); err != nil { 63 | fmt.Printf("new get met error: (%v)\n", err) 64 | return 65 | } 66 | if res, err = c.Get(get); err != nil { 67 | fmt.Printf("get met error: (%v)\n", err) 68 | return 69 | } 70 | } 71 | et = time.Now().UnixNano() 72 | fmt.Printf("get (%s) from (%s) for %d times cost %d ns (%d ms)\n", table, key, n, et-st, (et-st)/1000000) 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /libs/gohbase/tools/test_get_dapper/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strings" 7 | "time" 8 | 9 | "bfs/libs/gohbase" 10 | "bfs/libs/gohbase/conf" 11 | "bfs/libs/gohbase/hrpc" 12 | ) 13 | 14 | var ( 15 | zkStr string = "172.18.4.117:2181,172.18.4.118:2181,172.18.4.119:2181" 16 | testTable string = "test" 17 | spanTable string = "dapper_origin_v1" 18 | testKey string = "test" 19 | spanKey string = "13794899398466741090" 20 | tables = []string{testTable, spanTable} 21 | keys = []string{testKey, spanKey} 22 | ) 23 | 24 | func main() { 25 | var ( 26 | get *hrpc.Get 27 | err error 28 | res *hrpc.Result 29 | st, et int64 30 | ) 31 | zks := strings.Split(zkStr, ",") 32 | c := gohbase.NewClient(conf.NewConf(zks, "", "", "", 30*time.Second, 0, 0, 0)) 33 | if c == nil { 34 | fmt.Printf("new client get nil client for zks: (%v)\n", zks) 35 | return 36 | } 37 | var counter = 0 38 | for { 39 | time.Sleep(1 * time.Second) 40 | var i int = 0 41 | for ; i < 2; i++ { 42 | var key = keys[i] 43 | var table = tables[i] 44 | if get, err = hrpc.NewGetStr(context.Background(), table, key); err != nil { 45 | fmt.Printf("new get met error: (%v)\n", err) 46 | continue 47 | } 48 | st = time.Now().UnixNano() 49 | if res, err = c.Get(get); err != nil { 50 | fmt.Printf("get met error: (%v)\n", err) 51 | continue 52 | } else { 53 | for _, cell := range res.Cells { 54 | fmt.Sprintf("%s-%s-%s: %s;", string(cell.Row), string(cell.Family), string(cell.Qualifier), string(cell.Value)) 55 | } 56 | } 57 | et = time.Now().UnixNano() 58 | if (et-st)/1000000000 > 1 { 59 | fmt.Printf("get (%s) from (%s) cost %d ns (%d ms)\n", table, key, et-st, (et-st)/1000000) 60 | } 61 | } 62 | counter += 1 63 | if counter > 30 { 64 | fmt.Printf("time: (%v), done (%d) loops\n", time.Now(), counter) 65 | counter = 0 66 | } 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /libs/memcache/gomemcache/memcache/errors.go: -------------------------------------------------------------------------------- 1 | package memcache 2 | 3 | import ( 4 | "errors" 5 | ) 6 | 7 | var ( 8 | // ErrNotFound not found 9 | ErrNotFound = errors.New("gomemcache: key not found") 10 | // ErrExists exists 11 | ErrExists = errors.New("gomemcache: key exists") 12 | // ErrNotStored not stored 13 | ErrNotStored = errors.New("gomemcache: key not stored") 14 | 15 | // ErrPoolExhausted is returned from a pool connection method (Store, Get, 16 | // Delete, IncrDecr, Err) when the maximum number of database connections 17 | // in the pool has been reached. 18 | ErrPoolExhausted = errors.New("gomemcache: connection pool exhausted") 19 | // ErrPoolClosed pool closed 20 | ErrPoolClosed = errors.New("gomemcache: connection pool closed") 21 | // ErrConnClosed conn closed 22 | ErrConnClosed = errors.New("gomemcache: connection closed") 23 | ) 24 | -------------------------------------------------------------------------------- /libs/memcache/gomemcache/memcache/memcache.go: -------------------------------------------------------------------------------- 1 | package memcache 2 | 3 | // Error represents an error returned in a command reply. 4 | type Error string 5 | 6 | func (err Error) Error() string { return string(err) } 7 | 8 | // Reply is an reply to be got or stored in a memcached server. 9 | type Reply struct { 10 | // Key is the Item's key (250 bytes maximum). 11 | Key string 12 | 13 | // Value is the Item's value. 14 | Value []byte 15 | 16 | // Flags are server-opaque flags whose semantics are entirely 17 | // up to the app. 18 | Flags uint32 19 | 20 | // Expiration is the cache expiration time, in seconds: either a relative 21 | // time from now (up to 1 month), or an absolute Unix epoch time. 22 | // Zero means the Item has no expiration time. 23 | Expiration int32 24 | 25 | // Compare and swap ID. 26 | Cas uint64 27 | } 28 | 29 | // Conn represents a connection to a Memcache server. 30 | // Command Reference: https://github.com/memcached/memcached/wiki/Commands 31 | type Conn interface { 32 | // Close closes the connection. 33 | Close() error 34 | 35 | // Err returns a non-nil value if the connection is broken. The returned 36 | // value is either the first non-nil value returned from the underlying 37 | // network connection or a protocol parsing error. Applications should 38 | // close broken connections. 39 | Err() error 40 | 41 | // Store sends a command to the server for store data. 42 | // cmd: set, add, replace, append, prepend, cas 43 | Store(cmd, key string, value []byte, flags uint32, timeout int32, cas uint64) error 44 | 45 | // Get sends a command to the server for gets data. 46 | // cmd: get, gets 47 | Get(cmd string, key string) (*Reply, error) 48 | 49 | // Gets sends a command to the server for gets data. 50 | // cmd: get, gets 51 | Gets(cmd string, keys ...string) ([]*Reply, error) 52 | 53 | // Touch update the expiration time on an existing key. 54 | Touch(key string, timeout int32) error 55 | 56 | // Store sends a command to the server for delete data. 57 | Delete(key string) (err error) 58 | 59 | // IncrDecr sends a command to the server for incr/decr data. 60 | // cmd: incr, decr 61 | IncrDecr(cmd string, key string, delta uint64) (uint64, error) 62 | } 63 | -------------------------------------------------------------------------------- /libs/memcache/memcache.go: -------------------------------------------------------------------------------- 1 | package memcache 2 | 3 | import ( 4 | "time" 5 | 6 | xtime "bfs/libs/time" 7 | 8 | "bfs/libs/memcache/gomemcache/memcache" 9 | ) 10 | 11 | const ( 12 | _family = "memcache" 13 | _maxTTL = 30*24*60*60 - 1 14 | ) 15 | 16 | var ( 17 | // ErrNotFound not found 18 | ErrNotFound = memcache.ErrNotFound 19 | ) 20 | 21 | // Config client settings. 22 | type Config struct { 23 | Name string // memcache name, for trace 24 | Proto string 25 | Addr string 26 | Active int // pool 27 | Idle int // pool 28 | DialTimeout xtime.Duration 29 | ReadTimeout xtime.Duration 30 | WriteTimeout xtime.Duration 31 | IdleTimeout xtime.Duration 32 | } 33 | 34 | // Conn represents a connection to a Memcache server. 35 | type Conn struct { 36 | p *Pool 37 | c memcache.Conn 38 | } 39 | 40 | // Pool memcache conn pool. 41 | type Pool struct { 42 | *memcache.Pool 43 | c *Config 44 | } 45 | 46 | // NewPool new a memcache conn pool. 47 | func NewPool(c *Config) (p *Pool) { 48 | p = &Pool{c: c} 49 | cnop := memcache.DialConnectTimeout(time.Duration(c.DialTimeout)) 50 | rdop := memcache.DialReadTimeout(time.Duration(c.ReadTimeout)) 51 | wrop := memcache.DialWriteTimeout(time.Duration(c.WriteTimeout)) 52 | p.Pool = memcache.NewPool(func() (memcache.Conn, error) { 53 | return memcache.Dial(c.Proto, c.Addr, cnop, rdop, wrop) 54 | }, c.Idle) 55 | p.IdleTimeout = time.Duration(c.IdleTimeout) 56 | p.MaxActive = c.Active 57 | return 58 | } 59 | 60 | // Get gets a connection. The application must close the returned connection. 61 | func (p *Pool) Get() *Conn { 62 | return &Conn{p: p, c: p.Pool.Get()} 63 | } 64 | 65 | // Close closes the connection. 66 | func (p *Pool) Close() error { 67 | return p.Pool.Close() 68 | } 69 | 70 | // Close closes the connection. 71 | func (c *Conn) Close() error { 72 | return c.c.Close() 73 | } 74 | 75 | // Err returns a non-nil value if the connection is broken. The returned 76 | func (c *Conn) Err() error { 77 | return c.c.Err() 78 | } 79 | 80 | // Store sends a command to the server for store data. 81 | func (c *Conn) Store(cmd, key string, value []byte, flags uint32, timeout int32, cas uint64) (err error) { 82 | if timeout > _maxTTL { 83 | timeout = _maxTTL 84 | } 85 | err = c.c.Store(cmd, key, value, flags, timeout, cas) 86 | return 87 | } 88 | 89 | // Get sends a command to the server for gets data. 90 | func (c *Conn) Get(cmd string, cb func(*memcache.Reply), keys ...string) (err error) { 91 | var ( 92 | r *memcache.Reply 93 | res []*memcache.Reply 94 | ) 95 | if res, err = c.Gets(cmd, keys...); err != nil { 96 | return 97 | } 98 | for _, r = range res { 99 | cb(r) 100 | } 101 | return 102 | } 103 | 104 | // Get2 sends a command to the server for gets data. 105 | func (c *Conn) Get2(cmd string, key string) (bs []byte, err error) { 106 | var res *memcache.Reply 107 | res, err = c.c.Get(cmd, key) 108 | if err != nil { 109 | return 110 | } 111 | bs = res.Value 112 | return 113 | } 114 | 115 | // Gets sends a command to the server for gets data. 116 | func (c *Conn) Gets(cmd string, keys ...string) (res []*memcache.Reply, err error) { 117 | 118 | res, err = c.c.Gets(cmd, keys...) 119 | return 120 | } 121 | 122 | // Touch sends a command to the server for touch expire. 123 | func (c *Conn) Touch(key string, timeout int32) (err error) { 124 | 125 | if timeout > _maxTTL { 126 | timeout = _maxTTL 127 | } 128 | 129 | err = c.c.Touch(key, timeout) 130 | return 131 | } 132 | 133 | // Delete sends a command to the server for delete data. 134 | func (c *Conn) Delete(key string) (err error) { 135 | 136 | err = c.c.Delete(key) 137 | return 138 | } 139 | 140 | // IncrDecr sends a command to the server for incr/decr data. 141 | func (c *Conn) IncrDecr(cmd string, key string, delta uint64) (res uint64, err error) { 142 | 143 | res, err = c.c.IncrDecr(cmd, key, delta) 144 | return 145 | } 146 | -------------------------------------------------------------------------------- /libs/meta/const.go: -------------------------------------------------------------------------------- 1 | package meta 2 | 3 | const ( 4 | MaxBlockOffset = uint32(4294967295) 5 | ) 6 | -------------------------------------------------------------------------------- /libs/meta/directory.go: -------------------------------------------------------------------------------- 1 | package meta 2 | 3 | // StoreRet 4 | type StoreRet struct { 5 | Ret int `json:"ret"` 6 | } 7 | 8 | // Response 9 | type Response struct { 10 | Ret int `json:"ret"` 11 | Key int64 `json:"key"` 12 | Cookie int32 `json:"cookie"` 13 | Vid int32 `json:"vid"` 14 | Stores []string `json:"stores"` 15 | MTime int64 `json:"update_time"` 16 | Sha1 string `json:"sha1"` 17 | Mine string `json:"mine"` 18 | } 19 | -------------------------------------------------------------------------------- /libs/meta/file.go: -------------------------------------------------------------------------------- 1 | package meta 2 | 3 | // File meta info. 4 | type File struct { 5 | Filename string `json:"filename"` 6 | Key int64 `json:"key"` 7 | Sha1 string `json:"sha1"` 8 | Mine string `json:"mine"` 9 | Status int32 `json:"status"` 10 | MTime int64 `json:"update_time"` 11 | } 12 | -------------------------------------------------------------------------------- /libs/meta/needle.go: -------------------------------------------------------------------------------- 1 | package meta 2 | 3 | type Needle struct { 4 | Key int64 `json:"key"` 5 | Cookie int32 `json:"cookie"` 6 | Vid int32 `json:"vid"` 7 | MTime int64 `json:"update_time"` 8 | } 9 | -------------------------------------------------------------------------------- /libs/meta/store.go: -------------------------------------------------------------------------------- 1 | package meta 2 | 3 | import ( 4 | "bfs/libs/errors" 5 | "encoding/json" 6 | "fmt" 7 | "io/ioutil" 8 | "net/http" 9 | "time" 10 | 11 | log "github.com/golang/glog" 12 | ) 13 | 14 | const ( 15 | // bit 16 | StoreStatusEnableBit = 31 17 | StoreStatusReadBit = 0 18 | StoreStatusWriteBit = 1 19 | // status 20 | StoreStatusInit = 0 21 | StoreStatusEnable = (1 << StoreStatusEnableBit) 22 | StoreStatusRead = StoreStatusEnable | (1 << StoreStatusReadBit) 23 | StoreStatusWrite = StoreStatusEnable | (1 << StoreStatusWriteBit) 24 | StoreStatusHealth = StoreStatusRead | StoreStatusWrite 25 | StoreStatusFail = StoreStatusEnable 26 | // api 27 | statAPI = "http://%s/info" 28 | getAPI = "http://%s/get?key=%d&cookie=%d&vid=%d" 29 | probeAPI = "http://%s/probe?vid=%d" 30 | ) 31 | 32 | var ( 33 | _client = &http.Client{ 34 | Transport: &http.Transport{ 35 | DisableCompression: true, 36 | }, 37 | Timeout: 10 * time.Second, 38 | } 39 | ) 40 | 41 | type StoreList []*Store 42 | 43 | func (sl StoreList) Len() int { 44 | return len(sl) 45 | } 46 | 47 | func (sl StoreList) Less(i, j int) bool { 48 | return sl[i].Id < sl[j].Id 49 | } 50 | 51 | func (sl StoreList) Swap(i, j int) { 52 | sl[i], sl[j] = sl[j], sl[i] 53 | } 54 | 55 | // store zk meta data. 56 | type Store struct { 57 | Stat string `json:"stat"` 58 | Admin string `json:"admin"` 59 | Api string `json:"api"` 60 | Id string `json:"id"` 61 | Rack string `json:"rack"` 62 | Status int `json:"status"` 63 | } 64 | 65 | func (s *Store) String() string { 66 | return fmt.Sprintf(` 67 | ----------------------------- 68 | Id: %s 69 | Stat: %s 70 | Admin: %s 71 | Api: %s 72 | Rack: %s 73 | Status: %d 74 | ----------------------------- 75 | `, s.Id, s.Stat, s.Admin, s.Api, s.Rack, s.Status) 76 | } 77 | 78 | // statAPI get stat http api. 79 | func (s *Store) statAPI() string { 80 | return fmt.Sprintf(statAPI, s.Stat) 81 | } 82 | 83 | // getApi get file http api 84 | func (s *Store) getAPI(n *Needle, vid int32) string { 85 | return fmt.Sprintf(getAPI, s.Stat, n.Key, n.Cookie, vid) 86 | } 87 | 88 | // probeApi probe store 89 | func (s *Store) probeAPI(vid int32) string { 90 | return fmt.Sprintf(probeAPI, s.Admin, vid) 91 | } 92 | 93 | // Info get store volumes info. 94 | func (s *Store) Info() (vs []*Volume, err error) { 95 | var ( 96 | body []byte 97 | req *http.Request 98 | resp *http.Response 99 | data = new(Volumes) 100 | url = s.statAPI() 101 | ) 102 | if req, err = http.NewRequest("GET", url, nil); err != nil { 103 | log.Info("http.NewRequest(GET,%s) error(%v)", url, err) 104 | return 105 | } 106 | if resp, err = _client.Do(req); err != nil { 107 | log.Errorf("_client.do(%s) error(%v)", url, err) 108 | return 109 | } 110 | defer resp.Body.Close() 111 | if resp.StatusCode != http.StatusOK { 112 | err = errors.ErrInternal 113 | return 114 | } 115 | if body, err = ioutil.ReadAll(resp.Body); err != nil { 116 | log.Errorf("ioutil.ReadAll() error(%v)", err) 117 | return 118 | } 119 | if err = json.Unmarshal(body, &data); err != nil { 120 | log.Errorf("json.Unmarshal() error(%v)", err) 121 | return 122 | } 123 | vs = data.Volumes 124 | return 125 | } 126 | 127 | // Head send a head request to store. 128 | func (s *Store) Head(vid int32) (err error) { 129 | var ( 130 | req *http.Request 131 | resp *http.Response 132 | url string 133 | ) 134 | url = s.probeAPI(vid) 135 | if req, err = http.NewRequest("HEAD", url, nil); err != nil { 136 | log.Info("http.NewRequest(GET,%s) error(%v)", url, err) 137 | return 138 | } 139 | if resp, err = _client.Do(req); err != nil { 140 | log.Errorf("_client.do(%s) error(%v)", url, err) 141 | return 142 | } 143 | defer resp.Body.Close() 144 | if resp.StatusCode == http.StatusInternalServerError { 145 | err = errors.ErrInternal 146 | } 147 | return 148 | } 149 | 150 | // CanWrite reports whether the store can write. 151 | func (s *Store) CanWrite() bool { 152 | return s.Status == StoreStatusWrite || s.Status == StoreStatusHealth 153 | } 154 | 155 | // CanRead reports whether the store can read. 156 | func (s *Store) CanRead() bool { 157 | return s.Status == StoreStatusRead || s.Status == StoreStatusHealth 158 | } 159 | -------------------------------------------------------------------------------- /libs/meta/super_block.go: -------------------------------------------------------------------------------- 1 | package meta 2 | 3 | const ( 4 | blockLeftSpace = uint32(1024 * 1024 * 1) // 1 mb 5 | ) 6 | 7 | type SuperBlock struct { 8 | File string `json:"file"` 9 | Offset uint32 `json:"offset"` 10 | LastErr error `json:"last_err"` 11 | Ver byte `json:"ver"` 12 | Padding uint32 `json:"padding"` 13 | } 14 | 15 | // Full check the block full. 16 | func (b *SuperBlock) Full() bool { 17 | return ((MaxBlockOffset - b.Offset) < (blockLeftSpace / b.Padding)) 18 | } 19 | 20 | // FreeSpace cal rest space of volume 21 | func (b *SuperBlock) FreeSpace() uint32 { 22 | return MaxBlockOffset - b.Offset 23 | } 24 | -------------------------------------------------------------------------------- /libs/meta/volume.go: -------------------------------------------------------------------------------- 1 | package meta 2 | 3 | import "bfs/libs/stat" 4 | 5 | type Volume struct { 6 | Id int32 `json:"id"` 7 | Block *SuperBlock `json:"block"` 8 | Stats *stat.Stats `json:"stats"` 9 | } 10 | 11 | type Volumes struct { 12 | Volumes []*Volume `json:"volumes"` 13 | } 14 | 15 | // VolumeState for zk /volume stat 16 | type VolumeState struct { 17 | TotalWriteProcessed uint64 `json:"total_write_processed"` 18 | TotalWriteDelay uint64 `json:"total_write_delay"` 19 | FreeSpace uint32 `json:"free_space"` 20 | } 21 | -------------------------------------------------------------------------------- /libs/stat/stat_test.go: -------------------------------------------------------------------------------- 1 | package stat 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestStat(t *testing.T) { 8 | var ( 9 | s = &Stats{} 10 | s1 = &Stats{} 11 | ) 12 | // tps & qps 13 | s.TotalAddProcessed = 10 14 | s.TotalWriteProcessed = 15 15 | s.TotalDelProcessed = 20 16 | s.TotalGetProcessed = 25 17 | s.TotalFlushProcessed = 30 18 | s.TotalCompactProcessed = 35 19 | s1.TotalAddProcessed = 10 20 | s1.TotalWriteProcessed = 15 21 | s1.TotalDelProcessed = 20 22 | s1.TotalGetProcessed = 25 23 | s1.TotalFlushProcessed = 30 24 | s1.TotalCompactProcessed = 35 25 | // bytes 26 | // delay 27 | s.Calc() 28 | s1.Merge(s) 29 | s1.Calc() 30 | if s.AddTPS != 10 { 31 | t.Errorf("TotalAddTPS: %d not match", s.AddTPS) 32 | t.FailNow() 33 | } 34 | if s.WriteTPS != 15 { 35 | t.Errorf("TotalWriteTPS: %d not match", s.WriteTPS) 36 | t.FailNow() 37 | } 38 | if s.DelTPS != 20 { 39 | t.Errorf("TotalDelTPS: %d not match", s.DelTPS) 40 | t.FailNow() 41 | } 42 | if s.GetQPS != 25 { 43 | t.Errorf("TotalGetQPS: %d not match", s.GetQPS) 44 | t.FailNow() 45 | } 46 | if s.FlushTPS != 30 { 47 | t.Errorf("TotalFlushTPS: %d not match", s.FlushTPS) 48 | t.FailNow() 49 | } 50 | if s.TotalCommandsProcessed != 135 { 51 | t.Errorf("TotalCommandsProcessed: %d not match", s.TotalCommandsProcessed) 52 | t.FailNow() 53 | } 54 | if s1.TotalAddProcessed != 20 { 55 | t.Errorf("TotalAddProcessed: %d not match", s1.TotalAddProcessed) 56 | t.FailNow() 57 | } 58 | if s1.TotalWriteProcessed != 30 { 59 | t.Errorf("TotalWriteProcessed: %d not match", s1.TotalWriteProcessed) 60 | t.FailNow() 61 | } 62 | if s1.TotalDelProcessed != 40 { 63 | t.Errorf("TotalDelProcessed: %d not match", s1.TotalDelProcessed) 64 | t.FailNow() 65 | } 66 | if s1.TotalGetProcessed != 50 { 67 | t.Errorf("TotalGetProcessed: %d not match", s1.TotalGetProcessed) 68 | t.FailNow() 69 | } 70 | if s1.TotalFlushProcessed != 60 { 71 | t.Errorf("TotalFlushProcessed: %d not match", s1.TotalFlushProcessed) 72 | t.FailNow() 73 | } 74 | if s1.TotalCompactProcessed != 70 { 75 | t.Errorf("TotalCompactProcessed: %d not match", s1.TotalCompactProcessed) 76 | t.FailNow() 77 | } 78 | if s1.AddTPS != 20 { 79 | t.Errorf("TotalAddTPS: %d not match", s1.AddTPS) 80 | t.FailNow() 81 | } 82 | if s1.WriteTPS != 30 { 83 | t.Errorf("TotalWriteTPS: %d not match", s1.WriteTPS) 84 | t.FailNow() 85 | } 86 | if s1.DelTPS != 40 { 87 | t.Errorf("TotalDelTPS: %d not match", s1.DelTPS) 88 | t.FailNow() 89 | } 90 | if s1.GetQPS != 50 { 91 | t.Errorf("TotalGetQPS: %d not match", s1.GetQPS) 92 | t.FailNow() 93 | } 94 | if s1.FlushTPS != 60 { 95 | t.Errorf("TotalFlushTPS: %d not match", s1.FlushTPS) 96 | t.FailNow() 97 | } 98 | if s1.TotalCommandsProcessed != 270 { 99 | t.Errorf("TotalCommandsProcessed: %d not match", s1.TotalCommandsProcessed) 100 | t.FailNow() 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /libs/time/time.go: -------------------------------------------------------------------------------- 1 | package time 2 | 3 | import ( 4 | "database/sql/driver" 5 | "strconv" 6 | xtime "time" 7 | ) 8 | 9 | // Time be used to MySql timestamp converting. 10 | type Time int64 11 | 12 | // Scan scan time. 13 | func (jt *Time) Scan(src interface{}) (err error) { 14 | switch sc := src.(type) { 15 | case xtime.Time: 16 | *jt = Time(sc.Unix()) 17 | case string: 18 | var i int64 19 | i, err = strconv.ParseInt(sc, 10, 64) 20 | *jt = Time(i) 21 | } 22 | return 23 | } 24 | 25 | // Value get time value. 26 | func (jt Time) Value() (driver.Value, error) { 27 | return xtime.Unix(int64(jt), 0), nil 28 | } 29 | 30 | // Time get time. 31 | func (jt Time) Time() xtime.Time { 32 | return xtime.Unix(int64(jt), 0) 33 | } 34 | 35 | // Duration be used toml unmarshal string time, like 1s, 500ms. 36 | type Duration xtime.Duration 37 | 38 | // UnmarshalText unmarshal text to duration. 39 | func (d *Duration) UnmarshalText(text []byte) error { 40 | tmp, err := xtime.ParseDuration(string(text)) 41 | if err == nil { 42 | *d = Duration(tmp) 43 | } 44 | return err 45 | } 46 | -------------------------------------------------------------------------------- /libs/uuid/uuid.go: -------------------------------------------------------------------------------- 1 | package uuid 2 | 3 | import ( 4 | "crypto/rand" 5 | "errors" 6 | "fmt" 7 | "io" 8 | ) 9 | 10 | const ( 11 | Size = 16 12 | ) 13 | 14 | var ( 15 | ErrUUIDSize = errors.New("uuid size error") 16 | ) 17 | 18 | // New generate a uuid. 19 | func New() (str string, err error) { 20 | var ( 21 | n int 22 | uuid = make([]byte, Size) 23 | ) 24 | if n, err = io.ReadFull(rand.Reader, uuid); err != nil { 25 | return 26 | } 27 | if n != Size { 28 | return "", ErrUUIDSize 29 | } 30 | uuid[8] = uuid[8]&^0xc0 | 0x80 31 | uuid[6] = uuid[6]&^0xf0 | 0x40 32 | str = fmt.Sprintf("%x-%x-%x-%x-%x", uuid[0:4], uuid[4:6], uuid[6:8], uuid[8:10], uuid[10:]) 33 | return 34 | } 35 | -------------------------------------------------------------------------------- /libs/uuid/uuid_test.go: -------------------------------------------------------------------------------- 1 | package uuid 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestUUID(t *testing.T) { 8 | var ( 9 | str string 10 | err error 11 | ) 12 | if str, err = New(); err != nil { 13 | t.Errorf("New() error(%v)", err) 14 | t.FailNow() 15 | } 16 | t.Logf("uuid: %s", str) 17 | } 18 | -------------------------------------------------------------------------------- /ops/README.md: -------------------------------------------------------------------------------- 1 | # bfsops 2 | The background management interface of BFS. 3 | 4 | ## 初始化流程(请参照test/ops_initialization.py)： 5 | 6 | ###step 1 7 | 启动store、directory、proxy、pitchfork 8 | store启动后，zookeeper看到/rack 有store节点 9 | 10 | ###step 2: 11 | 调用space()函数，初始化store，调用完成后，磁盘空间会被分配；bfs存储目录看到生成的volume文件。 12 | 13 | ###step 3: 14 | 调用groups()函数，store分组，调用完成后，zookeeper看到/group/ 有组节点 15 | 16 | ###step 4: 17 | 调用volumes()函数， 生效volume，调用完成后，zookeeper看到/volume/有volume节点 18 | 19 | ### Done -------------------------------------------------------------------------------- /ops/commons/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from global_var import * 5 | from blogging import logger 6 | 7 | 8 | def parseSize(string_size): 9 | if string_size[-1] == 'G': 10 | return int(string_size[:-1]) 11 | elif string_size[len(string_size)-2:] == 'GB': 12 | return int(string_size[:-2]) 13 | elif string_size[-1] == 'T': 14 | return 1024 * int(string_size[:-1]) 15 | elif string_size[len(string_size)-2:] == 'TB': 16 | return 1024 * int(string_size[:-2]) 17 | else: 18 | return None 19 | 20 | def grouping_store(ips, copys, rack): 21 | #group and set to zk 22 | #if ip has been grouped ,ignore 23 | #copys + rack + free_volumes 24 | #[a[i:i+n] for i in range(0, len(a), n)] just a temporary plan 25 | group_stores = [ips[i:i+copys] for i in range(0, len(ips), copys)] 26 | return group_stores 27 | 28 | 29 | from flask import Flask 30 | def createApp(): 31 | app = Flask(__name__, template_folder='../templates', static_folder='../static') 32 | return app 33 | 34 | app = createApp() 35 | 36 | 37 | def initBfsData(): 38 | from zk_client import initFromZk 39 | if initFromZk(): 40 | logger.info("initFromZk() is called, success") 41 | else: 42 | logger.error("initFromZk() is called, failed") 43 | return False 44 | 45 | from store_client import initFromStore 46 | for store_ip in IP_TO_STORE: 47 | if initFromStore(store_ip): 48 | logger.info("initFromStore() is called, success") 49 | else: 50 | logger.info("initFromStore() is called, failed") 51 | return False 52 | 53 | logger.info("initBfsData() is called, success") 54 | return True 55 | 56 | 57 | if not initBfsData(): 58 | logger.error("initBfsData() called, failed QUIT NOW") 59 | import sys 60 | sys.exit() 61 | -------------------------------------------------------------------------------- /ops/commons/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Terry-Mao/bfs/6a964012975171345fcb931ea9183ee58666748e/ops/commons/__init__.pyc -------------------------------------------------------------------------------- /ops/commons/blogging.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import config 5 | import logging 6 | import logging.handlers 7 | 8 | handler = logging.handlers.RotatingFileHandler(config.log_dir+'/'+'ops.log', maxBytes = 1024*1024, backupCount = 5) 9 | formatter = logging.Formatter(fmt = '%(asctime)s - %(filename)s:%(lineno)s - %(name)s - %(message)s' ) 10 | 11 | handler.setFormatter(formatter) 12 | logger = logging.getLogger('bfs') 13 | logger.addHandler(handler) 14 | logger.setLevel(logging.DEBUG) 15 | 16 | -------------------------------------------------------------------------------- /ops/commons/blogging.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Terry-Mao/bfs/6a964012975171345fcb931ea9183ee58666748e/ops/commons/blogging.pyc -------------------------------------------------------------------------------- /ops/commons/config.py: -------------------------------------------------------------------------------- 1 | # -*- encoding:utf8 -*- 2 | 3 | # store. 4 | store_admin_port = 6063 5 | store_stat_port = 6061 6 | store_block_size = 32 # GB 7 | 8 | # zookeeper port:2181. 9 | zk_hosts = '127.0.0.1' 10 | 11 | # log dir. 12 | log_dir = '/tmp' 13 | -------------------------------------------------------------------------------- /ops/commons/config.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Terry-Mao/bfs/6a964012975171345fcb931ea9183ee58666748e/ops/commons/config.pyc -------------------------------------------------------------------------------- /ops/commons/global_var.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | RACK_STORE = {} #init from zk /rack 6 | GROUP_STORE = {} 7 | MAX_GROUP_ID = 0 8 | 9 | STORE_TO_IP = {} # store server_id to ip 10 | IP_TO_STORE = {} # store ip to server_id 11 | 12 | STORE_INFO = {} #init from store /info 13 | VOLUME_KEY = "volume" 14 | FREE_VOLUME_KEY = "free_volume" 15 | 16 | STORE_RACK = {} 17 | STORE_VOLUME = {} 18 | STORE_GROUP = {} 19 | 20 | MAX_VOLUME_ID = 0 -------------------------------------------------------------------------------- /ops/commons/global_var.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Terry-Mao/bfs/6a964012975171345fcb931ea9183ee58666748e/ops/commons/global_var.pyc -------------------------------------------------------------------------------- /ops/commons/store_client.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | import json 4 | import httplib 5 | import urllib 6 | import hashlib 7 | 8 | from blogging import logger 9 | import config 10 | from global_var import * 11 | 12 | global_store_conn_pool = {} 13 | 14 | class HttpConnection: 15 | DEFAULT_TIMEOUT = 60 16 | 17 | def __init__(self, host, port): 18 | self.host = host 19 | self.port = port 20 | self.http_conn = None 21 | 22 | def request(self, method, url, body=None, headers={}): 23 | '''return [True|False], status, data''' 24 | try: 25 | print url,body 26 | if self.http_conn is None: 27 | self.http_conn = httplib.HTTPConnection(self.host, self.port, timeout=HttpConnection.DEFAULT_TIMEOUT) 28 | 29 | self.http_conn.request(method, url, body, headers) 30 | conn_resp = self.http_conn.getresponse() 31 | status = conn_resp.status 32 | data = conn_resp.read() 33 | 34 | return (True, status, data) 35 | except Exception as ex: 36 | self.http_conn = None 37 | logger.warn('HttpConnection.request, %s %s, except %s', method, url, str(ex)) 38 | 39 | return (False, None, None) 40 | 41 | 42 | def genHttpHeaders(): 43 | return {"Content-Type":"application/x-www-form-urlencoded"} 44 | 45 | 46 | def genStoreConnKey(store_ip, store_port): 47 | return hashlib.sha1('%s/%d'%(store_ip, store_port)).hexdigest() 48 | 49 | 50 | def genStoreConn(store_ip, store_port): 51 | key = genStoreConnKey(store_ip, store_port) 52 | if global_store_conn_pool.has_key(key): 53 | store_conn = global_store_conn_pool[key] 54 | else: 55 | store_conn = HttpConnection(host=store_ip, port=store_port) 56 | global_store_conn_pool[key] = store_conn 57 | 58 | return store_conn 59 | 60 | 61 | def storeAddFreeVolume(store_ip, base_dir, num_volumes): 62 | if num_volumes <= 0: 63 | return None 64 | print store_ip 65 | store_conn = genStoreConn(store_ip, config.store_admin_port) 66 | url = "/add_free_volume" 67 | value = {} 68 | value['n'] = num_volumes 69 | value['bdir'] = base_dir 70 | value['idir'] = base_dir 71 | body = urllib.urlencode(value) 72 | 73 | retcode, status, data = store_conn.request('POST', url, body, headers=genHttpHeaders()) 74 | if retcode and status >= 200 and status < 300: 75 | return json.loads(data) 76 | 77 | logger.error("storeAddFreeVolume() called failed: store_ip: %s, num_volumes: %d base_dir: %s", 78 | store_ip, num_volumes, base_dir) 79 | return None 80 | 81 | 82 | def storeAddVolume(store_ip, volume_id): 83 | if volume_id <= 0: 84 | return None 85 | 86 | store_conn = genStoreConn(store_ip, config.store_admin_port) 87 | url = "/add_volume" 88 | value = {} 89 | value['vid'] = volume_id 90 | body = urllib.urlencode(value) 91 | 92 | retcode, status, data = store_conn.request('POST', url, body, headers=genHttpHeaders()) 93 | if retcode and status >= 200 and status < 300: 94 | return json.loads(data) 95 | 96 | logger.error("storeAddVolume() called failed: status: %d, store_ip: %s, volume_id: %d", status, store_ip, volume_id) 97 | return None 98 | 99 | 100 | def initFromStore(store_ip): 101 | store_conn = genStoreConn(store_ip, config.store_stat_port) 102 | url = "/info" 103 | 104 | retcode, status, data = store_conn.request('GET', url) 105 | if retcode and status >= 200 and status < 300: 106 | store_data = json.loads(data) 107 | if store_data['ret'] != 1: 108 | return False 109 | 110 | free_volumes_num = volumes_num = 0 111 | 112 | free_volumes = store_data['free_volumes'] 113 | if free_volumes is not None: 114 | free_volumes_num = len(free_volumes) - 1 115 | STORE_INFO[FREE_VOLUME_KEY+IP_TO_STORE[store_ip]] = free_volumes_num 116 | 117 | volumes = store_data['volumes'] 118 | if volumes is not None: 119 | volumes_num = len(volumes) 120 | STORE_INFO[VOLUME_KEY+IP_TO_STORE[store_ip]] = volumes_num 121 | return True 122 | 123 | logger.error("initFromStore() called failed: store_ip: %s", store_ip) 124 | return False 125 | 126 | 127 | def storeBulkVolume(store_ip, body): 128 | pass 129 | 130 | 131 | def storeCompactVolume(store_ip, body): 132 | pass 133 | -------------------------------------------------------------------------------- /ops/commons/store_client.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Terry-Mao/bfs/6a964012975171345fcb931ea9183ee58666748e/ops/commons/store_client.pyc -------------------------------------------------------------------------------- /ops/commons/zk_client.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Terry-Mao/bfs/6a964012975171345fcb931ea9183ee58666748e/ops/commons/zk_client.pyc -------------------------------------------------------------------------------- /ops/runserver.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf8 -*- 2 | 3 | import views 4 | from commons import * 5 | 6 | 7 | app.run(host = '0.0.0.0', port = 9000, debug = True) -------------------------------------------------------------------------------- /ops/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name="bfs", 5 | version="1.0", 6 | install_requires=[ 7 | "requests", 8 | "flask", 9 | ], 10 | ) 11 | -------------------------------------------------------------------------------- /ops/test/ops_initialization.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import urllib2 5 | import json 6 | 7 | # initialize bfs space 8 | def space(): 9 | url = 'http://127.0.0.1:9000/bfsops/initialization' 10 | # ips 机器的ip列表 11 | # dirs 磁盘目录 12 | # size 每块磁盘的空间大小 13 | value = {"ips":"xx.xx.xx.xx, yy.yy.yy.yy","dirs":"/data1/bfsdata/,/data2/bfsdata/","size":"10T"} 14 | 15 | jdata = json.dumps(value) 16 | req = urllib2.Request(url, jdata, headers = {"Content-type":"application/json"}) 17 | response = urllib2.urlopen(req) 18 | return response.read() 19 | # initialize groups 20 | def groups(): 21 | url = 'http://127.0.0.1:9000/bfsops/groups' 22 | # ips 要分组的ip列表 23 | # copys 副本数（包括本身） 24 | # racks 跨机架 默认1即可，具体请参考ops代码 25 | value = {"ips":"xx.xx.xx.xx, yy.yy.yy.yy", "copys":2, "rack":1} 26 | 27 | jdata = json.dumps(value) 28 | req = urllib2.Request(url, jdata, headers = {"Content-type":"application/json"}) 29 | response = urllib2.urlopen(req) 30 | return response.read() 31 | 32 | # initialize volumes 33 | def volumes(): 34 | url = 'http://127.0.0.1:9000/bfsops/volumes' 35 | # groups 生效某个group 36 | value = {"groups":"2"} 37 | 38 | jdata = json.dumps(value) 39 | req = urllib2.Request(url, jdata, headers = {"Content-type":"application/json"}) 40 | response = urllib2.urlopen(req) 41 | return response.read() 42 | 43 | # store启动后，zookeeper看到/rack 有store节点 44 | 45 | # 初始化流程： 46 | # 分别依次调用3个函数，分别完成：初始化store；store分组；生效volume. 47 | #step 1: 48 | #space() 49 | #调用完成后，磁盘空间会被分配；bfs存储目录看到生成的volume文件。 50 | 51 | #step 2: 52 | #groups() 53 | #调用完成后，zookeeper看到/group/ 有组节点 54 | 55 | #step 3: 56 | #volumes() 57 | #调用完成后，zookeeper看到/volume/有volume节点 58 | 59 | 60 | -------------------------------------------------------------------------------- /ops/views/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import ops -------------------------------------------------------------------------------- /ops/views/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Terry-Mao/bfs/6a964012975171345fcb931ea9183ee58666748e/ops/views/__init__.pyc -------------------------------------------------------------------------------- /ops/views/ops.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Terry-Mao/bfs/6a964012975171345fcb931ea9183ee58666748e/ops/views/ops.pyc -------------------------------------------------------------------------------- /pitchfork/conf/config.go: -------------------------------------------------------------------------------- 1 | package conf 2 | 3 | import ( 4 | "github.com/BurntSushi/toml" 5 | "io/ioutil" 6 | "os" 7 | "time" 8 | ) 9 | 10 | type Config struct { 11 | Store *Store 12 | Zookeeper *Zookeeper 13 | } 14 | 15 | type Store struct { 16 | StoreCheckInterval duration 17 | NeedleCheckInterval duration 18 | RackCheckInterval duration 19 | } 20 | 21 | type Zookeeper struct { 22 | VolumeRoot string 23 | StoreRoot string 24 | PitchforkRoot string 25 | Addrs []string 26 | Timeout duration 27 | } 28 | 29 | // Code to implement the TextUnmarshaler interface for `duration`: 30 | type duration struct { 31 | time.Duration 32 | } 33 | 34 | func (d *duration) UnmarshalText(text []byte) error { 35 | var err error 36 | d.Duration, err = time.ParseDuration(string(text)) 37 | return err 38 | } 39 | 40 | // NewConfig new a config. 41 | func NewConfig(conf string) (c *Config, err error) { 42 | var ( 43 | file *os.File 44 | blob []byte 45 | ) 46 | c = new(Config) 47 | if file, err = os.Open(conf); err != nil { 48 | return 49 | } 50 | if blob, err = ioutil.ReadAll(file); err != nil { 51 | return 52 | } 53 | err = toml.Unmarshal(blob, c) 54 | return 55 | } 56 | -------------------------------------------------------------------------------- /pitchfork/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bfs/pitchfork/conf" 5 | "flag" 6 | log "github.com/golang/glog" 7 | ) 8 | 9 | var ( 10 | configFile string 11 | ) 12 | 13 | func init() { 14 | flag.StringVar(&configFile, "c", "./pitchfork.toml", " set pitchfork config file path") 15 | } 16 | 17 | func main() { 18 | var ( 19 | config *conf.Config 20 | p *Pitchfork 21 | err error 22 | ) 23 | flag.Parse() 24 | defer log.Flush() 25 | log.Infof("bfs pitchfork start") 26 | if config, err = conf.NewConfig(configFile); err != nil { 27 | log.Errorf("NewConfig(\"%s\") error(%v)", configFile, err) 28 | return 29 | } 30 | log.Infof("register pitchfork...") 31 | if p, err = NewPitchfork(config); err != nil { 32 | log.Errorf("pitchfork NewPitchfork() failed, Quit now") 33 | return 34 | } 35 | log.Infof("starts probe stores...") 36 | go p.Probe() 37 | StartSignal() 38 | return 39 | } 40 | -------------------------------------------------------------------------------- /pitchfork/pitchfork.toml: -------------------------------------------------------------------------------- 1 | [zookeeper] 2 | # zookeeper cluster addrs, multiple addrs split by ",". 3 | Addrs = [ 4 | "localhost:2181" 5 | ] 6 | 7 | # zookeeper heartbeat timeout. 8 | Timeout = "1s" 9 | 10 | # zookeeper pitchfork path. 11 | PitchforkRoot = "/pitchfork" 12 | 13 | # zookeeper storeroot path. 14 | StoreRoot = "/rack" 15 | 16 | # zookeeper volumeroot path 17 | VolumeRoot = "/volume" 18 | 19 | 20 | [store] 21 | #check store interval 22 | StoreCheckInterval = "5s" 23 | 24 | #check needle interval 25 | NeedleCheckInterval = "60s" 26 | 27 | #rack 28 | RackCheckInterval = "300s" 29 | -------------------------------------------------------------------------------- /pitchfork/pitchfork_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bfs/pitchfork/conf" 5 | myzk "bfs/store/zk" 6 | "fmt" 7 | "testing" 8 | "time" 9 | ) 10 | 11 | func TestPitchfork(t *testing.T) { 12 | 13 | var ( 14 | config *Config 15 | zk *Zookeeper 16 | p *Pitchfork 17 | storelist StoreList 18 | store *Store 19 | pitchforklist PitchforkList 20 | err error 21 | ) 22 | 23 | if config, err = conf.NewConfig(configFile); err != nil { 24 | t.Errorf("NewConfig(\"%s\") error(%v)", configFile, err) 25 | return 26 | } 27 | 28 | if zk, err = myzk.NewZookeeper([]string{"localhost:2181"}, time.Second*1); err != nil { 29 | t.Errorf("NewZookeeper() error(%v)", err) 30 | t.FailNow() 31 | } 32 | 33 | p = NewPitchfork(zk, config) 34 | if err = p.Register(); err != nil { 35 | t.Errorf("pitchfork Register() failed, Quit now") 36 | t.FailNow() 37 | } 38 | 39 | storelist, _, err = p.WatchGetStores() 40 | if err != nil { 41 | t.Errorf("pitchfork WatchGetStores() failed, Quit now") 42 | t.FailNow() 43 | } 44 | for _, store = range storelist { 45 | fmt.Println(store.rack, store.ID, store.host, store.status) 46 | } 47 | 48 | pitchforklist, _, err = p.WatchGetPitchforks() 49 | if err != nil { 50 | t.Errorf("pitchfork WatchGetPitchforks() failed, Quit now") 51 | t.FailNow() 52 | } 53 | for _, p = range pitchforklist { 54 | fmt.Println(p.ID) 55 | } 56 | 57 | for _, store = range storelist { 58 | if err = p.getStore(store); err != nil { 59 | t.Errorf("probeStore() called error(%v)", err) 60 | t.FailNow() 61 | } 62 | } 63 | 64 | } 65 | -------------------------------------------------------------------------------- /pitchfork/signal.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | "os/signal" 6 | "syscall" 7 | 8 | log "github.com/golang/glog" 9 | ) 10 | 11 | // StartSignal register signals handler. 12 | func StartSignal() { 13 | var ( 14 | c chan os.Signal 15 | s os.Signal 16 | ) 17 | c = make(chan os.Signal, 1) 18 | signal.Notify(c, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, 19 | syscall.SIGINT, syscall.SIGSTOP) 20 | // Block until a signal is received. 21 | for { 22 | s = <-c 23 | log.Infof("get a signal %s", s.String()) 24 | switch s { 25 | case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGSTOP, syscall.SIGINT: 26 | return 27 | case syscall.SIGHUP: 28 | // TODO reload 29 | //return 30 | default: 31 | return 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /pitchfork/zk/zk_test.go: -------------------------------------------------------------------------------- 1 | package zk 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func TestZk(t *testing.T) { 10 | 11 | var ( 12 | config *Config 13 | zk *Zookeeper 14 | err error 15 | ) 16 | 17 | if config, err = NewConfig(configFile); err != nil { 18 | t.Errorf("NewConfig(\"%s\") error(%v)", configFile, err) 19 | return 20 | } 21 | 22 | if zk, err = NewZookeeper([]string{"localhost:2181"}, time.Second*1); err != nil { 23 | t.Errorf("NewZookeeper() error(%v)", err) 24 | t.FailNow() 25 | } 26 | 27 | pathStore := fmt.Sprintf("%s/%s/%s", config.ZookeeperStoreRoot, "bfs-1", "47E273ED-CD3A-4D6A-94CE-554BA9B195EB") 28 | status := uint32(0) 29 | if err = zk.setStoreStatus(pathStore, status); err != nil { 30 | t.Errorf("setStoreStatus() error(%v)", err) 31 | t.FailNow() 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /proxy/auth/auth.go: -------------------------------------------------------------------------------- 1 | package auth 2 | 3 | import ( 4 | "bfs/libs/errors" 5 | ibucket "bfs/proxy/bucket" 6 | "bfs/proxy/conf" 7 | "crypto/hmac" 8 | "crypto/sha1" 9 | "encoding/base64" 10 | "fmt" 11 | "hash" 12 | "strconv" 13 | "strings" 14 | "time" 15 | ) 16 | 17 | const ( 18 | _authExpire = 900 // 15min 19 | _template = "%s\n%s\n%s\n%d\n" // method bucket filename expire 20 | ) 21 | 22 | type Auth struct { 23 | c *conf.Config 24 | } 25 | 26 | // NewAuth 27 | func New(c *conf.Config) (a *Auth, err error) { 28 | a = &Auth{} 29 | a.c = c 30 | return 31 | } 32 | 33 | func (a *Auth) Authorize(item *ibucket.Item, method, bucket, file, token string) (err error) { 34 | // token keyid:sign:time 35 | var ( 36 | expire int64 37 | delta int64 38 | now int64 39 | keyId string 40 | ss = strings.Split(token, ":") 41 | ) 42 | if len(ss) != 3 { 43 | return errors.ErrAuthFailed 44 | } 45 | keyId = ss[0] 46 | if keyId != item.KeyId { 47 | return errors.ErrAuthFailed 48 | } 49 | if expire, err = strconv.ParseInt(ss[2], 10, 64); err != nil { 50 | return errors.ErrAuthFailed 51 | } 52 | now = time.Now().Unix() 53 | // > ±15 min is forbidden 54 | if expire > now { 55 | delta = expire - now 56 | } else { 57 | delta = now - expire 58 | } 59 | if delta > _authExpire { 60 | return errors.ErrAuthFailed 61 | } 62 | err = a.sign(ss[1], method, bucket, file, item.KeySecret, expire) 63 | return 64 | } 65 | 66 | func (a *Auth) sign(src, method, bucket, file, keySecret string, expire int64) (err error) { 67 | var ( 68 | content string 69 | mac hash.Hash 70 | ) 71 | content = fmt.Sprintf(_template, method, bucket, file, expire) 72 | mac = hmac.New(sha1.New, []byte(keySecret)) 73 | mac.Write([]byte(content)) 74 | if base64.StdEncoding.EncodeToString(mac.Sum(nil)) != src { 75 | return errors.ErrAuthFailed 76 | } 77 | return 78 | } 79 | -------------------------------------------------------------------------------- /proxy/bfs/bfs_test.go: -------------------------------------------------------------------------------- 1 | package bfs 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | // "fmt" 7 | ) 8 | 9 | func TestBfs(t *testing.T) { 10 | var ( 11 | err error 12 | ) 13 | 14 | } 15 | -------------------------------------------------------------------------------- /proxy/bucket/bucket.go: -------------------------------------------------------------------------------- 1 | package bucket 2 | 3 | import ( 4 | "fmt" 5 | 6 | "bfs/libs/errors" 7 | ) 8 | 9 | const ( 10 | // status bit 11 | _privateReadBit = 0 12 | _privateWriteBit = 1 13 | // status 14 | _public = int(0) 15 | _privateRead = int(1 << _privateReadBit) 16 | _privateWrite = int(1 << _privateWriteBit) 17 | _privateReadWrite = int(_privateRead | _privateWrite) 18 | ) 19 | 20 | // bucket_name property key_id key_secret 21 | type Bucket struct { 22 | data map[string]*Item 23 | } 24 | 25 | type Header struct { 26 | CacheControl int64 27 | } 28 | 29 | type Item struct { 30 | Name string 31 | KeyId string 32 | KeySecret string 33 | Domain string 34 | Header *Header 35 | // property 第0位：读 (0表示共有，1表示私有) 第1位：写 (0表示共有，1表示私有) 36 | property int 37 | } 38 | 39 | func (i *Item) String() string { 40 | return fmt.Sprintf("{name: %s, property: %d}", i.Name, i.property) 41 | } 42 | 43 | func (i *Item) writePublic() bool { 44 | return i.property&_privateWrite == 0 45 | } 46 | 47 | func (i *Item) readPublic() bool { 48 | return i.property&_privateRead == 0 49 | } 50 | 51 | // Public check the item is public or not. 52 | func (i *Item) Public(read bool) bool { 53 | if read { 54 | return i.readPublic() 55 | } 56 | return i.writePublic() 57 | } 58 | 59 | // New a bucket. 60 | func New() (b *Bucket, err error) { 61 | var item *Item 62 | b = new(Bucket) 63 | b.data = make(map[string]*Item) 64 | // bucket test 65 | item = new(Item) 66 | item.Name = "test" 67 | item.property = _privateWrite 68 | item.KeyId = "221bce6492eba70f" 69 | item.KeySecret = "6eb80603e85842542f9736eb13b7e3" 70 | b.data[item.Name] = item 71 | return 72 | } 73 | 74 | // Get get a bucket, if not exist then error. 75 | func (b *Bucket) Get(name string) (item *Item, err error) { 76 | var ok bool 77 | if item, ok = b.data[name]; !ok { 78 | err = errors.ErrBucketNotExist 79 | } 80 | return 81 | } 82 | -------------------------------------------------------------------------------- /proxy/cache/cache.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "time" 7 | 8 | "bfs/libs/memcache" 9 | "bfs/libs/meta" 10 | 11 | log "github.com/golang/glog" 12 | ) 13 | 14 | // Cache proxy cache. 15 | type Cache struct { 16 | mc *memcache.Pool 17 | expire int32 18 | } 19 | 20 | // New new cache instance. 21 | func New(c *memcache.Config, expire time.Duration) (cache *Cache) { 22 | cache = &Cache{} 23 | cache.expire = int32(time.Duration(expire) / time.Second) 24 | cache.mc = memcache.NewPool(c) 25 | return 26 | } 27 | 28 | // Ping check memcache health 29 | func (c *Cache) Ping() (err error) { 30 | conn := c.mc.Get() 31 | err = conn.Store("set", "ping", []byte{1}, 0, c.expire, 0) 32 | conn.Close() 33 | return 34 | } 35 | 36 | func fileKey(bucket, filename string) string { 37 | return fmt.Sprintf("f/%s/%s", bucket, filename) 38 | } 39 | 40 | func metaKey(bucket, filename string) string { 41 | return fmt.Sprintf("m/%s/%s", bucket, filename) 42 | } 43 | 44 | // Meta get meta info from cache. 45 | func (c *Cache) Meta(bucket, fileName string) (mf *meta.File, err error) { 46 | var ( 47 | bs []byte 48 | key = metaKey(bucket, fileName) 49 | ) 50 | bs, err = c.get(key) 51 | if err != nil { 52 | if err == memcache.ErrNotFound { 53 | err = nil 54 | return 55 | } 56 | log.Errorf("cache Meta.get(%v) error(%v)", key, err) 57 | return 58 | } 59 | mf = new(meta.File) 60 | if err = json.Unmarshal(bs, mf); err != nil { 61 | log.Errorf("cache Meta.Unmarshal(%s) error(%v)", bs, err) 62 | } 63 | return 64 | } 65 | 66 | // SetMeta set meta into cache. 67 | func (c *Cache) SetMeta(bucket, fileName string, mf *meta.File) (err error) { 68 | key := metaKey(bucket, fileName) 69 | bs, err := json.Marshal(mf) 70 | if err != nil { 71 | log.Errorf("cache setMeta() Marshal(%s) error(%v)", bs, err) 72 | return 73 | } 74 | if err = c.set(key, bs, c.expire); err != nil { 75 | log.Errorf("cache setMeta() set(%s,%s) error(%v)", key, string(bs), err) 76 | } 77 | return 78 | } 79 | 80 | // DelMeta del meta from cache. 81 | func (c *Cache) DelMeta(bucket, fileName string) (err error) { 82 | key := metaKey(bucket, fileName) 83 | if err = c.del(key); err != nil { 84 | log.Errorf("cache DelMeta(%s) error(%v)", key, err) 85 | } 86 | return 87 | } 88 | 89 | // File get file from cache. 90 | func (c *Cache) File(bucket, fileName string) (bs []byte, err error) { 91 | key := fileKey(bucket, fileName) 92 | bs, err = c.get(key) 93 | if err != nil { 94 | if err == memcache.ErrNotFound { 95 | err = nil 96 | return 97 | } 98 | log.Errorf("cache File(%s) error(%v)", key, err) 99 | } 100 | return 101 | } 102 | 103 | // SetFile set file into cache. 104 | func (c *Cache) SetFile(bucket, fileName string, bs []byte) (err error) { 105 | key := fileKey(bucket, fileName) 106 | if err = c.set(key, bs, c.expire); err != nil { 107 | log.Errorf("cache setFile(%s) error(%v)", key, err) 108 | } 109 | return 110 | } 111 | 112 | // DelFile del file from cache. 113 | func (c *Cache) DelFile(bucket, fileName string) (err error) { 114 | key := fileKey(bucket, fileName) 115 | if err = c.del(key); err != nil { 116 | log.Errorf("cache DelFile(%s) error(%v)", key, err) 117 | } 118 | return 119 | } 120 | 121 | func (c *Cache) set(key string, bs []byte, expire int32) (err error) { 122 | conn := c.mc.Get() 123 | defer conn.Close() 124 | return conn.Store("set", key, bs, 0, expire, 0) 125 | } 126 | 127 | func (c *Cache) get(key string) (bs []byte, err error) { 128 | var ( 129 | conn = c.mc.Get() 130 | ) 131 | defer conn.Close() 132 | if bs, err = conn.Get2("get", key); err != nil { 133 | return 134 | } 135 | return 136 | } 137 | 138 | func (c *Cache) del(key string) (err error) { 139 | conn := c.mc.Get() 140 | defer conn.Close() 141 | return conn.Delete(key) 142 | } 143 | -------------------------------------------------------------------------------- /proxy/conf/config.go: -------------------------------------------------------------------------------- 1 | package conf 2 | 3 | import ( 4 | "bfs/libs/memcache" 5 | "bfs/libs/time" 6 | "path" 7 | "strings" 8 | 9 | "github.com/BurntSushi/toml" 10 | ) 11 | 12 | type Config struct { 13 | PprofEnable bool 14 | PprofListen string 15 | 16 | // api 17 | HttpAddr string 18 | // directory 19 | BfsAddr string 20 | // download domain 21 | Domain string 22 | // location prefix 23 | Prefix string 24 | // file 25 | MaxFileSize int 26 | // aliyun 27 | AliyunKeyId string 28 | AliyunKeySecret string 29 | // netcenter 30 | NetUserName string 31 | NetPasswd string 32 | // qcloud 33 | QcloudKeyID string 34 | QcloudKeySecret string 35 | // ats server list 36 | Ats *Ats 37 | // purge channel 38 | PurgeMaxSize int 39 | // memcache 40 | ExpireMc time.Duration 41 | Mc *memcache.Config 42 | // limit rate 43 | Limit *Limit 44 | } 45 | 46 | type Ats struct { 47 | AtsServerList []string 48 | } 49 | 50 | // Limit limit rate 51 | type Limit struct { 52 | Rate float64 53 | Brust int 54 | } 55 | 56 | // NewConfig new a config. 57 | func NewConfig(conf string) (c *Config, err error) { 58 | c = new(Config) 59 | if _, err = toml.DecodeFile(conf, c); err != nil { 60 | return 61 | } 62 | // bfs,/bfs,/bfs/ convert to /bfs/ 63 | if c.Prefix != "" { 64 | c.Prefix = path.Join("/", c.Prefix) + "/" 65 | // http://domain/ covert to http://domain 66 | c.Domain = strings.TrimRight(c.Domain, "/") 67 | } 68 | return 69 | } 70 | -------------------------------------------------------------------------------- /proxy/http_api_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "strings" 6 | ) 7 | 8 | func dfs_recur(bucket, dirname string) { 9 | var ( 10 | exit bool 11 | dir string 12 | file string 13 | files []string 14 | dirs []string 15 | ) 16 | 17 | dirs, files, _ = scanDir(bucket, dirname) 18 | for _, dir = range dirs { 19 | dir = dirname + dir 20 | dfs_recur(bucket, dir) 21 | } 22 | for _, file = range files { 23 | delFile(bucket, file) 24 | } 25 | return 26 | } 27 | 28 | func delFile(bucket, filename string) (err error) { 29 | if strings.HasSufix(filename, "/") { 30 | s.bfs.Delete(bucket, filename) 31 | } 32 | } 33 | 34 | func scanDir(bucket, filename string) (keys []string, err error) { 35 | 36 | } 37 | 38 | func main() { 39 | var ( 40 | bucket string 41 | filename string 42 | ) 43 | flag.Parse() 44 | bucket = "test" 45 | filename = "aaaa" 46 | 47 | dfs(bucket, filename, delFile, scanDir) 48 | } 49 | -------------------------------------------------------------------------------- /proxy/http_perf.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | log "github.com/golang/glog" 5 | "net/http" 6 | _ "net/http/pprof" 7 | ) 8 | 9 | // StartPprof start a golang pprof. 10 | func StartPprof(addr string) { 11 | go func() { 12 | var err error 13 | if err = http.ListenAndServe(addr, nil); err != nil { 14 | log.Errorf("http.ListenAndServe(\"%s\") error(%v)", addr, err) 15 | return 16 | } 17 | }() 18 | } 19 | -------------------------------------------------------------------------------- /proxy/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bfs/proxy/conf" 5 | "flag" 6 | "os" 7 | "os/signal" 8 | "runtime" 9 | "syscall" 10 | 11 | log "github.com/golang/glog" 12 | ) 13 | 14 | const ( 15 | version = "1.0.0" 16 | ) 17 | 18 | var ( 19 | configFile string 20 | ) 21 | 22 | func init() { 23 | flag.StringVar(&configFile, "c", "./proxy.toml", " set directory config file path") 24 | } 25 | 26 | func main() { 27 | var ( 28 | c *conf.Config 29 | err error 30 | ) 31 | flag.Parse() 32 | defer log.Flush() 33 | log.Infof("bfs proxy [version: %s] start", version) 34 | if c, err = conf.NewConfig(configFile); err != nil { 35 | log.Errorf("NewConfig(\"%s\") error(%v)", configFile, err) 36 | panic(err) 37 | } 38 | runtime.GOMAXPROCS(runtime.NumCPU()) 39 | // init http 40 | if err = StartAPI(c); err != nil { 41 | log.Errorf("http.Init() error(%v)", err) 42 | panic(err) 43 | } 44 | if c.PprofEnable { 45 | log.Infof("init http pprof...") 46 | StartPprof(c.PprofListen) 47 | } 48 | 49 | ch := make(chan os.Signal, 1) 50 | signal.Notify(ch, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT, syscall.SIGSTOP) 51 | for { 52 | s := <-ch 53 | log.Infof("get a signal %s", s.String()) 54 | switch s { 55 | case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGSTOP, syscall.SIGINT: 56 | return 57 | case syscall.SIGHUP: 58 | // TODO reload 59 | default: 60 | return 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /proxy/proxy.toml: -------------------------------------------------------------------------------- 1 | PprofEnable = true 2 | 3 | PprofListen = "localhost:2231" 4 | 5 | HttpAddr = "localhost:2232" 6 | 7 | BfsAddr = "localhost:2235" 8 | 9 | Domain = "http://localhost:2232/" 10 | 11 | Prefix = "/bfs/" 12 | 13 | MaxFileSize = 20971520 14 | 15 | AliyunKeyId = "xxxxx" 16 | AliyunKeySecret = "xxxxx" 17 | 18 | NetUserName = "xxxx" 19 | NetPasswd = "xxxx" 20 | 21 | PurgeMaxSize = 100000 22 | 23 | ExpireMc = "20m" 24 | 25 | [limit] 26 | rate = 150.0 27 | Brust = 50 28 | 29 | [mc] 30 | name = "bfs" 31 | proto = "tcp" 32 | addr = "localhost:11213" 33 | idle = 5 34 | active = 10 35 | dialTimeout = "1s" 36 | readTimeout = "1s" 37 | writeTimeout = "1s" 38 | idleTimeout = "80s" 39 | 40 | [Ats] 41 | AtsServerList = [ "http://localhost:8080" ] 42 | 43 | 44 | -------------------------------------------------------------------------------- /proxy/service.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "io/ioutil" 7 | "time" 8 | 9 | "bfs/libs/errors" 10 | "bfs/libs/meta" 11 | "bfs/proxy/bfs" 12 | "bfs/proxy/cache" 13 | "bfs/proxy/conf" 14 | 15 | log "github.com/golang/glog" 16 | "golang.org/x/time/rate" 17 | ) 18 | 19 | const _mcMaxLength = 1024 * 1024 // memcache max value is 1MB 20 | 21 | // Service . 22 | type Service struct { 23 | cache *cache.Cache 24 | bfs *bfs.Bfs 25 | cacheChan chan func() 26 | rl *rate.Limiter 27 | } 28 | 29 | // NewService new service 30 | func NewService(c *conf.Config) (s *Service) { 31 | s = &Service{ 32 | cache: cache.New(c.Mc, time.Duration(c.ExpireMc)), 33 | bfs: bfs.New(c), 34 | rl: rate.NewLimiter(rate.Limit(c.Limit.Rate), c.Limit.Brust), 35 | cacheChan: make(chan func(), 1024), 36 | } 37 | go s.cacheproc() 38 | return 39 | } 40 | 41 | func (s *Service) addCache(f func()) { 42 | select { 43 | case s.cacheChan <- f: 44 | default: 45 | log.Warningf("s.cacheChan is full") 46 | } 47 | } 48 | 49 | func (s *Service) cacheproc() { 50 | for f := range s.cacheChan { 51 | f() 52 | } 53 | } 54 | 55 | // Get get 56 | func (s *Service) Get(bucket, filename string) (src io.ReadCloser, ctlen int, mtime int64, sha1, mine string, err error) { 57 | var ( 58 | mf *meta.File 59 | bs []byte 60 | ) 61 | if mf, err = s.cache.Meta(bucket, filename); err == nil && mf != nil { 62 | if bs, err = s.cache.File(bucket, filename); err == nil && len(bs) > 0 { 63 | mtime = mf.MTime 64 | sha1 = mf.Sha1 65 | mine = mf.Mine 66 | ctlen = len(bs) 67 | src = ioutil.NopCloser(bytes.NewReader(bs)) 68 | return 69 | } 70 | } 71 | if !s.rl.Allow() { 72 | err = errors.ErrServiceUnavailable 73 | log.Errorf("service.bfs.Get.RateLimit(%s,%s),error(%v)", bucket, filename, err) 74 | return 75 | } 76 | // get from bfs 77 | if src, ctlen, mtime, sha1, mine, err = s.bfs.Get(bucket, filename); err != nil { 78 | log.Errorf("service.bfs.Get(%s,%s),error(%v)", bucket, filename, err) 79 | } 80 | return 81 | } 82 | 83 | // Upload upload 84 | func (s *Service) Upload(bucket, filename, mine, sha1 string, buf []byte) (err error) { 85 | var ( 86 | mtime = time.Now().UnixNano() 87 | mf *meta.File 88 | ) 89 | if err = s.bfs.Upload(bucket, filename, mine, sha1, mtime, buf); err != nil && err != errors.ErrNeedleExist { 90 | log.Errorf("service.bfs.Upload(%s,%s),error(%s)", bucket, filename, err) 91 | return 92 | } 93 | mf = &meta.File{ 94 | MTime: mtime, 95 | Sha1: sha1, 96 | Mine: mine, 97 | } 98 | if len(buf) < _mcMaxLength { 99 | s.addCache(func() { 100 | s.cache.SetMeta(bucket, filename, mf) 101 | s.cache.SetFile(bucket, filename, buf) 102 | }) 103 | } 104 | return 105 | } 106 | 107 | // Delete delete 108 | func (s *Service) Delete(bucket, filename string) (err error) { 109 | if err = s.bfs.Delete(bucket, filename); err != nil { 110 | log.Errorf("service.bfs.Delete(%s,%s),error(%v)", bucket, filename, err) 111 | return 112 | } 113 | s.cache.DelMeta(bucket, filename) 114 | s.cache.DelFile(bucket, filename) 115 | return 116 | } 117 | 118 | // Ping . 119 | func (s *Service) Ping() (err error) { 120 | if err = s.bfs.Ping(); err != nil { 121 | return 122 | } 123 | err = s.cache.Ping() 124 | return 125 | } 126 | -------------------------------------------------------------------------------- /store/conf/config.go: -------------------------------------------------------------------------------- 1 | package conf 2 | 3 | import ( 4 | "bfs/store/needle" 5 | "github.com/BurntSushi/toml" 6 | "io/ioutil" 7 | "os" 8 | "time" 9 | ) 10 | 11 | type Config struct { 12 | Pprof bool 13 | PprofListen string 14 | StatListen string 15 | ApiListen string 16 | AdminListen string 17 | 18 | NeedleMaxSize int 19 | BlockMaxSize int 20 | BatchMaxNum int 21 | 22 | Store *Store 23 | Volume *Volume 24 | Block *Block 25 | Index *Index 26 | Limit *Limit 27 | Zookeeper *Zookeeper 28 | } 29 | 30 | type Store struct { 31 | VolumeIndex string 32 | FreeVolumeIndex string 33 | } 34 | 35 | type Volume struct { 36 | SyncDelete int 37 | SyncDeleteDelay Duration 38 | } 39 | 40 | type Block struct { 41 | BufferSize int `toml:"-"` 42 | SyncWrite int 43 | Syncfilerange bool 44 | } 45 | 46 | type Index struct { 47 | BufferSize int 48 | MergeDelay Duration 49 | MergeWrite int 50 | RingBuffer int 51 | SyncWrite int 52 | Syncfilerange bool 53 | } 54 | 55 | type Zookeeper struct { 56 | Root string 57 | Rack string 58 | ServerId string 59 | Addrs []string 60 | Timeout Duration 61 | } 62 | 63 | type Rate struct { 64 | Rate float64 65 | Brust int 66 | } 67 | 68 | type Limit struct { 69 | Read *Rate 70 | Write *Rate 71 | Delete *Rate 72 | } 73 | 74 | // Code to implement the TextUnmarshaler interface for `Duration`: 75 | // 76 | type Duration struct { 77 | time.Duration 78 | } 79 | 80 | func (d *Duration) UnmarshalText(text []byte) error { 81 | var err error 82 | d.Duration, err = time.ParseDuration(string(text)) 83 | return err 84 | } 85 | 86 | // NewConfig new a config. 87 | func NewConfig(conf string) (c *Config, err error) { 88 | var ( 89 | file *os.File 90 | blob []byte 91 | ) 92 | c = new(Config) 93 | if file, err = os.Open(conf); err != nil { 94 | return 95 | } 96 | if blob, err = ioutil.ReadAll(file); err != nil { 97 | return 98 | } 99 | if err = toml.Unmarshal(blob, c); err == nil { 100 | c.BlockMaxSize = needle.Size(c.NeedleMaxSize) 101 | c.Block.BufferSize = needle.Size(c.NeedleMaxSize) 102 | } 103 | return 104 | } 105 | -------------------------------------------------------------------------------- /store/conf/config_test.go: -------------------------------------------------------------------------------- 1 | package conf 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestConfig(t *testing.T) { 8 | } 9 | -------------------------------------------------------------------------------- /store/http.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bfs/libs/errors" 5 | "bfs/libs/stat" 6 | "bfs/store/conf" 7 | "encoding/json" 8 | log "github.com/golang/glog" 9 | "golang.org/x/time/rate" 10 | "mime/multipart" 11 | "net" 12 | "net/http" 13 | "os" 14 | "strconv" 15 | "time" 16 | ) 17 | 18 | type Server struct { 19 | store *Store 20 | conf *conf.Config 21 | info *stat.Info 22 | // server 23 | statSvr net.Listener 24 | adminSvr net.Listener 25 | apiSvr net.Listener 26 | // limit 27 | rl *rate.Limiter 28 | wl *rate.Limiter 29 | dl *rate.Limiter 30 | } 31 | 32 | func NewServer(s *Store, c *conf.Config) (svr *Server, err error) { 33 | svr = &Server{ 34 | store: s, 35 | conf: c, 36 | rl: rate.NewLimiter(rate.Limit(c.Limit.Read.Rate), c.Limit.Read.Brust), 37 | wl: rate.NewLimiter(rate.Limit(c.Limit.Write.Rate), c.Limit.Write.Brust), 38 | dl: rate.NewLimiter(rate.Limit(c.Limit.Delete.Rate), c.Limit.Delete.Brust), 39 | } 40 | if svr.statSvr, err = net.Listen("tcp", c.StatListen); err != nil { 41 | log.Errorf("net.Listen(%s) error(%v)", c.StatListen, err) 42 | return 43 | } 44 | if svr.apiSvr, err = net.Listen("tcp", c.ApiListen); err != nil { 45 | log.Errorf("net.Listen(%s) error(%v)", c.ApiListen, err) 46 | return 47 | } 48 | if svr.adminSvr, err = net.Listen("tcp", c.AdminListen); err != nil { 49 | log.Errorf("net.Listen(%s) error(%v)", c.AdminListen, err) 50 | return 51 | } 52 | go svr.startStat() 53 | go svr.startApi() 54 | go svr.startAdmin() 55 | if c.Pprof { 56 | go StartPprof(c.PprofListen) 57 | } 58 | return 59 | } 60 | 61 | func (s *Server) Close() { 62 | if s.statSvr != nil { 63 | s.statSvr.Close() 64 | } 65 | if s.adminSvr != nil { 66 | s.adminSvr.Close() 67 | } 68 | if s.apiSvr != nil { 69 | s.apiSvr.Close() 70 | } 71 | return 72 | } 73 | 74 | type sizer interface { 75 | Size() int64 76 | } 77 | 78 | // checkFileSize get multipart.File size 79 | func checkFileSize(file multipart.File, maxSize int) (size int64, err error) { 80 | var ( 81 | ok bool 82 | sr sizer 83 | fr *os.File 84 | fi os.FileInfo 85 | ) 86 | if sr, ok = file.(sizer); ok { 87 | size = sr.Size() 88 | } else if fr, ok = file.(*os.File); ok { 89 | if fi, err = fr.Stat(); err != nil { 90 | log.Errorf("file.Stat() error(%v)", err) 91 | return 92 | } 93 | size = fi.Size() 94 | } 95 | if size > int64(maxSize) { 96 | err = errors.ErrNeedleTooLarge 97 | } 98 | return 99 | } 100 | 101 | func checkContentLength(r *http.Request, maxSize int) (err error) { 102 | var size int64 103 | // check total content-length 104 | if size, err = strconv.ParseInt(r.Header.Get("Content-Length"), 10, 64); err != nil { 105 | err = errors.ErrInternal 106 | return 107 | } 108 | if size > int64(maxSize) { 109 | err = errors.ErrNeedleTooLarge 110 | } 111 | return 112 | } 113 | 114 | func HttpPostWriter(r *http.Request, wr http.ResponseWriter, start time.Time, err *error, result map[string]interface{}) { 115 | var ( 116 | ok bool 117 | byteJson []byte 118 | err1 error 119 | errStr string 120 | uerr errors.Error 121 | ret = errors.RetOK 122 | ) 123 | if *err != nil { 124 | errStr = (*err).Error() 125 | if uerr, ok = (*err).(errors.Error); ok { 126 | ret = int(uerr) 127 | } else { 128 | ret = errors.RetInternalErr 129 | } 130 | } 131 | result["ret"] = ret 132 | if byteJson, err1 = json.Marshal(result); err1 != nil { 133 | log.Errorf("json.Marshal(\"%v\") failed (%v)", result, err1) 134 | return 135 | } 136 | wr.Header().Set("Content-Type", "application/json;charset=utf-8") 137 | if _, err1 = wr.Write(byteJson); err1 != nil { 138 | log.Errorf("http Write() error(%v)", err1) 139 | return 140 | } 141 | log.Infof("%s path:%s(params:%s,time:%f,ret:%v[%v])", r.Method, 142 | r.URL.Path, r.Form.Encode(), time.Now().Sub(start).Seconds(), ret, errStr) 143 | } 144 | 145 | func HttpGetWriter(r *http.Request, wr http.ResponseWriter, start time.Time, err *error, ret *int) { 146 | var errStr string 147 | if *ret != http.StatusOK { 148 | if *err != nil { 149 | errStr = (*err).Error() 150 | } 151 | http.Error(wr, errStr, *ret) 152 | } 153 | log.Infof("%s path:%s(params:%s,time:%f,err:%s,ret:%v[%v])", r.Method, 154 | r.URL.Path, r.URL.String(), time.Now().Sub(start).Seconds(), errStr, *ret, errStr) 155 | } 156 | -------------------------------------------------------------------------------- /store/http_perf.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | log "github.com/golang/glog" 5 | "net/http" 6 | _ "net/http/pprof" 7 | ) 8 | 9 | // StartPprof start a golang pprof. 10 | func StartPprof(addr string) { 11 | var err error 12 | if err = http.ListenAndServe(addr, nil); err != nil { 13 | log.Errorf("http.ListenAndServe(\"%s\") error(%v)", addr, err) 14 | panic(err) 15 | return 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /store/http_stat.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bfs/libs/errors" 5 | "bfs/libs/stat" 6 | "bfs/store/volume" 7 | "encoding/json" 8 | log "github.com/golang/glog" 9 | "net/http" 10 | "time" 11 | ) 12 | 13 | const ( 14 | statDuration = 1 * time.Second 15 | ) 16 | 17 | func (s *Server) startStat() { 18 | var ( 19 | err error 20 | serveMux = http.NewServeMux() 21 | server = &http.Server{ 22 | Addr: s.conf.StatListen, 23 | Handler: serveMux, 24 | // TODO read/write timeout 25 | } 26 | ) 27 | s.info = &stat.Info{ 28 | Ver: Ver, 29 | GitSHA1: GitSHA1, 30 | StartTime: time.Now(), 31 | Stats: &stat.Stats{}, 32 | } 33 | go s.statproc() 34 | serveMux.HandleFunc("/info", s.stat) 35 | if err = server.Serve(s.statSvr); err != nil { 36 | log.Errorf("server.Serve() error(%v)", err) 37 | } 38 | log.Info("http stat stop") 39 | return 40 | } 41 | 42 | func (s *Server) stat(wr http.ResponseWriter, r *http.Request) { 43 | if r.Method != "GET" { 44 | http.Error(wr, "Method Not Allowed", http.StatusMethodNotAllowed) 45 | return 46 | } 47 | var ( 48 | err error 49 | data []byte 50 | v *volume.Volume 51 | volumes = make([]*volume.Volume, 0, len(s.store.Volumes)) 52 | res = map[string]interface{}{"ret": errors.RetOK} 53 | ) 54 | for _, v = range s.store.Volumes { 55 | volumes = append(volumes, v) 56 | } 57 | res["server"] = s.info 58 | res["volumes"] = volumes 59 | res["free_volumes"] = s.store.FreeVolumes 60 | if data, err = json.Marshal(res); err == nil { 61 | if _, err = wr.Write(data); err != nil { 62 | log.Errorf("wr.Write() error(%v)", err) 63 | } 64 | } else { 65 | log.Errorf("json.Marshal() error(%v)", err) 66 | } 67 | return 68 | } 69 | 70 | // statproc stat the store. 71 | func (s *Server) statproc() { 72 | var ( 73 | v *volume.Volume 74 | olds *stat.Stats 75 | news = new(stat.Stats) 76 | ) 77 | for { 78 | olds = s.info.Stats 79 | *news = *olds 80 | s.info.Stats = news // use news instead, for current display 81 | olds.Reset() 82 | for _, v = range s.store.Volumes { 83 | v.Stats.Calc() 84 | olds.Merge(v.Stats) 85 | } 86 | olds.Calc() 87 | s.info.Stats = olds 88 | time.Sleep(statDuration) 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /store/index/index_test.go: -------------------------------------------------------------------------------- 1 | package index 2 | 3 | import ( 4 | "bfs/libs/errors" 5 | "bfs/store/conf" 6 | "bfs/store/needle" 7 | "os" 8 | "testing" 9 | "time" 10 | ) 11 | 12 | var ( 13 | testConf = &conf.Config{ 14 | NeedleMaxSize: 4 * 1024 * 1024, 15 | BlockMaxSize: needle.Size(4 * 1024 * 1024), 16 | Index: &conf.Index{ 17 | BufferSize: 4 * 1024 * 1024, 18 | MergeDelay: conf.Duration{10 * time.Second}, 19 | MergeWrite: 5, 20 | RingBuffer: 10, 21 | SyncWrite: 10, 22 | Syncfilerange: true, 23 | }, 24 | } 25 | ) 26 | 27 | func TestIndex(t *testing.T) { 28 | var ( 29 | i *Indexer 30 | err error 31 | noffset uint32 32 | file = "../test/test.idx" 33 | needles = make(map[int64]int64) 34 | ) 35 | os.Remove(file) 36 | defer os.Remove(file) 37 | if i, err = NewIndexer(file, testConf); err != nil { 38 | t.Errorf("NewIndexer() error(%v)", err) 39 | t.FailNow() 40 | } 41 | i.Close() 42 | // test closed 43 | if err = i.Add(1, 1, 8); err != errors.ErrIndexClosed { 44 | t.Errorf("Add() error(%v)", err) 45 | t.FailNow() 46 | } 47 | // test open 48 | if err = i.Open(); err != nil { 49 | t.Errorf("Open() error(%v)", err) 50 | t.FailNow() 51 | } 52 | defer i.Close() 53 | // test add 54 | if err = i.Add(1, 1, 8); err != nil { 55 | t.Errorf("Add() error(%v)", err) 56 | t.FailNow() 57 | } 58 | if err = i.Add(2, 2, 8); err != nil { 59 | t.Errorf("Add() error(%v)", err) 60 | t.FailNow() 61 | } 62 | if err = i.Add(5, 3, 8); err != nil { 63 | t.Errorf("Add() error(%v)", err) 64 | t.FailNow() 65 | } 66 | if err = i.Add(6, 4, 8); err != nil { 67 | t.Errorf("Add() error(%v)", err) 68 | t.FailNow() 69 | } 70 | i.Signal() 71 | time.Sleep(1 * time.Second) 72 | i.Flush() 73 | // test recovery 74 | if err = i.Recovery(func(ix *Index) error { 75 | needles[ix.Key] = needle.NewCache(ix.Offset, ix.Size) 76 | noffset = ix.Offset + needle.NeedleOffset(int64(ix.Size)) 77 | return nil 78 | }); err != nil { 79 | t.Errorf("Recovery() error(%v)", err) 80 | t.FailNow() 81 | } 82 | // add 4 index, start with 5 83 | if noffset != 5 { 84 | t.Errorf("noffset: %d not match", noffset) 85 | t.FailNow() 86 | } 87 | if o, s := needle.Cache(needles[1]); o != 1 && s != 8 { 88 | t.Error("needle cache not match") 89 | t.FailNow() 90 | } 91 | if o, s := needle.Cache(needles[2]); o != 2 && s != 8 { 92 | t.Error("needle cache not match") 93 | t.FailNow() 94 | } 95 | if o, s := needle.Cache(needles[5]); o != 3 && s != 8 { 96 | t.Error("needle cache not match") 97 | t.FailNow() 98 | } 99 | if o, s := needle.Cache(needles[6]); o != 4 && s != 8 { 100 | t.Error("needle cache not match") 101 | t.FailNow() 102 | } 103 | // test write 104 | if err = i.Write(10, 5, 8); err != nil { 105 | t.Error("Write() error(%v)", err) 106 | t.FailNow() 107 | } 108 | if err = i.Flush(); err != nil { 109 | t.Error("Flush() error(%v)", err) 110 | t.FailNow() 111 | } 112 | // test recovery 113 | noffset = 0 114 | if err = i.Recovery(func(ix *Index) error { 115 | needles[ix.Key] = needle.NewCache(ix.Offset, ix.Size) 116 | noffset = ix.Offset + needle.NeedleOffset(int64(ix.Size)) 117 | return nil 118 | }); err != nil { 119 | t.Errorf("Recovery() error(%v)", err) 120 | t.FailNow() 121 | } 122 | // add 5 index, start with 6 123 | if noffset != 6 { 124 | t.Errorf("noffset: %d not match", noffset) 125 | t.FailNow() 126 | } 127 | if o, s := needle.Cache(needles[10]); o != 5 && s != 8 { 128 | t.Error("needle.Value(1) not match") 129 | t.FailNow() 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /store/index/ring.go: -------------------------------------------------------------------------------- 1 | package index 2 | 3 | import ( 4 | "bfs/libs/errors" 5 | ) 6 | 7 | type Ring struct { 8 | // read 9 | rn int64 10 | rp int 11 | // write 12 | wn int64 13 | wp int 14 | // info 15 | num int 16 | data []Index 17 | } 18 | 19 | func NewRing(num int) *Ring { 20 | r := new(Ring) 21 | r.data = make([]Index, num) 22 | r.num = num 23 | return r 24 | } 25 | 26 | func (r *Ring) Init(num int) { 27 | r.data = make([]Index, num) 28 | r.num = num 29 | } 30 | 31 | func (r *Ring) Get() (index *Index, err error) { 32 | if r.wn == r.rn { 33 | return nil, errors.ErrRingEmpty 34 | } 35 | index = &r.data[r.rp] 36 | return 37 | } 38 | 39 | func (r *Ring) GetAdv() { 40 | if r.rp++; r.rp >= r.num { 41 | r.rp = 0 42 | } 43 | r.rn++ 44 | //if Conf.Debug { 45 | // log.Debug("ring rn: %d, rp: %d", r.rn, r.rp) 46 | //} 47 | } 48 | 49 | func (r *Ring) Set() (index *Index, err error) { 50 | if r.Buffered() >= r.num { 51 | return nil, errors.ErrRingFull 52 | } 53 | index = &r.data[r.wp] 54 | return 55 | } 56 | 57 | func (r *Ring) SetAdv() { 58 | if r.wp++; r.wp >= r.num { 59 | r.wp = 0 60 | } 61 | r.wn++ 62 | //if Conf.Debug { 63 | // log.Debug("ring wn: %d, wp: %d", r.wn, r.wp) 64 | //} 65 | } 66 | 67 | func (r *Ring) Buffered() int { 68 | return int(r.wn - r.rn) 69 | } 70 | 71 | func (r *Ring) Reset() { 72 | r.rn = 0 73 | r.rp = 0 74 | r.wn = 0 75 | r.wp = 0 76 | } 77 | -------------------------------------------------------------------------------- /store/index/ring_test.go: -------------------------------------------------------------------------------- 1 | package index 2 | 3 | import ( 4 | "bfs/libs/errors" 5 | "testing" 6 | ) 7 | 8 | func TestRing(t *testing.T) { 9 | r := NewRing(3) 10 | p0, err := r.Set() 11 | if err != nil { 12 | t.Error(err) 13 | t.FailNow() 14 | } 15 | p0.Key = 10 16 | r.SetAdv() 17 | p1, err := r.Set() 18 | if err != nil { 19 | t.Error(err) 20 | t.FailNow() 21 | } 22 | p1.Key = 11 23 | r.SetAdv() 24 | p2, err := r.Set() 25 | if err != nil { 26 | t.Error(err) 27 | t.FailNow() 28 | } 29 | p2.Key = 12 30 | r.SetAdv() 31 | p3, err := r.Set() 32 | if err != errors.ErrRingFull || p3 != nil { 33 | t.Error(err) 34 | t.FailNow() 35 | } 36 | p0, err = r.Get() 37 | if err != nil && p0.Key != 10 { 38 | t.Error(err) 39 | t.FailNow() 40 | } 41 | r.GetAdv() 42 | p1, err = r.Get() 43 | if err != nil && p1.Key != 11 { 44 | t.Error(err) 45 | t.FailNow() 46 | } 47 | r.GetAdv() 48 | p2, err = r.Get() 49 | if err != nil && p2.Key != 12 { 50 | t.Error(err) 51 | t.FailNow() 52 | } 53 | r.GetAdv() 54 | p3, err = r.Get() 55 | if err != errors.ErrRingEmpty || p3 != nil { 56 | t.Error(err) 57 | t.FailNow() 58 | } 59 | p0, err = r.Set() 60 | if err != nil { 61 | t.Error(err) 62 | t.FailNow() 63 | } 64 | p0.Key = 10 65 | r.SetAdv() 66 | p1, err = r.Set() 67 | if err != nil { 68 | t.Error(err) 69 | t.FailNow() 70 | } 71 | p1.Key = 11 72 | r.SetAdv() 73 | p2, err = r.Set() 74 | if err != nil { 75 | t.Error(err) 76 | t.FailNow() 77 | } 78 | p2.Key = 12 79 | r.SetAdv() 80 | p3, err = r.Set() 81 | if err != errors.ErrRingFull || p3 != nil { 82 | t.Error(err) 83 | t.FailNow() 84 | } 85 | p0, err = r.Get() 86 | if err != nil && p0.Key != 10 { 87 | t.Error(err) 88 | t.FailNow() 89 | } 90 | r.GetAdv() 91 | p1, err = r.Get() 92 | if err != nil && p1.Key != 11 { 93 | t.Error(err) 94 | t.FailNow() 95 | } 96 | r.GetAdv() 97 | p2, err = r.Get() 98 | if err != nil && p2.Key != 12 { 99 | t.Error(err) 100 | t.FailNow() 101 | } 102 | r.GetAdv() 103 | p3, err = r.Get() 104 | if err != errors.ErrRingEmpty || p3 != nil { 105 | t.Error(err) 106 | t.FailNow() 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /store/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bfs/store/conf" 5 | "flag" 6 | log "github.com/golang/glog" 7 | ) 8 | 9 | var ( 10 | configFile string 11 | ) 12 | 13 | func init() { 14 | flag.StringVar(&configFile, "c", "./store.toml", " set store config file path") 15 | } 16 | 17 | func main() { 18 | var ( 19 | c *conf.Config 20 | store *Store 21 | server *Server 22 | err error 23 | ) 24 | flag.Parse() 25 | defer log.Flush() 26 | log.Infof("bfs store[%s] start", Ver) 27 | defer log.Infof("bfs store[%s] stop", Ver) 28 | if c, err = conf.NewConfig(configFile); err != nil { 29 | log.Errorf("NewConfig(\"%s\") error(%v)", configFile, err) 30 | return 31 | } 32 | if store, err = NewStore(c); err != nil { 33 | return 34 | } 35 | if server, err = NewServer(store, c); err != nil { 36 | return 37 | } 38 | if err = store.SetZookeeper(); err != nil { 39 | return 40 | } 41 | log.Infof("wait signal...") 42 | StartSignal(store, server) 43 | return 44 | } 45 | -------------------------------------------------------------------------------- /store/needle/cache.go: -------------------------------------------------------------------------------- 1 | package needle 2 | 3 | const ( 4 | _cacheOffsetBit = 32 5 | // del offset 6 | CacheDelOffset = uint32(0) 7 | ) 8 | 9 | // NeedleCache needle meta data in memory. 10 | // high 32bit = Offset 11 | // low 32 bit = Size 12 | // ---------------- 13 | // | int64 | 14 | // ---------------- 15 | // | 32bit | 32bit | 16 | // | offset | size | 17 | // ---------------- 18 | 19 | // NewCache new a needle cache. 20 | func NewCache(offset uint32, size int32) int64 { 21 | return int64(offset)<<_cacheOffsetBit + int64(size) 22 | } 23 | 24 | // Cache get needle cache data. 25 | // return offset, size 26 | func Cache(n int64) (uint32, int32) { 27 | return uint32(n >> _cacheOffsetBit), int32(n) 28 | } 29 | -------------------------------------------------------------------------------- /store/needle/cache_test.go: -------------------------------------------------------------------------------- 1 | package needle 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestCache(t *testing.T) { 8 | var nc = NewCache(134, 1064) 9 | if offset, size := Cache(nc); offset != 134 || size != 1064 { 10 | t.FailNow() 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /store/needle/needle_test.go: -------------------------------------------------------------------------------- 1 | package needle 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "hash/crc32" 7 | "testing" 8 | ) 9 | 10 | func TestNeedle(t *testing.T) { 11 | var ( 12 | err error 13 | n, tn *Needle 14 | br *bufio.Reader 15 | data1 = []byte("tes1") 16 | checksum1 = crc32.Update(0, _crc32Table, data1) 17 | data2 = []byte("tes2") 18 | checksum2 = crc32.Update(0, _crc32Table, data2) 19 | buf = &bytes.Buffer{} 20 | ) 21 | // WriteFrom 22 | if _, err = buf.Write(data1); err != nil { 23 | t.Error(err) 24 | t.FailNow() 25 | } 26 | n = NewWriter(3, 3, 4) 27 | defer n.Close() 28 | if err = n.ReadFrom(buf); err != nil { 29 | t.Error(err) 30 | t.FailNow() 31 | } 32 | t.Log(n) 33 | tn = new(Needle) 34 | tn.buffer = n.Buffer() 35 | // Parse 36 | if err = tn.Parse(); err != nil { 37 | t.Error(err) 38 | t.FailNow() 39 | } 40 | t.Log(n) 41 | compareNeedle(t, tn, 3, 3, data1, FlagOK, checksum1) 42 | buf.Write(data2) 43 | n = NewWriter(4, 4, 4) 44 | defer n.Close() 45 | if err = n.ReadFrom(buf); err != nil { 46 | t.Error(err) 47 | t.FailNow() 48 | } 49 | tn = new(Needle) 50 | tn.buffer = n.Buffer() 51 | if err = tn.Parse(); err != nil { 52 | t.Error(err) 53 | t.FailNow() 54 | } 55 | compareNeedle(t, tn, 4, 4, data2, FlagOK, checksum2) 56 | // ParseFrom 57 | if _, err = buf.Write(n.Buffer()); err != nil { 58 | t.Error(err) 59 | t.FailNow() 60 | } 61 | br = bufio.NewReader(buf) 62 | tn = new(Needle) 63 | if err = tn.ParseFrom(br); err != nil { 64 | t.Error(err) 65 | t.FailNow() 66 | } 67 | t.Log(tn) 68 | compareNeedle(t, tn, 4, 4, data2, FlagOK, checksum2) 69 | } 70 | 71 | func TestAlign(t *testing.T) { 72 | var i, m int32 73 | i = 1 74 | m = (i-1)/PaddingSize + 1 75 | if align(i) != m*PaddingSize { 76 | t.Errorf("align: %d != %d", align(i), m*PaddingSize) 77 | t.FailNow() 78 | } 79 | i = 2 80 | m = (i-1)/PaddingSize + 1 81 | if align(i) != m*PaddingSize { 82 | t.Errorf("align: %d != %d", align(i), m*PaddingSize) 83 | t.FailNow() 84 | } 85 | i = 3 86 | m = (i-1)/PaddingSize + 1 87 | if align(i) != m*PaddingSize { 88 | t.Errorf("align: %d != %d", align(i), m*PaddingSize) 89 | t.FailNow() 90 | } 91 | i = 4 92 | m = (i-1)/PaddingSize + 1 93 | if align(i) != m*PaddingSize { 94 | t.Errorf("align: %d != %d", align(i), m*PaddingSize) 95 | t.FailNow() 96 | } 97 | i = 5 98 | m = (i-1)/PaddingSize + 1 99 | if align(i) != m*PaddingSize { 100 | t.Errorf("align: %d != %d", align(i), m*PaddingSize) 101 | t.FailNow() 102 | } 103 | i = 6 104 | m = (i-1)/PaddingSize + 1 105 | if align(i) != m*PaddingSize { 106 | t.Errorf("align: %d != %d", align(i), m*PaddingSize) 107 | t.FailNow() 108 | } 109 | i = 7 110 | m = (i-1)/PaddingSize + 1 111 | if align(i) != m*PaddingSize { 112 | t.Errorf("align: %d != %d", align(i), m*PaddingSize) 113 | t.FailNow() 114 | } 115 | i = 8 116 | m = (i-1)/PaddingSize + 1 117 | if align(i) != m*PaddingSize { 118 | t.Errorf("align: %d != %d", align(i), m*PaddingSize) 119 | t.FailNow() 120 | } 121 | } 122 | 123 | func TestNeedleOffset(t *testing.T) { 124 | var ( 125 | offset int64 126 | noffset uint32 127 | ) 128 | offset = 32 129 | if noffset = NeedleOffset(offset); noffset != uint32(offset/int64(PaddingSize)) { 130 | t.Errorf("noffset: %d not match", noffset) 131 | t.FailNow() 132 | } 133 | offset = 48 134 | if noffset = NeedleOffset(offset); noffset != uint32(offset/int64(PaddingSize)) { 135 | t.Errorf("noffset: %d not match", noffset) 136 | t.FailNow() 137 | } 138 | offset = 8 139 | if noffset = NeedleOffset(offset); noffset != uint32(offset/int64(PaddingSize)) { 140 | t.Errorf("noffset: %d not match", noffset) 141 | t.FailNow() 142 | } 143 | } 144 | 145 | func TestBlockOffset(t *testing.T) { 146 | var ( 147 | offset int64 148 | noffset uint32 149 | ) 150 | noffset = 1 151 | if offset = BlockOffset(noffset); offset != int64(noffset*PaddingSize) { 152 | t.Errorf("offset: %d not match", offset) 153 | t.FailNow() 154 | } 155 | noffset = 2 156 | if offset = BlockOffset(noffset); offset != int64(noffset*PaddingSize) { 157 | t.Errorf("offset: %d not match", offset) 158 | t.FailNow() 159 | } 160 | noffset = 4 161 | if offset = BlockOffset(noffset); offset != int64(noffset*PaddingSize) { 162 | t.Errorf("offset: %d not match", offset) 163 | t.FailNow() 164 | } 165 | } 166 | 167 | func TestSize(t *testing.T) { 168 | if Size(4) != 40 { 169 | t.FailNow() 170 | } 171 | } 172 | -------------------------------------------------------------------------------- /store/needle/needles.go: -------------------------------------------------------------------------------- 1 | package needle 2 | 3 | import ( 4 | "bfs/libs/errors" 5 | "io" 6 | ) 7 | 8 | // Needles is needle list. 9 | type Needles struct { 10 | rn int 11 | wn int 12 | Num int 13 | needles []Needle 14 | TotalSize int32 15 | } 16 | 17 | // NewNeedles new a needles. 18 | func NewNeedles(num int) *Needles { 19 | var ns = new(Needles) 20 | ns.Num = num 21 | ns.needles = make([]Needle, num) 22 | return ns 23 | } 24 | 25 | // ReadFrom Write needle from io.Reader into buffer. 26 | func (ns *Needles) ReadFrom(key int64, cookie, size int32, rd io.Reader) (err error) { 27 | if ns.wn >= ns.Num { 28 | return errors.ErrNeedleFull 29 | } 30 | var n = &ns.needles[ns.wn] 31 | n.InitWriter(key, cookie, size) 32 | if err = n.ReadFrom(rd); err != nil { 33 | n.Close() 34 | return 35 | } 36 | ns.wn++ 37 | ns.TotalSize += n.TotalSize 38 | return 39 | } 40 | 41 | // Next get a needle from needles. 42 | func (ns *Needles) Next() (n *Needle) { 43 | if ns.rn >= ns.wn { 44 | return nil 45 | } 46 | n = &ns.needles[ns.rn] 47 | ns.rn++ 48 | return 49 | } 50 | 51 | func (ns *Needles) Close() { 52 | for i := 0; i < ns.wn; i++ { 53 | ns.needles[i].Close() 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /store/needle/needles_test.go: -------------------------------------------------------------------------------- 1 | package needle 2 | 3 | import ( 4 | "bfs/libs/errors" 5 | "bytes" 6 | "hash/crc32" 7 | "testing" 8 | ) 9 | 10 | func TestNeedles(t *testing.T) { 11 | var ( 12 | err error 13 | tn *Needle 14 | data1 = []byte("tes1") 15 | checksum1 = crc32.Update(0, _crc32Table, data1) 16 | data2 = []byte("tes2") 17 | checksum2 = crc32.Update(0, _crc32Table, data2) 18 | ns = NewNeedles(2) 19 | buf = &bytes.Buffer{} 20 | ) 21 | if _, err = buf.Write(data1); err != nil { 22 | t.FailNow() 23 | } 24 | if err = ns.ReadFrom(1, 1, 4, buf); err != nil { 25 | t.FailNow() 26 | } 27 | if _, err = buf.Write(data2); err != nil { 28 | t.FailNow() 29 | } 30 | if err = ns.ReadFrom(2, 2, 4, buf); err != nil { 31 | t.FailNow() 32 | } 33 | tn = new(Needle) 34 | tn.buffer = ns.Next().Buffer() 35 | if err = tn.Parse(); err != nil { 36 | t.FailNow() 37 | } 38 | t.Log(tn) 39 | compareNeedle(t, tn, 1, 1, data1, FlagOK, checksum1) 40 | tn = new(Needle) 41 | tn.buffer = ns.Next().Buffer() 42 | if err = tn.Parse(); err != nil { 43 | t.FailNow() 44 | } 45 | t.Log(tn) 46 | compareNeedle(t, tn, 2, 2, data2, FlagOK, checksum2) 47 | if err = ns.ReadFrom(3, 3, 4, buf); err != errors.ErrNeedleFull { 48 | t.FailNow() 49 | } 50 | if tn = ns.Next(); tn != nil { 51 | t.FailNow() 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /store/needle/test_test.go: -------------------------------------------------------------------------------- 1 | package needle 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | ) 7 | 8 | func compareNeedle(t *testing.T, n *Needle, key int64, cookie int32, data []byte, flag byte, checksum uint32) { 9 | if n.Key != key || n.Cookie != cookie || !bytes.Equal(n.Data, data) || n.Flag != flag || n.Checksum != checksum { 10 | t.Errorf("not match: %s, %d, %d, %d", n, key, cookie, checksum) 11 | t.FailNow() 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /store/needle/utils.go: -------------------------------------------------------------------------------- 1 | package needle 2 | 3 | // NeedleOffset convert offset to needle offset. 4 | func NeedleOffset(offset int64) uint32 { 5 | return uint32(offset / PaddingSize) 6 | } 7 | 8 | // BlockOffset get super block file offset. 9 | func BlockOffset(offset uint32) int64 { 10 | return int64(offset) * PaddingSize 11 | } 12 | 13 | // align get aligned size. 14 | func align(d int32) int32 { 15 | return (d + _paddingAlign) & ^_paddingAlign 16 | } 17 | 18 | // Size get a needle size with meta data. 19 | func Size(n int) int { 20 | return int(align(_headerSize + int32(n) + _footerSize)) 21 | } 22 | -------------------------------------------------------------------------------- /store/os/fadvise_darwin.go: -------------------------------------------------------------------------------- 1 | // +build darwin 2 | package os 3 | 4 | const ( 5 | POSIX_FADV_NORMAL = 0 6 | POSIX_FADV_SEQUENTIAL = 0 7 | POSIX_FADV_RANDOM = 0 8 | POSIX_FADV_NOREUSE = 0 9 | POSIX_FADV_WILLNEED = 0 10 | POSIX_FADV_DONTNEED = 0 11 | ) 12 | 13 | func Fadvise(fd uintptr, off int64, len int64, advise int) (err error) { 14 | return 15 | } 16 | -------------------------------------------------------------------------------- /store/os/fadvise_linux.go: -------------------------------------------------------------------------------- 1 | // +build linux 2 | package os 3 | 4 | /* 5 | #define _XOPEN_SOURCE 600 6 | #include 7 | #include 8 | */ 9 | import "C" 10 | 11 | import ( 12 | "syscall" 13 | ) 14 | 15 | const ( 16 | POSIX_FADV_NORMAL = int(C.POSIX_FADV_NORMAL) 17 | POSIX_FADV_SEQUENTIAL = int(C.POSIX_FADV_SEQUENTIAL) 18 | POSIX_FADV_RANDOM = int(C.POSIX_FADV_RANDOM) 19 | POSIX_FADV_NOREUSE = int(C.POSIX_FADV_NOREUSE) 20 | POSIX_FADV_WILLNEED = int(C.POSIX_FADV_WILLNEED) 21 | POSIX_FADV_DONTNEED = int(C.POSIX_FADV_DONTNEED) 22 | ) 23 | 24 | func Fadvise(fd uintptr, off int64, size int64, advise int) (err error) { 25 | var errno int 26 | if errno = int(C.posix_fadvise(C.int(fd), C.__off_t(off), C.__off_t(size), C.int(advise))); errno != 0 { 27 | err = syscall.Errno(errno) 28 | } 29 | return 30 | } 31 | -------------------------------------------------------------------------------- /store/os/falloc_darwin.go: -------------------------------------------------------------------------------- 1 | // +build darwin 2 | package os 3 | 4 | const ( 5 | FALLOC_FL_KEEP_SIZE = 0x01 /* default is extend size */ 6 | ) 7 | 8 | func Fallocate(fd uintptr, mode uint32, off int64, len int64) (err error) { 9 | return 10 | } 11 | -------------------------------------------------------------------------------- /store/os/falloc_linux.go: -------------------------------------------------------------------------------- 1 | // +build linux 2 | package os 3 | 4 | /* 5 | #define _GNU_SOURCE 6 | #include 7 | #include 8 | */ 9 | import "C" 10 | 11 | import ( 12 | "syscall" 13 | ) 14 | 15 | const ( 16 | FALLOC_FL_KEEP_SIZE = uint32(C.FALLOC_FL_KEEP_SIZE) 17 | ) 18 | 19 | func Fallocate(fd uintptr, mode uint32, off int64, size int64) error { 20 | return syscall.Fallocate(int(fd), mode, off, size) 21 | } 22 | -------------------------------------------------------------------------------- /store/os/fdatasync_darwin.go: -------------------------------------------------------------------------------- 1 | // +build darwin 2 | package os 3 | 4 | func Fdatasync(fd uintptr) (err error) { 5 | return 6 | } 7 | -------------------------------------------------------------------------------- /store/os/fdatasync_linux.go: -------------------------------------------------------------------------------- 1 | // +build linux 2 | package os 3 | 4 | import ( 5 | "syscall" 6 | ) 7 | 8 | func Fdatasync(fd uintptr) (err error) { 9 | return syscall.Fdatasync(int(fd)) 10 | } 11 | -------------------------------------------------------------------------------- /store/os/file.go: -------------------------------------------------------------------------------- 1 | package os 2 | 3 | import ( 4 | los "os" 5 | ) 6 | 7 | // Exist check a file exist or not. 8 | func Exist(filename string) bool { 9 | var err error 10 | _, err = los.Stat(filename) 11 | return err == nil || los.IsExist(err) 12 | } 13 | -------------------------------------------------------------------------------- /store/os/io_darwin.go: -------------------------------------------------------------------------------- 1 | // +build darwin 2 | package os 3 | 4 | const ( 5 | O_NOATIME = 0 // darwin no O_NOATIME set to O_LARGEFILE 6 | ) 7 | -------------------------------------------------------------------------------- /store/os/io_linux.go: -------------------------------------------------------------------------------- 1 | // +build linux 2 | package os 3 | 4 | import ( 5 | "syscall" 6 | ) 7 | 8 | const ( 9 | O_NOATIME = syscall.O_NOATIME 10 | ) 11 | -------------------------------------------------------------------------------- /store/os/sync_file_range_darwin.go: -------------------------------------------------------------------------------- 1 | // +build darwin 2 | package os 3 | 4 | const ( 5 | SYNC_FILE_RANGE_WAIT_BEFORE = 1 6 | SYNC_FILE_RANGE_WRITE = 2 7 | SYNC_FILE_RANGE_WAIT_AFTER = 4 8 | ) 9 | 10 | func Syncfilerange(fd uintptr, off int64, n int64, flags int) (err error) { 11 | return 12 | } 13 | -------------------------------------------------------------------------------- /store/os/sync_file_range_linux.go: -------------------------------------------------------------------------------- 1 | // +build linux 2 | package os 3 | 4 | import ( 5 | "syscall" 6 | ) 7 | 8 | const ( 9 | SYNC_FILE_RANGE_WAIT_BEFORE = 1 10 | SYNC_FILE_RANGE_WRITE = 2 11 | SYNC_FILE_RANGE_WAIT_AFTER = 4 12 | ) 13 | 14 | func Syncfilerange(fd uintptr, off int64, n int64, flags int) (err error) { 15 | return syscall.SyncFileRange(int(fd), off, n, flags) 16 | } 17 | -------------------------------------------------------------------------------- /store/signal.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | log "github.com/golang/glog" 5 | "os" 6 | "os/signal" 7 | "syscall" 8 | ) 9 | 10 | // StartSignal register signals handler. 11 | func StartSignal(store *Store, server *Server) { 12 | var ( 13 | c chan os.Signal 14 | s os.Signal 15 | ) 16 | c = make(chan os.Signal, 1) 17 | signal.Notify(c, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, 18 | syscall.SIGINT, syscall.SIGSTOP) 19 | // Block until a signal is received. 20 | for { 21 | s = <-c 22 | log.Infof("get a signal %s", s.String()) 23 | switch s { 24 | case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGSTOP, syscall.SIGINT: 25 | server.Close() 26 | store.Close() 27 | return 28 | case syscall.SIGHUP: 29 | // TODO reload 30 | //return 31 | default: 32 | return 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /store/store.toml: -------------------------------------------------------------------------------- 1 | # This is a TOML document. Boom. 2 | 3 | # store golang pprof 4 | Pprof = true 5 | PprofListen = "localhost:6060" 6 | 7 | # store stat listen 8 | StatListen = "localhost:6061" 9 | 10 | # api listen, get/upload/delete 11 | ApiListen = "localhost:6062" 12 | 13 | # admin listen, add/del volume 14 | AdminListen = "localhost:6063" 15 | 16 | # needle(pic) max size 17 | NeedleMaxSize = 10485760 18 | 19 | # max batch upload 20 | BatchMaxNum = 9 21 | 22 | [Store] 23 | # volume meta index 24 | VolumeIndex = "/tmp/volume.idx" 25 | 26 | # free volume meta index 27 | FreeVolumeIndex = "/tmp/free_volume.idx" 28 | 29 | [Volume] 30 | # sync delete operation after N delete 31 | SyncDelete = 1024 32 | 33 | # sync delete delay duration 34 | SyncDeleteDelay = "10s" 35 | 36 | [Block] 37 | # sync write operation after N write 38 | SyncWrite = 1 39 | 40 | # use new kernel syscall syncfilerange 41 | Syncfilerange = true 42 | 43 | [Index] 44 | # index bufio size 45 | BufferSize = 4096 46 | 47 | # merge delay duration 48 | MergeDelay = "10s" 49 | 50 | # merge write after N write 51 | MergeWrite = 1024 52 | 53 | # ring buffer cache 54 | RingBuffer = 10240 55 | 56 | # sync write operation after N write 57 | SyncWrite = 1024 58 | 59 | # use new kernel syscall syncfilerange 60 | Syncfilerange = true 61 | 62 | [Limit] 63 | # rate r and permits bursts of at most settings 64 | # 65 | # limit read iops speed 66 | [Limit.Read] 67 | Rate = 150.0 68 | Brust = 50 69 | # limit write iops speed 70 | [Limit.Write] 71 | Rate = 150.0 72 | Brust = 50 73 | # limit delete iops speed 74 | [Limit.Delete] 75 | Rate = 150.0 76 | Brust = 50 77 | 78 | [Zookeeper] 79 | # zookeeper root path. 80 | Root = "/rack" 81 | 82 | # store machine in which rack. 83 | Rack = "bfs-test" 84 | 85 | # serverid for store server, must unique in cluster 86 | ServerId = "47E273ED-CD3A-4D6A-94CE-554BA9B195EB" 87 | 88 | # zookeeper cluster addrs 89 | Addrs = [ 90 | "localhost:2181" 91 | ] 92 | 93 | # zookeeper heartbeat timeout. 94 | Timeout = "1s" 95 | -------------------------------------------------------------------------------- /store/store_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bfs/store/needle" 5 | "bfs/store/volume" 6 | "bfs/store/zk" 7 | "bytes" 8 | "os" 9 | "testing" 10 | ) 11 | 12 | func TestStore(t *testing.T) { 13 | var ( 14 | s *Store 15 | z *zk.Zookeeper 16 | v *volume.Volume 17 | n *needle.Needle 18 | err error 19 | buf = &bytes.Buffer{} 20 | ) 21 | os.Remove(testConf.Store.VolumeIndex) 22 | os.Remove(testConf.Store.FreeVolumeIndex) 23 | os.Remove("./test/_free_block_1") 24 | os.Remove("./test/_free_block_1.idx") 25 | os.Remove("./test/_free_block_2") 26 | os.Remove("./test/_free_block_2.idx") 27 | os.Remove("./test/_free_block_3") 28 | os.Remove("./test/_free_block_3.idx") 29 | os.Remove("./test/1_0") 30 | os.Remove("./test/1_0.idx") 31 | os.Remove("./test/1_1") 32 | os.Remove("./test/1_1.idx") 33 | os.Remove("./test/block_store_1") 34 | os.Remove("./test/block_store_1.idx") 35 | defer os.Remove(testConf.Store.VolumeIndex) 36 | defer os.Remove(testConf.Store.FreeVolumeIndex) 37 | defer os.Remove("./test/_free_block_1") 38 | defer os.Remove("./test/_free_block_1.idx") 39 | defer os.Remove("./test/_free_block_2") 40 | defer os.Remove("./test/_free_block_2.idx") 41 | defer os.Remove("./test/_free_block_3") 42 | defer os.Remove("./test/_free_block_3.idx") 43 | defer os.Remove("./test/1_0") 44 | defer os.Remove("./test/1_0.idx") 45 | defer os.Remove("./test/1_1") 46 | defer os.Remove("./test/1_1.idx") 47 | defer os.Remove("./test/block_store_1") 48 | defer os.Remove("./test/block_store_1.idx") 49 | if z, err = zk.NewZookeeper(testConf); err != nil { 50 | t.Errorf("NewZookeeper() error(%v)", err) 51 | t.FailNow() 52 | } 53 | defer z.Close() 54 | z.DelVolume(1) 55 | z.DelVolume(2) 56 | z.DelVolume(3) 57 | defer z.DelVolume(1) 58 | defer z.DelVolume(2) 59 | defer z.DelVolume(3) 60 | if s, err = NewStore(testConf); err != nil { 61 | t.Errorf("NewStore() error(%v)", err) 62 | t.FailNow() 63 | } 64 | defer s.Close() 65 | if _, err = s.AddFreeVolume(2, "./test", "./test"); err != nil { 66 | t.Errorf("s.AddFreeVolume() error(%v)", err) 67 | t.FailNow() 68 | } 69 | if v, err = s.AddVolume(1); err != nil { 70 | t.Errorf("AddVolume() error(%v)", err) 71 | t.FailNow() 72 | } 73 | if v = s.Volumes[1]; v == nil { 74 | t.Error("Volume(1) not exist") 75 | t.FailNow() 76 | } 77 | buf.WriteString("test") 78 | n = needle.NewWriter(1, 1, 4) 79 | if err = n.ReadFrom(buf); err != nil { 80 | t.Errorf("n.ReadFrom() error(%v)", err) 81 | t.FailNow() 82 | } 83 | if err = v.Write(n); err != nil { 84 | t.Errorf("v.Add(1) error(%v)", err) 85 | t.FailNow() 86 | } 87 | if _, err = v.Read(1, 1); err != nil { 88 | t.Errorf("v.WriteTo(1) error(%v)", err) 89 | t.FailNow() 90 | } 91 | if err = s.BulkVolume(2, "./test/block_store_1", "./test/block_store_1.idx"); err != nil { 92 | t.Errorf("Bulk(1) error(%v)", err) 93 | t.FailNow() 94 | } 95 | if v = s.Volumes[2]; v == nil { 96 | t.Error("Volume(2) not exist") 97 | t.FailNow() 98 | } 99 | buf.WriteString("test") 100 | n = needle.NewWriter(1, 1, 4) 101 | if err = n.ReadFrom(buf); err != nil { 102 | t.Errorf("n.ReadFrom() error(%v)", err) 103 | t.FailNow() 104 | } 105 | if err = v.Write(n); err != nil { 106 | t.Errorf("v.Add() error(%v)", err) 107 | t.FailNow() 108 | } 109 | if n, err = v.Read(1, 1); err != nil { 110 | t.Errorf("v.WriteTo(1) error(%v)", err) 111 | t.FailNow() 112 | } else { 113 | n.Close() 114 | } 115 | if err = s.CompactVolume(1); err != nil { 116 | t.Errorf("Compress(1) error(%v)", err) 117 | t.FailNow() 118 | } 119 | if v = s.Volumes[1]; v == nil { 120 | t.Error("Volume(1) not exist") 121 | t.FailNow() 122 | } 123 | if n, err = v.Read(1, 1); err != nil { 124 | t.Errorf("v.WriteTo(1) error(%v)", err) 125 | t.FailNow() 126 | } else { 127 | n.Close() 128 | } 129 | s.DelVolume(1) 130 | if v = s.Volumes[1]; v != nil { 131 | t.Error(err) 132 | t.FailNow() 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /store/test/1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Terry-Mao/bfs/6a964012975171345fcb931ea9183ee58666748e/store/test/1.jpg -------------------------------------------------------------------------------- /store/test/10.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Terry-Mao/bfs/6a964012975171345fcb931ea9183ee58666748e/store/test/10.jpg -------------------------------------------------------------------------------- /store/test/2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Terry-Mao/bfs/6a964012975171345fcb931ea9183ee58666748e/store/test/2.jpg -------------------------------------------------------------------------------- /store/test/3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Terry-Mao/bfs/6a964012975171345fcb931ea9183ee58666748e/store/test/3.jpg -------------------------------------------------------------------------------- /store/test/4.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Terry-Mao/bfs/6a964012975171345fcb931ea9183ee58666748e/store/test/4.jpg -------------------------------------------------------------------------------- /store/test/5.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Terry-Mao/bfs/6a964012975171345fcb931ea9183ee58666748e/store/test/5.jpg -------------------------------------------------------------------------------- /store/test/6.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Terry-Mao/bfs/6a964012975171345fcb931ea9183ee58666748e/store/test/6.jpg -------------------------------------------------------------------------------- /store/test/7.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Terry-Mao/bfs/6a964012975171345fcb931ea9183ee58666748e/store/test/7.jpg -------------------------------------------------------------------------------- /store/test/8.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Terry-Mao/bfs/6a964012975171345fcb931ea9183ee58666748e/store/test/8.jpg -------------------------------------------------------------------------------- /store/test/9.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Terry-Mao/bfs/6a964012975171345fcb931ea9183ee58666748e/store/test/9.jpg -------------------------------------------------------------------------------- /store/test/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # add free volume 4 | curl -d "n=2&bdir=/tmp&idir=/tmp" http://localhost:6063/add_free_volume 5 | 6 | # add volume 7 | curl -d "vid=1" http://localhost:6063/add_volume 8 | curl -d "vid=2" http://localhost:6063/add_volume 9 | 10 | # uploads 11 | curl -F 'file=@"./3.jpg"' -F 'file=@"./4.jpg"' -F "vid=1" -F "keys=13" -F "keys=14" -F "cookies=13" -F "cookies=14" http://localhost:6062/uploads 12 | 13 | # upload 14 | for i in {1..10}; do curl -F 'file=@"./'$i'.jpg"' -F "vid=2" -F "key=$i" -F "cookie=$i" http://localhost:6062/upload; done 15 | 16 | # del 17 | curl -d "key=13&vid=1" http://localhost:6062/del 18 | 19 | # get 20 | curl "http://localhost:6062/get?key=5&cookie=5&vid=2" 21 | -------------------------------------------------------------------------------- /store/test_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bfs/store/conf" 5 | "bfs/store/needle" 6 | "time" 7 | ) 8 | 9 | var ( 10 | testConf = &conf.Config{ 11 | Pprof: false, 12 | AdminListen: "localhost:6063", 13 | ApiListen: "localhost:6064", 14 | StatListen: "localhost:6065", 15 | NeedleMaxSize: 4 * 1024 * 1024, 16 | BlockMaxSize: needle.Size(4 * 1024 * 1024), 17 | BatchMaxNum: 16, 18 | Zookeeper: &conf.Zookeeper{ 19 | Root: "/rack", 20 | Rack: "rack-a", 21 | ServerId: "store-a", 22 | Addrs: []string{"localhost:2181"}, 23 | Timeout: conf.Duration{time.Second}, 24 | }, 25 | Store: &conf.Store{ 26 | VolumeIndex: "./test/volume.idx", 27 | FreeVolumeIndex: "./test/free_volume.idx", 28 | }, 29 | Volume: &conf.Volume{ 30 | SyncDelete: 10, 31 | SyncDeleteDelay: conf.Duration{10 * time.Second}, 32 | }, 33 | Block: &conf.Block{ 34 | BufferSize: 4 * 1024 * 1024, 35 | SyncWrite: 1024, 36 | Syncfilerange: true, 37 | }, 38 | Index: &conf.Index{ 39 | BufferSize: 4 * 1024 * 1024, 40 | MergeDelay: conf.Duration{10 * time.Second}, 41 | MergeWrite: 5, 42 | RingBuffer: 10, 43 | SyncWrite: 10, 44 | Syncfilerange: true, 45 | }, 46 | Limit: &conf.Limit{ 47 | Read: &conf.Rate{ 48 | Rate: 150.0, 49 | Brust: 200, 50 | }, 51 | Write: &conf.Rate{ 52 | Rate: 150.0, 53 | Brust: 200, 54 | }, 55 | Delete: &conf.Rate{ 56 | Rate: 150.0, 57 | Brust: 200, 58 | }, 59 | }, 60 | } 61 | ) 62 | 63 | type testRet struct { 64 | Ret int `json:"ret"` 65 | } 66 | -------------------------------------------------------------------------------- /store/ver.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | const ( 4 | Ver = "0.1" 5 | GitSHA1 = "42dff06" 6 | ) 7 | -------------------------------------------------------------------------------- /store/zk/zk_test.go: -------------------------------------------------------------------------------- 1 | package zk 2 | 3 | import ( 4 | "bfs/store/conf" 5 | "fmt" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | var ( 11 | testConf = &conf.Config{ 12 | Zookeeper: &conf.Zookeeper{ 13 | Root: "/rack", 14 | Rack: "rack-a", 15 | ServerId: "store-a", 16 | Addrs: []string{"localhost:2181"}, 17 | Timeout: conf.Duration{time.Second}, 18 | }, 19 | } 20 | ) 21 | 22 | func volumeMeta(bfile, ifile string, id int32) []byte { 23 | return []byte(fmt.Sprintf("%s,%s,%d", bfile, ifile, id)) 24 | } 25 | 26 | func TestZookeeper(t *testing.T) { 27 | var ( 28 | zk *Zookeeper 29 | err error 30 | lines []string 31 | bfile = "./test/hijohn_1" 32 | ifile = "./test/hijohn_1.idx" 33 | ) 34 | if zk, err = NewZookeeper(testConf); err != nil { 35 | t.Errorf("Newzookeeper() error(%v)", err) 36 | t.FailNow() 37 | } 38 | zk.DelVolume(1) 39 | zk.DelVolume(2) 40 | if err = zk.AddVolume(1, volumeMeta(bfile, ifile, 1)); err != nil { 41 | t.Errorf("zk.AddVolume() error(%v)", err) 42 | t.FailNow() 43 | } 44 | if err = zk.AddVolume(2, volumeMeta(bfile, ifile, 2)); err != nil { 45 | t.Errorf("zk.AddVolume() error(%v)", err) 46 | t.FailNow() 47 | } 48 | if lines, err = zk.Volumes(); err != nil { 49 | t.Errorf("zk.Volumes() error(%v)", err) 50 | t.FailNow() 51 | } 52 | if len(lines) != 2 || lines[0] != fmt.Sprintf("%s,%s,%d", bfile, ifile, 1) || lines[1] != fmt.Sprintf("%s,%s,%d", bfile, ifile, 2) { 53 | t.FailNow() 54 | } 55 | } 56 | --------------------------------------------------------------------------------