├── pkg ├── gunkan-index-proto │ └── .gitignore └── gunkan │ ├── sanity.go │ ├── index_client_api.go │ ├── errors.go │ ├── http_client_api.go │ ├── blob_client_api.go │ ├── part_client_api.go │ ├── balancer.go │ ├── const.go │ ├── ordering_test.go │ ├── catalog.go │ ├── log.go │ ├── index_key.go │ ├── balancer_simple.go │ ├── index_client_grpc.go │ ├── http_client_direct.go │ ├── part.go │ ├── blob.go │ ├── catalog_consul.go │ ├── index_client_pooled.go │ ├── part_client_direct.go │ └── blob_client_direct.go ├── AUTHORS.md ├── CONTRIBUTING.md ├── .gitmodules ├── .gitignore ├── internal ├── cmd-data-gate │ ├── const.go │ ├── cmd.go │ ├── service.go │ └── handlers.go ├── cmd-blob-store-fs │ ├── const.go │ ├── cmd.go │ ├── service.go │ ├── handlers.go │ └── repository.go ├── cmd-blob-client │ ├── cmd_srv_info.go │ ├── cmd_srv_health.go │ ├── cmd_srv_metrics.go │ ├── cmd.go │ ├── cmd_get.go │ ├── cmd_list.go │ ├── cmd_put.go │ └── cmd_del.go ├── cmd-index-gate │ ├── cmd.go │ └── service.go ├── cmd-index-store-rocksdb │ ├── cmd.go │ └── service.go ├── helpers-grpc │ └── cnx.go ├── helpers-http │ └── ghttp.go └── cmd-index-client │ └── cmd.go ├── cmd ├── gunkan-data-gate │ └── main.go ├── gunkan-index-gate │ └── main.go ├── gunkan-blob-store-fs │ └── main.go ├── gunkan-index-store-rocksdb │ └── main.go └── gunkan │ └── main.go ├── Makefile ├── INSTALL.md ├── api └── index.proto ├── .circleci └── config.yml ├── README.md ├── BLOB.md ├── ci └── run.py ├── LICENSE └── go.sum /pkg/gunkan-index-proto/.gitignore: -------------------------------------------------------------------------------- 1 | index.pb.go 2 | -------------------------------------------------------------------------------- /AUTHORS.md: -------------------------------------------------------------------------------- 1 | # The awesome Authors of Gunkan 2 | 3 | ## Individuals 4 | 5 | ## Corporate 6 | * @jfsmig, Jean-Francois Smigielski, OpenIO SAS 7 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contribute to Gunkan 2 | 3 | ## Code organisation 4 | 5 | * `./api` Protobuf descriptions of gRPC services. 6 | * `./ci` Tools used in the continuous integration process 7 | * `./cmd` One subdirectory per CLI tool of the gunkan suite 8 | * `./internal` one subdirectory per internal library 9 | * `./pkg` one subdirectory per public API -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "third_party/nng"] 2 | path = third_party/nng 3 | url = https://github.com/nanomsg/nng.git 4 | [submodule "third_party/flatbuffers"] 5 | path = third_party/flatbuffers 6 | url = https://github.com/google/flatbuffers.git 7 | [submodule "third_party/rocksdb"] 8 | path = third_party/rocksdb 9 | url = https://github.com/facebook/rocksdb.git 10 | -------------------------------------------------------------------------------- /pkg/gunkan/sanity.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package gunkan 7 | 8 | func ValidateBucketName(n string) bool { 9 | return len(n) > 0 && len(n) < 1024 10 | } 11 | 12 | func ValidateContentName(n string) bool { 13 | return len(n) > 0 && len(n) < 1024 14 | } 15 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Jetbrains IDE 2 | cmake-build-*/ 3 | .idea/ 4 | **/.idea/ 5 | 6 | # 7 | # Prerequisites 8 | *.d 9 | 10 | # Compiled Object files 11 | *.slo 12 | *.lo 13 | *.o 14 | *.obj 15 | 16 | # Precompiled Headers 17 | *.gch 18 | *.pch 19 | 20 | # Compiled Dynamic libraries 21 | *.so 22 | *.dylib 23 | *.dll 24 | 25 | # Fortran module files 26 | *.mod 27 | *.smod 28 | 29 | # Compiled Static libraries 30 | *.lai 31 | *.la 32 | *.a 33 | *.lib 34 | 35 | # Executables 36 | *.exe 37 | *.out 38 | *.app 39 | -------------------------------------------------------------------------------- /internal/cmd-data-gate/const.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package cmd_data_gate 7 | 8 | import ( 9 | "github.com/jfsmig/object-storage/pkg/gunkan" 10 | ) 11 | 12 | const ( 13 | routeList = "/v1/list" 14 | prefixData = "/v1/part/" 15 | infoString = "gunkan/data-gate-" + gunkan.VersionString 16 | ) 17 | 18 | const ( 19 | HeaderPrefixCommon = "X-gk-" 20 | HeaderNameObjectPolicy = HeaderPrefixCommon + "obj-policy" 21 | ) 22 | -------------------------------------------------------------------------------- /pkg/gunkan/index_client_api.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package gunkan 7 | 8 | import ( 9 | "context" 10 | ) 11 | 12 | type IndexClient interface { 13 | Put(ctx context.Context, key BaseKey, value string) error 14 | 15 | Get(ctx context.Context, key BaseKey) (string, error) 16 | 17 | Delete(ctx context.Context, key BaseKey) error 18 | 19 | List(ctx context.Context, marker BaseKey, max uint32) ([]string, error) 20 | } 21 | -------------------------------------------------------------------------------- /pkg/gunkan/errors.go: -------------------------------------------------------------------------------- 1 | package gunkan 2 | 3 | import ( 4 | "errors" 5 | ) 6 | 7 | var ( 8 | ErrNotFound = errors.New("404/Not-Found") 9 | ErrForbidden = errors.New("403/Forbidden") 10 | ErrAlreadyExists = errors.New("409/Conflict") 11 | ErrStorageError = errors.New("502/Backend-Error") 12 | ErrInternalError = errors.New("500/Internal Error") 13 | ) 14 | 15 | func MapCodeToError(code int) error { 16 | switch code { 17 | case 404: 18 | return ErrNotFound 19 | case 403: 20 | return ErrForbidden 21 | case 409: 22 | return ErrAlreadyExists 23 | case 200, 201, 204: 24 | return nil 25 | default: 26 | return ErrInternalError 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /pkg/gunkan/http_client_api.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package gunkan 7 | 8 | import ( 9 | "context" 10 | "net/http" 11 | ) 12 | 13 | type HttpSimpleClient struct { 14 | Endpoint string 15 | UserAgent string 16 | Http http.Client 17 | } 18 | 19 | type HttpMonitorClient interface { 20 | Info(ctx context.Context) ([]byte, error) 21 | Health(ctx context.Context) ([]byte, error) 22 | Metrics(ctx context.Context) ([]byte, error) 23 | } 24 | -------------------------------------------------------------------------------- /cmd/gunkan-data-gate/main.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package main 7 | 8 | import ( 9 | "github.com/jfsmig/object-storage/internal/cmd-data-gate" 10 | "github.com/jfsmig/object-storage/pkg/gunkan" 11 | ) 12 | 13 | func main() { 14 | rootCmd := cmd_data_gate.MainCommand() 15 | gunkan.PatchCommandLogs(rootCmd) 16 | rootCmd.Use = "gunkan-data" 17 | if err := rootCmd.Execute(); err != nil { 18 | gunkan.Logger.Fatal().Err(err).Msg("Command error") 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /cmd/gunkan-index-gate/main.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package main 7 | 8 | import ( 9 | "github.com/jfsmig/object-storage/internal/cmd-index-gate" 10 | "github.com/jfsmig/object-storage/pkg/gunkan" 11 | ) 12 | 13 | func main() { 14 | rootCmd := cmd_index_gate.MainCommand() 15 | gunkan.PatchCommandLogs(rootCmd) 16 | rootCmd.Use = "gunkan-index-gate" 17 | if err := rootCmd.Execute(); err != nil { 18 | gunkan.Logger.Fatal().Err(err).Msg("Command error") 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /cmd/gunkan-blob-store-fs/main.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package main 7 | 8 | import ( 9 | "github.com/jfsmig/object-storage/internal/cmd-blob-store-fs" 10 | "github.com/jfsmig/object-storage/pkg/gunkan" 11 | ) 12 | 13 | func main() { 14 | rootCmd := cmd_blob_store_fs.MainCommand() 15 | gunkan.PatchCommandLogs(rootCmd) 16 | rootCmd.Use = "gunkan-blobstore" 17 | if err := rootCmd.Execute(); err != nil { 18 | gunkan.Logger.Fatal().Err(err).Msg("Command error") 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /cmd/gunkan-index-store-rocksdb/main.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package main 7 | 8 | import ( 9 | "github.com/jfsmig/object-storage/internal/cmd-index-store-rocksdb" 10 | "github.com/jfsmig/object-storage/pkg/gunkan" 11 | ) 12 | 13 | func main() { 14 | rootCmd := cmd_index_store_rocksdb.MainCommand() 15 | gunkan.PatchCommandLogs(rootCmd) 16 | rootCmd.Use = "gunkan-index-store-rocksdb" 17 | if err := rootCmd.Execute(); err != nil { 18 | gunkan.Logger.Fatal().Err(err).Msg("Command error") 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /pkg/gunkan/blob_client_api.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package gunkan 7 | 8 | import ( 9 | "context" 10 | "io" 11 | ) 12 | 13 | type BlobClient interface { 14 | Put(ctx context.Context, id BlobId, data io.Reader) (string, error) 15 | PutN(ctx context.Context, id BlobId, data io.Reader, size int64) (string, error) 16 | 17 | Get(ctx context.Context, realId string) (io.ReadCloser, error) 18 | 19 | Delete(ctx context.Context, realId string) error 20 | 21 | List(ctx context.Context, max uint) ([]BlobListItem, error) 22 | ListAfter(ctx context.Context, max uint, marker string) ([]BlobListItem, error) 23 | } 24 | 25 | type BlobListItem struct { 26 | Real string 27 | Logical BlobId 28 | } 29 | -------------------------------------------------------------------------------- /internal/cmd-blob-store-fs/const.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package cmd_blob_store_fs 7 | 8 | import ( 9 | "github.com/jfsmig/object-storage/pkg/gunkan" 10 | "golang.org/x/sys/unix" 11 | ) 12 | 13 | const ( 14 | flagsCommon int = unix.O_NOATIME | unix.O_NONBLOCK | unix.O_CLOEXEC 15 | flagsRO = flagsCommon | unix.O_RDONLY 16 | flagsRW = flagsCommon | unix.O_RDWR 17 | flagsCreate = flagsRW | unix.O_EXCL | unix.O_CREAT 18 | flagsOpenDir = flagsRO | unix.O_DIRECTORY | unix.O_PATH 19 | flagsOpenRead = flagsRO 20 | ) 21 | 22 | const ( 23 | routeList = "/v1/list" 24 | prefixData = "/v1/blob/" 25 | infoString = "gunkan/blob-store-" + gunkan.VersionString 26 | ) 27 | -------------------------------------------------------------------------------- /pkg/gunkan/part_client_api.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package gunkan 7 | 8 | import ( 9 | "context" 10 | "io" 11 | ) 12 | 13 | type PartClient interface { 14 | Put(ctx context.Context, id PartId, data io.Reader) error 15 | PutN(ctx context.Context, id PartId, data io.Reader, size int64) error 16 | 17 | Get(ctx context.Context, id PartId) (io.ReadCloser, error) 18 | 19 | Delete(ctx context.Context, id PartId) error 20 | 21 | // Returns the first page of known parts in the current pod 22 | List(ctx context.Context, max uint32) ([]PartId, error) 23 | 24 | // Returns the next page of known parts in the current pod 25 | ListAfter(ctx context.Context, max uint32, id PartId) ([]PartId, error) 26 | } 27 | -------------------------------------------------------------------------------- /pkg/gunkan/balancer.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package gunkan 7 | 8 | type Balancer interface { 9 | // Returns the URL of an available Data Gate service 10 | PollDataGate() (string, error) 11 | 12 | // Returns the URL of an available Index Gate service 13 | PollIndexGate() (string, error) 14 | 15 | // Returns the URL of an available Blob Store service. 16 | PollBlobStore() (string, error) 17 | } 18 | 19 | // Returns a discovery client initiated 20 | func NewBalancerDefault() (Balancer, error) { 21 | if catalog, err := NewCatalogDefault(); err != nil { 22 | return nil, err 23 | } else if discovery, err := NewBalancerSimple(catalog); err != nil { 24 | return nil, err 25 | } else { 26 | return discovery, nil 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /pkg/gunkan/const.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package gunkan 7 | 8 | const ( 9 | VersionMajor = "0" 10 | VersionMinor = "1" 11 | VersionString = VersionMajor + "." + VersionMinor 12 | ) 13 | 14 | const ( 15 | ConsulSrvIndexGate = "gkindex-gate" 16 | ConsulSrvIndexStore = "gkindex-store" 17 | ConsulSrvDataGate = "gkdata-gate" 18 | ConsulSrvBlobStore = "gkblob-store" 19 | ) 20 | 21 | const ( 22 | // Returns a simple string containing the type of the service. 23 | RouteInfo = "/info" 24 | 25 | // Provides the feedback expected by Consul.io 26 | RouteHealth = "/health" 27 | 28 | // Provides metrics using the standard of a Prometheus Exporter 29 | // Thus also collectable with InfluxDB 30 | RouteMetrics = "/metrics" 31 | ) 32 | 33 | const ( 34 | ListHardMax = 10000 35 | ) 36 | -------------------------------------------------------------------------------- /pkg/gunkan/ordering_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package gunkan 7 | 8 | import ( 9 | "sort" 10 | "testing" 11 | ) 12 | 13 | type SetOfVersioned []KeyVersion 14 | 15 | func (s SetOfVersioned) Len() int { 16 | return len(s) 17 | } 18 | 19 | func (s SetOfVersioned) Less(i, j int) bool { 20 | return s[i].Encode() < s[j].Encode() 21 | } 22 | 23 | func (s SetOfVersioned) Swap(i, j int) { 24 | s[i], s[j] = s[j], s[i] 25 | } 26 | 27 | func TestKeyOrdering(t *testing.T) { 28 | tab := SetOfVersioned{ 29 | {"A", "plap", 4, true}, 30 | {"A", "plap", 3, true}, 31 | {"A", "plip", 3, true}, 32 | {"A", "plip", 2, false}, 33 | {"A", "plip", 1, true}, 34 | {"A", "plip", 0, true}, 35 | {"A", "plipA", 1, true}, 36 | } 37 | if !sort.IsSorted(tab) { 38 | t.Fatal() 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /pkg/gunkan/catalog.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package gunkan 7 | 8 | type Catalog interface { 9 | // Returns the list of all the Data Gate services 10 | ListDataGate() ([]string, error) 11 | 12 | // Returns the list of all the Index Gate services 13 | ListIndexGate() ([]string, error) 14 | 15 | // Returns the list of all the Blob Store services 16 | ListBlobStore() ([]string, error) 17 | 18 | // Returns the list of all the Index Store services 19 | ListIndexStore() ([]string, error) 20 | } 21 | 22 | // Returns a discovery client initiated 23 | func NewCatalogDefault() (Catalog, error) { 24 | if consul, err := GetConsulEndpoint(); err != nil { 25 | return nil, err 26 | } else if discovery, err := NewCatalogConsul(consul); err != nil { 27 | return nil, err 28 | } else { 29 | return discovery, nil 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /internal/cmd-blob-client/cmd_srv_info.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package cmd_blob_client 7 | 8 | import ( 9 | "context" 10 | "github.com/jfsmig/object-storage/pkg/gunkan" 11 | "github.com/spf13/cobra" 12 | "os" 13 | ) 14 | 15 | func SrvInfoCommand() *cobra.Command { 16 | var cfg config 17 | 18 | client := &cobra.Command{ 19 | Use: "info", 20 | Aliases: []string{"describe", "wot", "who"}, 21 | Short: "Get the service type", 22 | RunE: func(cmd *cobra.Command, args []string) error { 23 | client, err := gunkan.DialBlob(cfg.url) 24 | if err != nil { 25 | return err 26 | } 27 | body, err := client.(gunkan.HttpMonitorClient).Info(context.Background()) 28 | if err != nil { 29 | return err 30 | } 31 | _, err = os.Stdout.Write(body) 32 | return err 33 | }, 34 | } 35 | 36 | client.Flags().StringVar(&cfg.url, "url", "", "IP:PORT endpoint of the service to contact") 37 | 38 | return client 39 | } 40 | -------------------------------------------------------------------------------- /internal/cmd-blob-client/cmd_srv_health.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package cmd_blob_client 7 | 8 | import ( 9 | "context" 10 | "github.com/jfsmig/object-storage/pkg/gunkan" 11 | "github.com/spf13/cobra" 12 | "os" 13 | ) 14 | 15 | func SrvHealthCommand() *cobra.Command { 16 | var cfg config 17 | 18 | client := &cobra.Command{ 19 | Use: "health", 20 | Aliases: []string{"ruok", "check", "ping"}, 21 | Short: "Get the service type", 22 | RunE: func(cmd *cobra.Command, args []string) error { 23 | client, err := gunkan.DialBlob(cfg.url) 24 | if err != nil { 25 | return err 26 | } 27 | body, err := client.(gunkan.HttpMonitorClient).Health(context.Background()) 28 | if err != nil { 29 | return err 30 | } 31 | _, err = os.Stdout.Write(body) 32 | return err 33 | }, 34 | } 35 | 36 | client.Flags().StringVar(&cfg.url, "url", "", "IP:PORT endpoint of the service to contact") 37 | 38 | return client 39 | } 40 | -------------------------------------------------------------------------------- /internal/cmd-blob-client/cmd_srv_metrics.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package cmd_blob_client 7 | 8 | import ( 9 | "context" 10 | "github.com/jfsmig/object-storage/pkg/gunkan" 11 | "github.com/spf13/cobra" 12 | 13 | "os" 14 | ) 15 | 16 | func SrvMetricsCommand() *cobra.Command { 17 | var cfg config 18 | 19 | client := &cobra.Command{ 20 | Use: "metrics", 21 | Aliases: []string{"stats", "stat", "status"}, 22 | Short: "Get the usage statistics of a BLOB service", 23 | RunE: func(cmd *cobra.Command, args []string) error { 24 | client, err := gunkan.DialBlob(cfg.url) 25 | if err != nil { 26 | return err 27 | } 28 | body, err := client.(gunkan.HttpMonitorClient).Metrics(context.Background()) 29 | if err != nil { 30 | return err 31 | } 32 | _, err = os.Stdout.Write(body) 33 | return err 34 | }, 35 | } 36 | 37 | client.Flags().StringVar(&cfg.url, "url", "", "IP:PORT endpoint of the service to contact") 38 | 39 | return client 40 | } 41 | -------------------------------------------------------------------------------- /cmd/gunkan/main.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package main 7 | 8 | import ( 9 | "github.com/jfsmig/object-storage/internal/cmd-blob-client" 10 | "github.com/jfsmig/object-storage/internal/cmd-index-client" 11 | "github.com/jfsmig/object-storage/pkg/gunkan" 12 | "github.com/spf13/cobra" 13 | ) 14 | 15 | func main() { 16 | rootCmd := &cobra.Command{ 17 | Use: "gunkan", 18 | Short: "Manage your data and metadata on hunkan services", 19 | RunE: func(cmd *cobra.Command, args []string) error { 20 | return cobra.ErrSubCommandRequired 21 | }, 22 | } 23 | gunkan.PatchCommandLogs(rootCmd) 24 | 25 | blobCmd := cmd_blob_client.MainCommand() 26 | blobCmd.Use = "blob" 27 | blobCmd.Aliases = []string{} 28 | 29 | kvCmd := cmd_index_client.MainCommand() 30 | kvCmd.Use = "kv" 31 | kvCmd.Aliases = []string{} 32 | 33 | rootCmd.AddCommand(blobCmd) 34 | rootCmd.AddCommand(kvCmd) 35 | if err := rootCmd.Execute(); err != nil { 36 | gunkan.Logger.Fatal().Err(err).Msg("Command error") 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /pkg/gunkan/log.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package gunkan 7 | 8 | import ( 9 | "github.com/rs/zerolog" 10 | "github.com/spf13/cobra" 11 | "os" 12 | "time" 13 | ) 14 | 15 | var ( 16 | Logger = zerolog. 17 | New(zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: time.RFC3339}). 18 | With().Timestamp().Logger() 19 | 20 | flagVerbose = 0 21 | flagQuiet = false 22 | ) 23 | 24 | func PatchCommandLogs(cmd *cobra.Command) { 25 | cmd.PersistentFlags().CountVarP(&flagVerbose, "verbose", "v", "Increase the verbosity level") 26 | cmd.PersistentFlags().BoolVarP(&flagQuiet, "quiet", "q", flagQuiet, "Shut the logs") 27 | 28 | cmd.PersistentPreRun = func(cmd *cobra.Command, args []string) { 29 | if flagQuiet { 30 | zerolog.SetGlobalLevel(zerolog.Disabled) 31 | } else { 32 | switch flagVerbose { 33 | case 0: 34 | zerolog.SetGlobalLevel(zerolog.InfoLevel) 35 | case 1: 36 | zerolog.SetGlobalLevel(zerolog.DebugLevel) 37 | case 2: 38 | zerolog.SetGlobalLevel(zerolog.TraceLevel) 39 | } 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | BASE=github.com/jfsmig/object-storage 2 | GO=go 3 | PROTOC=protoc 4 | 5 | AUTO= 6 | AUTO+= pkg/gunkan-index-proto/index.pb.go 7 | 8 | all: prepare 9 | $(GO) mod download 10 | $(GO) install $(BASE)/cmd/gunkan 11 | $(GO) install $(BASE)/cmd/gunkan-data-gate 12 | $(GO) install $(BASE)/cmd/gunkan-blob-store-fs 13 | $(GO) install $(BASE)/cmd/gunkan-index-gate 14 | $(GO) install $(BASE)/cmd/gunkan-index-store-rocksdb 15 | 16 | prepare: $(AUTO) 17 | 18 | pkg/gunkan-index-proto/%.pb.go: api/index.proto 19 | $(PROTOC) -I api api/index.proto --go_out=plugins=grpc:pkg/gunkan-index-proto 20 | 21 | clean: 22 | -rm $(AUTO) 23 | 24 | .PHONY: all prepare clean test bench fmt 25 | 26 | fmt: 27 | find * -type f -name '*.go' \ 28 | | grep -v -e '_auto.go$$' -e '.pb.go$$' \ 29 | | while read F ; do dirname $$F ; done \ 30 | | sort | uniq | while read D ; do ( set -x ; cd $$D && go fmt ) done 31 | 32 | test: all 33 | find * -type f -name '*_test.go' \ 34 | | while read F ; do dirname $$F ; done \ 35 | | sort | uniq | while read D ; do ( set -x ; cd $$D && go test ) done 36 | 37 | bench: all 38 | find * -type f -name '*_test.go' \ 39 | | while read F ; do dirname $$F ; done \ 40 | | sort | uniq | while read D ; do ( set -x ; cd $$D && go -bench=. test ) done 41 | 42 | -------------------------------------------------------------------------------- /INSTALL.md: -------------------------------------------------------------------------------- 1 | # Gunkan Object Storage: Installation Manual 2 | 3 | ## Dependencies 4 | 5 | The build process requires tools and libraries: 6 | * [gRPC](https://grpc.io) and the [Go gRPC](https://github.com/grpc/grpc-go) implementation. 7 | * [Protobuf]() as part of [gRPC](https://grpc.io) 8 | * [Go >= 1.11](https://golang.org) with the support of [Go Modules](https://blog.golang.org/using-go-modules) enabled 9 | * A set of modules described in [go.mod](./go.mod) 10 | 11 | The deployment and the runtime requires additional tools: 12 | * [Consul](https://consul.io) 13 | * A decent implementation of TLS and its suite of tools, to generate certificates. 14 | 15 | ## Install from scratch 16 | 17 | Build each of the set of ``gunkan`` commands. 18 | ```shell script 19 | BASE=github.com/gunkan-io/object-storage 20 | go mod download 21 | go install ${BASE}/cmd/gunkan 22 | go install ${BASE}/cmd/gunkan-data-gate 23 | go install ${BASE}/cmd/gunkan-blob-store-fs 24 | go install ${BASE}/cmd/gunkan-index-gate 25 | go install ${BASE}/cmd/gunkan-index-store-rocksdb 26 | ``` 27 | 28 | ## Deploy a sandbox 29 | 30 | Gunkan provides [run.py](./ci/run.py), a tool to deploy test environments. 31 | 32 | Use [run.py](./ci/run.py) to spawn a minimal environment deployed if subdirectories 33 | of ``/tmp``, then hit ``Ctrl-C`` to make it exit gracefully): 34 | ```shell script 35 | ./ci/run.py 36 | ``` -------------------------------------------------------------------------------- /internal/cmd-blob-client/cmd.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package cmd_blob_client 7 | 8 | import ( 9 | "github.com/jfsmig/object-storage/pkg/gunkan" 10 | "github.com/spf13/cobra" 11 | ) 12 | 13 | func MainCommand() *cobra.Command { 14 | client := &cobra.Command{ 15 | Use: "cli", 16 | Aliases: []string{"client"}, 17 | Short: "Client of BLOB services", 18 | RunE: func(cmd *cobra.Command, args []string) error { 19 | return cobra.ErrSubCommandRequired 20 | }, 21 | } 22 | client.AddCommand(PutCommand()) 23 | client.AddCommand(GetCommand()) 24 | client.AddCommand(DelCommand()) 25 | client.AddCommand(ListCommand()) 26 | client.AddCommand(SrvInfoCommand()) 27 | client.AddCommand(SrvHealthCommand()) 28 | client.AddCommand(SrvMetricsCommand()) 29 | return client 30 | } 31 | 32 | func debug(id string, err error) { 33 | gunkan.Logger.Info().Str("id", id).Err(err) 34 | if err == nil { 35 | gunkan.Logger.Info().Str("id", id).Msg("ok") 36 | } else { 37 | gunkan.Logger.Info().Str("id", id).Err(err) 38 | } 39 | } 40 | 41 | // Common configuration for all the subcommands 42 | type config struct { 43 | url string 44 | } 45 | -------------------------------------------------------------------------------- /api/index.proto: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | syntax = "proto3"; 7 | 8 | package gunkan.index.proto; 9 | 10 | service Index { 11 | // Push a BLOB reference in the index 12 | rpc Put (PutRequest) returns (None) {} 13 | 14 | // Remove a BLOB reference from the index 15 | rpc Delete (DeleteRequest) returns (None) {} 16 | 17 | // Fetch the BLOB reference for the given Key 18 | rpc Get (GetRequest) returns (GetReply) {} 19 | 20 | // Fetch a slice of keys of BLOB references from the index 21 | rpc List (ListRequest) returns (ListReply) {} 22 | } 23 | 24 | message None { 25 | } 26 | 27 | message PutRequest { 28 | string base = 1; 29 | string key = 2; 30 | string value = 4; 31 | } 32 | 33 | message DeleteRequest { 34 | string base = 1; 35 | string key = 2; 36 | } 37 | 38 | message GetRequest { 39 | string base = 1; 40 | string key = 2; 41 | } 42 | 43 | message GetReply { 44 | uint64 version = 1; 45 | string value = 2; 46 | } 47 | 48 | message ListRequest { 49 | string base = 1; 50 | string marker = 2; 51 | uint32 max = 3; 52 | } 53 | 54 | message ListReply { 55 | repeated string items = 1; 56 | } 57 | -------------------------------------------------------------------------------- /internal/cmd-blob-client/cmd_get.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package cmd_blob_client 7 | 8 | import ( 9 | "context" 10 | "errors" 11 | "github.com/jfsmig/object-storage/pkg/gunkan" 12 | "github.com/spf13/cobra" 13 | "io" 14 | "os" 15 | ) 16 | 17 | func GetCommand() *cobra.Command { 18 | var cfg config 19 | 20 | cmd := &cobra.Command{ 21 | Use: "get", 22 | Aliases: []string{"fetch", "retrieve", "download", "dl"}, 23 | Short: "Get data from a BLOB service", 24 | RunE: func(cmd *cobra.Command, args []string) (err error) { 25 | if len(args) != 1 { 26 | return errors.New("Missing Blob ID") 27 | } 28 | client, err := gunkan.DialBlob(cfg.url) 29 | if err != nil { 30 | return err 31 | } 32 | 33 | return getOne(client, args[0]) 34 | }, 35 | } 36 | 37 | cmd.Flags().StringVar(&cfg.url, "url", "", "IP:PORT endpoint of the service to contact") 38 | 39 | return cmd 40 | } 41 | 42 | func getOne(client gunkan.BlobClient, strid string) error { 43 | var err error 44 | 45 | if _, err = gunkan.DecodeBlobId(strid); err != nil { 46 | return err 47 | } else { 48 | r, err := client.Get(context.Background(), strid) 49 | if err != nil { 50 | return err 51 | } else { 52 | defer r.Close() 53 | io.Copy(os.Stdout, r) 54 | return nil 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /internal/cmd-blob-client/cmd_list.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package cmd_blob_client 7 | 8 | import ( 9 | "context" 10 | "errors" 11 | "flag" 12 | "fmt" 13 | "github.com/jfsmig/object-storage/pkg/gunkan" 14 | "github.com/spf13/cobra" 15 | ) 16 | 17 | func ListCommand() *cobra.Command { 18 | var cfg config 19 | 20 | cmd := &cobra.Command{ 21 | Use: "list", 22 | Aliases: []string{"list"}, 23 | Short: "List items stored on a BLOB service", 24 | RunE: func(cmd *cobra.Command, args []string) (err error) { 25 | var realid string 26 | 27 | if flag.NArg() > 1 { 28 | if flag.NArg() > 2 { 29 | return errors.New("Too many BLOB id") 30 | } 31 | realid = flag.Arg(1) 32 | } 33 | 34 | client, err := gunkan.DialBlob(cfg.url) 35 | if err != nil { 36 | return err 37 | } 38 | 39 | var items []gunkan.BlobListItem 40 | if len(realid) <= 0 { 41 | items, err = client.List(context.Background(), 1000) 42 | } else { 43 | items, err = client.ListAfter(context.Background(), 1000, realid) 44 | } 45 | if err != nil { 46 | return err 47 | } 48 | for _, item := range items { 49 | fmt.Println(item) 50 | } 51 | return nil 52 | }, 53 | } 54 | 55 | cmd.Flags().StringVar(&cfg.url, "url", "", "IP:PORT endpoint of the service to contact") 56 | 57 | return cmd 58 | } 59 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | jobs: 3 | build: 4 | docker: 5 | - image: jfsmig/anustart:ci-6 6 | steps: 7 | - checkout 8 | - run: 9 | name: Golang deps 10 | command: | 11 | source /etc/profile 12 | go get github.com/google/subcommands 13 | go get github.com/gorilla/websocket 14 | go get github.com/google/flatbuffers/go 15 | go get github.com/nanomsg/mangos-v2 16 | - run: 17 | name: Build the Golang parts 18 | command: | 19 | source /etc/profile 20 | cd kv/client/golang/client 21 | go build -o gunkan-kv-client 22 | sudo install gunkan-kv-cli /usr/local/bin 23 | - run: 24 | name: Build the C++ parts 25 | command: | 26 | source /etc/profile 27 | cmake \ 28 | -D CMAKE_BUILD_TYPE=RelWithDebInfo \ 29 | -D CMAKE_INSTALL_PREFIX=/usr/local \ 30 | -D CMAKE_INSTALL_LIBDIR=/usr/local/lib \ 31 | -D NNG_INCDIR=/usr/local/include \ 32 | -D NNG_LIBDIR=/usr/local/lib \ 33 | -D FLATBUFFERS_INCDIR=/usr/local/include \ 34 | -D FLATBUFFERS_LIBDIR=/usr/local/lib \ 35 | -D ROCKSDB_INCDIR=/usr/local/include \ 36 | -D ROCKSDB_LIBDIR=/usr/local/lib \ 37 | . 38 | make 39 | sudo make install 40 | - run: 41 | name: Functional tests 42 | command: | 43 | source /etc/profile 44 | cd kv/server 45 | go test -v 46 | cd - 47 | cd blob/server 48 | make test 49 | 50 | -------------------------------------------------------------------------------- /pkg/gunkan/index_key.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package gunkan 7 | 8 | import ( 9 | "fmt" 10 | "strings" 11 | ) 12 | 13 | type BaseKey struct { 14 | Key string 15 | Base string 16 | } 17 | 18 | type ArrayOfKey []string 19 | 20 | func (v ArrayOfKey) Len() int { return len(v) } 21 | func (v ArrayOfKey) Swap(i, j int) { v[i], v[j] = v[j], v[i] } 22 | func (v ArrayOfKey) Less(i, j int) bool { 23 | return strings.Compare(v[i], v[j]) < 0 24 | } 25 | 26 | func BK(base, key string) BaseKey { 27 | return BaseKey{Base: base, Key: key} 28 | } 29 | 30 | func (n *BaseKey) Reset() { 31 | n.Key = n.Key[:] 32 | n.Base = n.Base[:] 33 | } 34 | 35 | func (n BaseKey) Encode() string { 36 | return fmt.Sprintf("%s,%s", n.Base, n.Key) 37 | } 38 | 39 | func (n *BaseKey) DecodeBytes(b []byte) error { 40 | return n.DecodeString(string(b)) 41 | } 42 | 43 | func (n *BaseKey) DecodeString(s string) error { 44 | step := parsingBase 45 | sb := strings.Builder{} 46 | sb.Grow(256) 47 | 48 | n.Reset() 49 | for _, c := range s { 50 | switch step { 51 | case parsingBase: 52 | if c == ',' { 53 | n.Base = sb.String() 54 | sb.Reset() 55 | step = parsingKey 56 | } else { 57 | sb.WriteRune(c) 58 | } 59 | case parsingKey: 60 | sb.WriteRune(c) 61 | } 62 | } 63 | 64 | switch step { 65 | case parsingBase: 66 | n.Base = sb.String() 67 | case parsingKey: 68 | n.Key = sb.String() 69 | } 70 | return nil 71 | } 72 | 73 | const ( 74 | parsingBase = iota 75 | parsingKey = iota 76 | ) 77 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Gunkan Object Storage 2 | 3 | Minimalistic object storage solution. 4 | 5 | ## Getting Started 6 | 7 | First, build and install the set of ``gunkan`` commands using our Makefile. 8 | If you have a variant of a ``make`` tool installed: 9 | ```shell script 10 | make 11 | ``` 12 | 13 | Then, if you have ``python3`` installed: 14 | ```shell script 15 | ./ci/run.py 16 | ``` 17 | 18 | For more information, please refer to [INSTALL.md](./INSTALL.md). 19 | 20 | ## How To Contribute 21 | 22 | Contributions are what make the open source community such an amazing place. 23 | Any contributions you make are greatly appreciated. 24 | 25 | 1. Fork the Project 26 | 2. Create your Feature Branch (git checkout -b feature/AmazingFeature) 27 | 3. Commit your Changes (git commit -m 'Add some AmazingFeature') 28 | 4. Push to the Branch (git push origin feature/AmazingFeature) 29 | 5. Open a Pull Request 30 | 31 | For more information, please refer to [PARTICIPATE.md](./PARTICIPATE.md). 32 | 33 | ## License 34 | 35 | Distributed under the MIT License. See [LICENSE](./LICENSE) for more information. 36 | 37 | I strongly believe in Open Source for many reasons: 38 | * For software quality purposes because a software with open sources is the best 39 | way to have its bugs identified and fixed as soon as possible. 40 | * For a greater adoption, we chose a deliberately liberal license so that 41 | there cannot be any legal concern. 42 | * Because I do not expect to make any money with this, ever. 43 | 44 | ## Contact 45 | 46 | Follow the development on GitHub with the 47 | [gunkan-io/object-storage](https://github.com/gunkan-io/oject-storage) project. 48 | 49 | ## Acknowledgements 50 | 51 | We welcome any volunteer and we already have a list of 52 | [amazing authors of Gunkan](./AUTHORS.md). 53 | -------------------------------------------------------------------------------- /internal/cmd-blob-client/cmd_put.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package cmd_blob_client 7 | 8 | import ( 9 | "context" 10 | "errors" 11 | "github.com/jfsmig/object-storage/pkg/gunkan" 12 | "github.com/spf13/cobra" 13 | "os" 14 | ) 15 | 16 | func PutCommand() *cobra.Command { 17 | var cfg config 18 | 19 | cmd := &cobra.Command{ 20 | Use: "put", 21 | Aliases: []string{"push", "store", "add"}, 22 | Short: "Put data in a BLOB service", 23 | RunE: func(cmd *cobra.Command, args []string) error { 24 | var err error 25 | var id gunkan.BlobId 26 | 27 | if len(args) < 1 { 28 | return errors.New("Missing Blob ID") 29 | } 30 | if id, err = gunkan.DecodeBlobId(args[0]); err != nil { 31 | return err 32 | } 33 | 34 | client, err := gunkan.DialBlob(cfg.url) 35 | if err != nil { 36 | return err 37 | } 38 | 39 | var realid string 40 | if len(args) == 2 { 41 | path := args[1] 42 | if fin, err := os.Open(path); err == nil { 43 | defer fin.Close() 44 | var finfo os.FileInfo 45 | finfo, err = fin.Stat() 46 | if err == nil { 47 | realid, err = client.PutN(context.Background(), id, fin, finfo.Size()) 48 | } 49 | } 50 | } else { 51 | realid, err = client.Put(context.Background(), id, os.Stdin) 52 | } 53 | 54 | if err != nil { 55 | gunkan.Logger.Info().Str("id", id.Encode()).Str("real", realid).Msg("ok") 56 | } 57 | return err 58 | }, 59 | } 60 | 61 | cmd.Flags().StringVar(&cfg.url, "url", "", "IP:PORT endpoint of the service to contact") 62 | 63 | return cmd 64 | } 65 | -------------------------------------------------------------------------------- /pkg/gunkan/balancer_simple.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package gunkan 7 | 8 | import ( 9 | "errors" 10 | "math/rand" 11 | ) 12 | 13 | type simpleBalancer struct { 14 | catalog Catalog 15 | } 16 | 17 | var ( 18 | errNotAvailableDataGate = errors.New("No data gateway available") 19 | errNotAvailableIndexGate = errors.New("No index gateway available") 20 | errNotAvailableBlobStore = errors.New("No blob store available") 21 | errNotAvailableIndexStore = errors.New("No inndex store available") 22 | ) 23 | 24 | func NewBalancerSimple(catalog Catalog) (Balancer, error) { 25 | return &simpleBalancer{catalog: catalog}, nil 26 | } 27 | 28 | func (self *simpleBalancer) PollIndexGate() (string, error) { 29 | addrv, err := self.catalog.ListIndexGate() 30 | if err != nil { 31 | return "", err 32 | } else if len(addrv) <= 0 { 33 | return "", errNotAvailableIndexGate 34 | } else { 35 | return addrv[rand.Intn(len(addrv))], nil 36 | } 37 | } 38 | 39 | func (self *simpleBalancer) PollDataGate() (string, error) { 40 | addrv, err := self.catalog.ListDataGate() 41 | if err != nil { 42 | return "", err 43 | } else if len(addrv) <= 0 { 44 | return "", errNotAvailableDataGate 45 | } else { 46 | return addrv[rand.Intn(len(addrv))], nil 47 | } 48 | } 49 | 50 | func (self *simpleBalancer) PollBlobStore() (string, error) { 51 | addrv, err := self.catalog.ListBlobStore() 52 | if err != nil { 53 | return "", err 54 | } else if len(addrv) <= 0 { 55 | return "", errNotAvailableBlobStore 56 | } else { 57 | return addrv[rand.Intn(len(addrv))], nil 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /internal/cmd-blob-client/cmd_del.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package cmd_blob_client 7 | 8 | import ( 9 | "context" 10 | "errors" 11 | "github.com/jfsmig/object-storage/pkg/gunkan" 12 | "github.com/spf13/cobra" 13 | ) 14 | 15 | func DelCommand() *cobra.Command { 16 | var cfg config 17 | 18 | client := &cobra.Command{ 19 | Use: "del", 20 | Aliases: []string{"delete", "remove", "rm", "erase"}, 21 | Short: "Delete BLOBs from a service", 22 | RunE: func(cmd *cobra.Command, args []string) error { 23 | if len(args) < 1 { 24 | return errors.New(("Missing Blob ID")) 25 | } 26 | client, err := gunkan.DialBlob(cfg.url) 27 | if err != nil { 28 | return err 29 | } 30 | if len(args) == 1 { 31 | id := args[0] 32 | err = delOne(client, id) 33 | debug(id, err) 34 | return err 35 | } 36 | 37 | strongError := false 38 | for _, id := range args { 39 | err = delOne(client, id) 40 | debug(id, err) 41 | if err != gunkan.ErrNotFound { 42 | strongError = true 43 | } 44 | } 45 | if strongError { 46 | err = errors.New("Unrecoverable error") 47 | } else { 48 | err = nil 49 | } 50 | return err 51 | }, 52 | } 53 | 54 | client.Flags().StringVar(&cfg.url, "url", "", "IP:PORT endpoint of the service to contact") 55 | 56 | return client 57 | } 58 | 59 | func delOne(client gunkan.BlobClient, strid string) error { 60 | var err error 61 | 62 | if _, err = gunkan.DecodeBlobId(strid); err != nil { 63 | return err 64 | } else { 65 | return client.Delete(context.Background(), strid) 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /internal/cmd-data-gate/cmd.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package cmd_data_gate 7 | 8 | import ( 9 | "errors" 10 | "fmt" 11 | ghttp "github.com/jfsmig/object-storage/internal/helpers-http" 12 | "github.com/spf13/cobra" 13 | "net/http" 14 | ) 15 | 16 | func MainCommand() *cobra.Command { 17 | var cfg config 18 | 19 | server := &cobra.Command{ 20 | Use: "proxy", 21 | Aliases: []string{}, 22 | Short: "Start a BLOB proxy", 23 | RunE: func(cmd *cobra.Command, args []string) error { 24 | if len(args) != 1 { 25 | return errors.New("Missing positional args: ADDR") 26 | } else { 27 | cfg.addrBind = args[0] 28 | } 29 | 30 | // FIXME(jfsmig): Fix the sanitizing of the input 31 | if cfg.addrBind == "" { 32 | return errors.New("Missing bind address") 33 | } 34 | if cfg.addrAnnounce == "" { 35 | cfg.addrAnnounce = cfg.addrBind 36 | } 37 | 38 | srv, err := newService(cfg) 39 | if err != nil { 40 | return err 41 | } 42 | httpService := ghttp.NewHttpApi(cfg.addrAnnounce, infoString) 43 | httpService.Route(routeList, ghttp.Get(srv.handleList())) 44 | httpService.Route(prefixData, srv.handlePart()) 45 | err = http.ListenAndServe(cfg.addrBind, httpService.Handler()) 46 | if err != nil { 47 | return errors.New(fmt.Sprintf("HTTP error [%s]", cfg.addrBind, err.Error())) 48 | } 49 | return nil 50 | }, 51 | } 52 | 53 | const ( 54 | publicUsage = "Public address of the service." 55 | tlsUsage = "Path to a directory with the TLS configuration" 56 | ) 57 | server.Flags().StringVar(&cfg.dirConfig, "tls", "", tlsUsage) 58 | server.Flags().StringVar(&cfg.addrAnnounce, "pub", "", publicUsage) 59 | return server 60 | } 61 | -------------------------------------------------------------------------------- /pkg/gunkan/index_client_grpc.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package gunkan 7 | 8 | import ( 9 | "github.com/jfsmig/object-storage/internal/helpers-grpc" 10 | kv "github.com/jfsmig/object-storage/pkg/gunkan-index-proto" 11 | "google.golang.org/grpc" 12 | 13 | "context" 14 | ) 15 | 16 | func DialIndexGrpc(url, dirConfig string) (IndexClient, error) { 17 | cnx, err := helpers_grpc.DialTLSInsecure(url) 18 | if err != nil { 19 | return nil, err 20 | } 21 | return &IndexGrpcClient{cnx: cnx}, err 22 | } 23 | 24 | type IndexGrpcClient struct { 25 | cnx *grpc.ClientConn 26 | } 27 | 28 | func (self *IndexGrpcClient) Get(ctx context.Context, key BaseKey) (string, error) { 29 | client := kv.NewIndexClient(self.cnx) 30 | req := kv.GetRequest{Base: key.Base, Key: key.Key} 31 | rep, err := client.Get(ctx, &req) 32 | if err != nil { 33 | return "", err 34 | } 35 | 36 | return rep.Value, nil 37 | } 38 | 39 | func (self *IndexGrpcClient) List(ctx context.Context, key BaseKey, max uint32) ([]string, error) { 40 | client := kv.NewIndexClient(self.cnx) 41 | req := kv.ListRequest{Base: key.Base, Marker: key.Key, Max: max} 42 | rep, err := client.List(ctx, &req) 43 | if err != nil { 44 | return []string{}, err 45 | } 46 | 47 | return rep.Items, err 48 | } 49 | 50 | func (self *IndexGrpcClient) Put(ctx context.Context, key BaseKey, value string) error { 51 | client := kv.NewIndexClient(self.cnx) 52 | req := kv.PutRequest{Base: key.Base, Key: key.Key, Value: value} 53 | _, err := client.Put(ctx, &req) 54 | return err 55 | } 56 | 57 | func (self *IndexGrpcClient) Delete(ctx context.Context, key BaseKey) error { 58 | client := kv.NewIndexClient(self.cnx) 59 | req := kv.DeleteRequest{Base: key.Base, Key: key.Key} 60 | _, err := client.Delete(ctx, &req) 61 | return err 62 | } 63 | -------------------------------------------------------------------------------- /pkg/gunkan/http_client_direct.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package gunkan 7 | 8 | import ( 9 | "context" 10 | "io" 11 | "io/ioutil" 12 | "net/http" 13 | "strings" 14 | ) 15 | 16 | func (self *HttpSimpleClient) Init(url string) error { 17 | // FIXME(jfsmig): Sanitizes the URL 18 | self.Endpoint = url 19 | self.Http = http.Client{} 20 | return nil 21 | } 22 | 23 | func (self *HttpSimpleClient) BuildUrl(path string) string { 24 | b := strings.Builder{} 25 | b.WriteString("http://") 26 | b.WriteString(self.Endpoint) 27 | b.WriteString(path) 28 | return b.String() 29 | } 30 | 31 | func (self *HttpSimpleClient) srvGet(ctx context.Context, tag string) ([]byte, error) { 32 | req, err := self.makeRequest(ctx, "GET", self.BuildUrl(tag), nil) 33 | if err != nil { 34 | return []byte{}, err 35 | } 36 | 37 | rep, err := self.Http.Do(req) 38 | if err != nil { 39 | return []byte{}, err 40 | } 41 | 42 | defer rep.Body.Close() 43 | return ioutil.ReadAll(rep.Body) 44 | } 45 | 46 | func (self *HttpSimpleClient) Info(ctx context.Context) ([]byte, error) { 47 | return self.srvGet(ctx, RouteInfo) 48 | } 49 | 50 | func (self *HttpSimpleClient) Health(ctx context.Context) ([]byte, error) { 51 | return self.srvGet(ctx, RouteHealth) 52 | } 53 | 54 | func (self *HttpSimpleClient) Metrics(ctx context.Context) ([]byte, error) { 55 | return self.srvGet(ctx, RouteMetrics) 56 | } 57 | 58 | func (self *HttpSimpleClient) makeRequest(ctx context.Context, method string, path string, body io.Reader) (*http.Request, error) { 59 | req, err := http.NewRequestWithContext(ctx, method, path, body) 60 | if err == nil { 61 | req.Close = true 62 | if self.UserAgent != "" { 63 | req.Header.Set("User-Agent", self.UserAgent) 64 | } else { 65 | req.Header.Set("User-Agent", "gunkan-http-go-api/1") 66 | } 67 | req.Header.Del("Accept-Encoding") 68 | } 69 | return req, err 70 | } 71 | -------------------------------------------------------------------------------- /internal/cmd-blob-store-fs/cmd.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package cmd_blob_store_fs 7 | 8 | import ( 9 | "errors" 10 | "fmt" 11 | ghttp "github.com/jfsmig/object-storage/internal/helpers-http" 12 | "github.com/spf13/cobra" 13 | "net/http" 14 | ) 15 | 16 | func MainCommand() *cobra.Command { 17 | var cfg config 18 | 19 | server := &cobra.Command{ 20 | Use: "srv", 21 | Aliases: []string{"server", "service", "worker", "agent"}, 22 | Short: "Start a BLOB server", 23 | RunE: func(cmd *cobra.Command, args []string) error { 24 | if len(args) != 2 { 25 | return errors.New("Missing positional args: ADDR DIRECTORY") 26 | } else { 27 | cfg.addrBind = args[0] 28 | cfg.dirBase = args[1] 29 | } 30 | 31 | // FIXME(jfsmig): Fix the sanitizing of the input 32 | if cfg.addrBind == "" { 33 | return errors.New("Missing bind address") 34 | } 35 | if cfg.dirBase == "" { 36 | return errors.New("Missing base directory") 37 | } 38 | if cfg.addrAnnounce == "" { 39 | cfg.addrAnnounce = cfg.addrBind 40 | } 41 | 42 | srv, err := newService(cfg) 43 | if err != nil { 44 | return errors.New(fmt.Sprintf("Repository error [%s]", cfg.dirBase, err.Error())) 45 | } 46 | 47 | api := ghttp.NewHttpApi(cfg.addrAnnounce, infoString) 48 | api.Route(routeList, ghttp.Get(srv.handleList())) 49 | api.Route(prefixData, srv.handleBlob()) 50 | err = http.ListenAndServe(cfg.addrBind, api.Handler()) 51 | if err != nil { 52 | return errors.New(fmt.Sprintf("HTTP error [%s]", cfg.addrBind, err.Error())) 53 | } 54 | return nil 55 | }, 56 | } 57 | 58 | const ( 59 | publicUsage = "Public address of the service" 60 | tlsUsage = "Path to a directory with the TLS configuration" 61 | smrUsage = "Use a SMR ready naming policy of objects" 62 | ) 63 | server.Flags().StringVar(&cfg.dirConfig, "tls", "", tlsUsage) 64 | server.Flags().StringVar(&cfg.addrAnnounce, "pub", "", publicUsage) 65 | return server 66 | } 67 | -------------------------------------------------------------------------------- /internal/cmd-data-gate/service.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package cmd_data_gate 7 | 8 | import ( 9 | "github.com/jfsmig/object-storage/pkg/gunkan" 10 | "github.com/prometheus/client_golang/prometheus" 11 | "github.com/prometheus/client_golang/prometheus/promauto" 12 | "math" 13 | "time" 14 | ) 15 | 16 | type config struct { 17 | uuid string 18 | addrBind string 19 | addrAnnounce string 20 | dirConfig string 21 | } 22 | 23 | type service struct { 24 | config config 25 | 26 | lb gunkan.Balancer 27 | 28 | timePut prometheus.Histogram 29 | timeGet prometheus.Histogram 30 | timeDel prometheus.Histogram 31 | timeList prometheus.Histogram 32 | } 33 | 34 | func newService(cfg config) (*service, error) { 35 | var err error 36 | srv := service{config: cfg} 37 | srv.lb, err = gunkan.NewBalancerDefault() 38 | 39 | buckets := []float64{0.01, 0.02, 0.03, 0.04, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 1, 2, 3, 4, 5, math.Inf(1)} 40 | 41 | srv.timeList = promauto.NewHistogram(prometheus.HistogramOpts{ 42 | Name: "gunkan_part_list_ttlb", 43 | Help: "Repartition of the request times of List requests", 44 | Buckets: buckets, 45 | }) 46 | 47 | srv.timePut = promauto.NewHistogram(prometheus.HistogramOpts{ 48 | Name: "gunkan_part_put_ttlb", 49 | Help: "Repartition of the request times of put requests", 50 | Buckets: buckets, 51 | }) 52 | 53 | srv.timeGet = promauto.NewHistogram(prometheus.HistogramOpts{ 54 | Name: "gunkan_part_get_ttlb", 55 | Help: "Repartition of the request times of get requests", 56 | Buckets: buckets, 57 | }) 58 | 59 | srv.timeDel = promauto.NewHistogram(prometheus.HistogramOpts{ 60 | Name: "gunkan_part_del_ttlb", 61 | Help: "Repartition of the request times of del requests", 62 | Buckets: buckets, 63 | }) 64 | 65 | if err != nil { 66 | return nil, err 67 | } else { 68 | return &srv, nil 69 | } 70 | } 71 | 72 | func (srv *service) isOverloaded(now time.Time) bool { 73 | return false 74 | } 75 | -------------------------------------------------------------------------------- /pkg/gunkan/part.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package gunkan 7 | 8 | import ( 9 | "errors" 10 | "strings" 11 | ) 12 | 13 | const ( 14 | stepPartParsingContent = 0 15 | stepPartParsingPart = 1 16 | stepPartParsingBucket = 2 17 | ) 18 | 19 | type PartId struct { 20 | Bucket string 21 | Content string 22 | PartId string 23 | } 24 | 25 | func (self *PartId) Encode() string { 26 | var b strings.Builder 27 | self.EncodeIn(&b) 28 | return b.String() 29 | } 30 | 31 | func (self *PartId) EncodeIn(b *strings.Builder) { 32 | b.Grow(256) 33 | b.WriteString(self.Content) 34 | b.WriteRune(',') 35 | b.WriteString(self.PartId) 36 | b.WriteRune(',') 37 | b.WriteString(self.Bucket) 38 | } 39 | 40 | func (self *PartId) EncodeMarker() string { 41 | var b strings.Builder 42 | self.EncodeMarkerIn(&b) 43 | return b.String() 44 | } 45 | 46 | func (self *PartId) EncodeMarkerIn(b *strings.Builder) { 47 | b.Grow(256) 48 | b.WriteString(self.Bucket) 49 | if len(self.Content) > 0 { 50 | b.WriteRune(',') 51 | b.WriteString(self.Content) 52 | if len(self.PartId) > 0 { 53 | b.WriteRune(',') 54 | b.WriteString(self.PartId) 55 | } 56 | } 57 | } 58 | 59 | func (self *PartId) Decode(packed string) error { 60 | b := strings.Builder{} 61 | step := stepPartParsingBucket 62 | for _, c := range packed { 63 | switch step { 64 | case stepPartParsingBucket: 65 | if c == ',' { 66 | step = stepPartParsingContent 67 | self.Content = b.String() 68 | b.Reset() 69 | } else { 70 | b.WriteRune(c) 71 | } 72 | case stepPartParsingContent: 73 | if c == ',' { 74 | step = stepPartParsingPart 75 | self.Content = b.String() 76 | b.Reset() 77 | } else { 78 | b.WriteRune(c) 79 | } 80 | case stepPartParsingPart: 81 | if c == ',' { 82 | return errors.New("Invalid PART id") 83 | } else { 84 | b.WriteRune(c) 85 | } 86 | default: 87 | panic("Invalid State") 88 | } 89 | } 90 | self.PartId = b.String() 91 | return nil 92 | } 93 | -------------------------------------------------------------------------------- /BLOB.md: -------------------------------------------------------------------------------- 1 | # Gunkan - Object Storage - Blob service 2 | 3 | **gunkan-blob** serves BLOB's hosted on a local filesystem. 4 | 5 | The protocol is a subset of HTTP/1.1 over TCP/IP connections: 6 | * `HTTP/1.1` is the only protocol version accepted 7 | * `Connection: keepalive` is ignored, the connections are always closed and `Connection: close` is invariably returned 8 | * `Expect: 100-continue` is honored 9 | * `Transfer-Encoding: chunked` and `Transfer-Encoding: inline` are honored. `inline` is implied when nothing is mentioned. 10 | * Redirections are never emitted. 11 | 12 | 13 | ## Build, Install, Run 14 | 15 | ``` 16 | cmake 17 | make 18 | make install 19 | ``` 20 | 21 | 22 | ## API 23 | 24 | Special request fields: 25 | * ``BLOB-ID`` is a unique identifier for a BLOB. 26 | The format is a hexadecimal string of 64 characters. 27 | 28 | Common Request headers: 29 | * `Host` must be present and valued to the identifier of the service as known 30 | in the Catalog. 31 | * `X-gunkan-token` must be present and valued as an authentication / authorization 32 | token as issued by the Access Manager 33 | 34 | ### GET /info 35 | 36 | Returns a description of the service. 37 | 38 | ### GET /v1/status 39 | 40 | Returns usage statistics about the current service. 41 | The body contains JSON 42 | 43 | ### GET /v1/list 44 | 45 | Returns a list of ``{BLOB-ID}``, one per line, with en `CRLF` as a line separator. 46 | 47 | Optional query string arguments are honored: 48 | * ``marker`` a prefix of a ``{BLOB-ID}`` that must be past by the iterator. 49 | * ``max`` the maximum number of items in the answer 50 | 51 | ### PUT /v1/blob/{BLOB-ID} 52 | 53 | Add a BLOB on the storage of the service. 54 | 55 | ### GET /v1/blob/{BLOB-ID} 56 | 57 | Fetch a BLOB. The data will be served as the body and the metadata will be 58 | present in the header fields of the reply. 59 | 60 | ### HEAD /v1/blob/{BLOB-ID} 61 | 62 | Fetch metadata information about a BLOB. The metadata will be present as fields 63 | in the header of the reply. 64 | 65 | That route respects the semantics of a HEAD HTTP request: e.g. the `content-length` 66 | field is present but no body is expected. 67 | 68 | ### DELETE /v1/blob/{BLOB-ID} 69 | 70 | Remove a BLOB from the storage of the service. 71 | -------------------------------------------------------------------------------- /internal/cmd-index-gate/cmd.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package cmd_index_gate 7 | 8 | import ( 9 | "errors" 10 | grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" 11 | "github.com/jfsmig/object-storage/internal/helpers-grpc" 12 | "github.com/jfsmig/object-storage/pkg/gunkan-index-proto" 13 | "github.com/prometheus/client_golang/prometheus/promhttp" 14 | "github.com/spf13/cobra" 15 | "net" 16 | "net/http" 17 | ) 18 | 19 | func MainCommand() *cobra.Command { 20 | var cfg serviceConfig 21 | 22 | server := &cobra.Command{ 23 | Use: "gate", 24 | Aliases: []string{"proxy", "gateway"}, 25 | Short: "Start a stateless index gateway", 26 | RunE: func(cmd *cobra.Command, args []string) error { 27 | if len(args) != 1 { 28 | return errors.New("Missing positional args: ADDR") 29 | } else { 30 | cfg.addrBind = args[0] 31 | } 32 | 33 | // FIXME(jfsmig): Fix the sanitizing of the input 34 | if cfg.addrBind == "" { 35 | return errors.New("Missing bind address") 36 | } 37 | if cfg.addrAnnounce == "" { 38 | cfg.addrAnnounce = cfg.addrBind 39 | } 40 | 41 | lis, err := net.Listen("tcp", cfg.addrBind) 42 | if err != nil { 43 | return err 44 | } 45 | service, err := NewService(cfg) 46 | if err != nil { 47 | return err 48 | } 49 | httpServer, err := helpers_grpc.ServerTLS(cfg.dirConfig) 50 | if err != nil { 51 | return err 52 | } 53 | 54 | gunkan_index_proto.RegisterIndexServer(httpServer, service) 55 | grpc_prometheus.Register(httpServer) 56 | http.Handle("/metrics", promhttp.Handler()) 57 | http.HandleFunc("/info", func(rep http.ResponseWriter, req *http.Request) { 58 | rep.Write([]byte("Yallah!")) 59 | }) 60 | return httpServer.Serve(lis) 61 | }, 62 | } 63 | 64 | const ( 65 | publicUsage = "Public address of the service." 66 | tlsUsage = "Path to a directory with the TLS configuration" 67 | ) 68 | server.Flags().StringVar(&cfg.dirConfig, "tls", "", tlsUsage) 69 | server.Flags().StringVar(&cfg.addrAnnounce, "pub", "", publicUsage) 70 | return server 71 | } 72 | -------------------------------------------------------------------------------- /internal/cmd-index-store-rocksdb/cmd.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package cmd_index_store_rocksdb 7 | 8 | import ( 9 | "errors" 10 | grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" 11 | "github.com/jfsmig/object-storage/internal/helpers-grpc" 12 | "github.com/jfsmig/object-storage/pkg/gunkan-index-proto" 13 | "github.com/prometheus/client_golang/prometheus/promhttp" 14 | "github.com/spf13/cobra" 15 | "net" 16 | "net/http" 17 | ) 18 | 19 | func MainCommand() *cobra.Command { 20 | var cfg serviceConfig 21 | 22 | cmd := &cobra.Command{ 23 | Use: "srv", 24 | Aliases: []string{"server"}, 25 | Short: "Start an index server", 26 | RunE: func(cmd *cobra.Command, args []string) error { 27 | if len(args) != 2 { 28 | return errors.New("Missing positional args: ADDR DIRECTORY") 29 | } else { 30 | cfg.addrBind = args[0] 31 | cfg.dirBase = args[1] 32 | } 33 | 34 | // FIXME(jfsmig): Fix the sanitizing of the input 35 | if cfg.addrBind == "" { 36 | return errors.New("Missing bind address") 37 | } 38 | if cfg.dirBase == "" { 39 | return errors.New("Missing base directory") 40 | } 41 | if cfg.addrAnnounce == "" { 42 | cfg.addrAnnounce = cfg.addrBind 43 | } 44 | 45 | lis, err := net.Listen("tcp", cfg.addrBind) 46 | if err != nil { 47 | return err 48 | } 49 | service, err := NewService(cfg) 50 | if err != nil { 51 | return err 52 | } 53 | httpServer, err := helpers_grpc.ServerTLS(cfg.dirConfig) 54 | if err != nil { 55 | return err 56 | } 57 | gunkan_index_proto.RegisterIndexServer(httpServer, service) 58 | grpc_prometheus.Register(httpServer) 59 | http.Handle("/metrics", promhttp.Handler()) 60 | http.HandleFunc("/info", func(rep http.ResponseWriter, req *http.Request) { 61 | rep.Write([]byte("Yallah!")) 62 | }) 63 | return httpServer.Serve(lis) 64 | }, 65 | } 66 | 67 | const ( 68 | publicUsage = "Public address of the service." 69 | tlsUsage = "Path to a directory with the TLS configuration" 70 | ) 71 | cmd.Flags().StringVar(&cfg.dirConfig, "tls", "", tlsUsage) 72 | cmd.Flags().StringVar(&cfg.addrAnnounce, "pub", "", publicUsage) 73 | return cmd 74 | } 75 | -------------------------------------------------------------------------------- /internal/cmd-blob-store-fs/service.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package cmd_blob_store_fs 7 | 8 | import ( 9 | "github.com/prometheus/client_golang/prometheus" 10 | "github.com/prometheus/client_golang/prometheus/promauto" 11 | "math" 12 | "time" 13 | ) 14 | 15 | type config struct { 16 | uuid string 17 | addrBind string 18 | addrAnnounce string 19 | dirConfig string 20 | dirBase string 21 | 22 | delayIoError time.Duration 23 | delayFullError time.Duration 24 | } 25 | 26 | type service struct { 27 | config config 28 | 29 | repo Repo 30 | 31 | lastIoError time.Time 32 | lastFullError time.Time 33 | 34 | timePut prometheus.Histogram 35 | timeGet prometheus.Histogram 36 | timeDel prometheus.Histogram 37 | timeList prometheus.Histogram 38 | } 39 | 40 | func newService(cfg config) (*service, error) { 41 | var err error 42 | srv := service{config: cfg} 43 | 44 | srv.repo, err = MakePostNamed(cfg.dirBase) 45 | if err != nil { 46 | return nil, err 47 | } 48 | 49 | buckets := []float64{0.01, 0.02, 0.03, 0.04, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 1, 2, 3, 4, 5, math.Inf(1)} 50 | 51 | srv.timeList = promauto.NewHistogram(prometheus.HistogramOpts{ 52 | Name: "gunkan_blob_list_ttlb", 53 | Help: "Repartition of the request times of List requests", 54 | Buckets: buckets, 55 | }) 56 | 57 | srv.timePut = promauto.NewHistogram(prometheus.HistogramOpts{ 58 | Name: "gunkan_blob_put_ttlb", 59 | Help: "Repartition of the request times of put requests", 60 | Buckets: buckets, 61 | }) 62 | 63 | srv.timeGet = promauto.NewHistogram(prometheus.HistogramOpts{ 64 | Name: "gunkan_blob_get_ttlb", 65 | Help: "Repartition of the request times of get requests", 66 | Buckets: buckets, 67 | }) 68 | 69 | srv.timeDel = promauto.NewHistogram(prometheus.HistogramOpts{ 70 | Name: "gunkan_blob_del_ttlb", 71 | Help: "Repartition of the request times of del requests", 72 | Buckets: buckets, 73 | }) 74 | 75 | if err != nil { 76 | return nil, err 77 | } else { 78 | return &srv, nil 79 | } 80 | } 81 | 82 | func (srv *service) isFull(now time.Time) bool { 83 | return !srv.lastFullError.IsZero() && now.Sub(srv.lastFullError) > srv.config.delayFullError 84 | } 85 | 86 | func (srv *service) isError(now time.Time) bool { 87 | return !srv.lastIoError.IsZero() && now.Sub(srv.lastIoError) > srv.config.delayIoError 88 | } 89 | 90 | func (srv *service) isOverloaded(now time.Time) bool { 91 | return false 92 | } 93 | -------------------------------------------------------------------------------- /pkg/gunkan/blob.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package gunkan 7 | 8 | import ( 9 | "errors" 10 | "strconv" 11 | "strings" 12 | ) 13 | 14 | const ( 15 | stepBlobParsingContent = 0 16 | stepBlobParsingPart = 1 17 | stepBlobParsingPosition = 2 18 | stepBlobParsingBucket = 3 19 | ) 20 | 21 | type BlobId struct { 22 | Bucket string 23 | Content string 24 | PartId string 25 | Position uint 26 | } 27 | 28 | func (self BlobId) EncodeMarker() string { 29 | var b strings.Builder 30 | self.EncodeMarkerIn(&b) 31 | return b.String() 32 | } 33 | 34 | func (self BlobId) EncodeMarkerIn(b *strings.Builder) { 35 | b.Grow(256) 36 | b.WriteString(self.Bucket) 37 | if len(self.Content) > 0 { 38 | b.WriteRune(',') 39 | b.WriteString(self.Content) 40 | if len(self.PartId) > 0 { 41 | b.WriteRune(',') 42 | b.WriteString(self.PartId) 43 | } 44 | } 45 | } 46 | 47 | func (self BlobId) Encode() string { 48 | var b strings.Builder 49 | self.EncodeIn(&b) 50 | return b.String() 51 | } 52 | 53 | func (self BlobId) EncodeIn(b *strings.Builder) { 54 | b.Grow(256) 55 | b.WriteString(self.Bucket) 56 | b.WriteRune(',') 57 | b.WriteString(self.Content) 58 | b.WriteRune(',') 59 | b.WriteString(self.PartId) 60 | b.WriteRune(',') 61 | b.WriteString(strconv.FormatUint(uint64(self.Position), 10)) 62 | } 63 | 64 | func DecodeBlobId(packed string) (BlobId, error) { 65 | var id BlobId 66 | b := strings.Builder{} 67 | step := stepBlobParsingBucket 68 | for _, c := range packed { 69 | switch step { 70 | case stepBlobParsingBucket: 71 | if c == ',' { 72 | step = stepBlobParsingContent 73 | id.Bucket = b.String() 74 | b.Reset() 75 | } else { 76 | b.WriteRune(c) 77 | } 78 | case stepBlobParsingContent: 79 | if c == ',' { 80 | step = stepBlobParsingPart 81 | id.Content = b.String() 82 | b.Reset() 83 | } else { 84 | b.WriteRune(c) 85 | } 86 | case stepBlobParsingPart: 87 | if c == ',' { 88 | step = stepBlobParsingPosition 89 | id.PartId = b.String() 90 | b.Reset() 91 | } else { 92 | b.WriteRune(c) 93 | } 94 | case stepBlobParsingPosition: 95 | if c == ',' { 96 | return id, errors.New("Invalid BLOB id") 97 | } else { 98 | b.WriteRune(c) 99 | } 100 | default: 101 | panic("Invalid State") 102 | } 103 | } 104 | 105 | u64, err := strconv.ParseUint(b.String(), 10, 31) 106 | id.Position = uint(u64) 107 | return id, err 108 | } 109 | -------------------------------------------------------------------------------- /internal/helpers-grpc/cnx.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package helpers_grpc 7 | 8 | import ( 9 | "crypto/tls" 10 | "crypto/x509" 11 | "errors" 12 | grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" 13 | "google.golang.org/grpc" 14 | "google.golang.org/grpc/credentials" 15 | "io/ioutil" 16 | ) 17 | 18 | func DialTLS(addrConnect, dirConfig string) (*grpc.ClientConn, error) { 19 | var caBytes, certBytes []byte 20 | var err error 21 | 22 | if caBytes, err = ioutil.ReadFile(dirConfig + "/ca.cert"); err != nil { 23 | return nil, err 24 | } 25 | if certBytes, err = ioutil.ReadFile(dirConfig + "/service.pem"); err != nil { 26 | return nil, err 27 | } 28 | //if keyBytes, err = ioutil.ReadFile(dirConfig + "/service.key"); err != nil { 29 | // return nil, err 30 | //} 31 | 32 | certPool := x509.NewCertPool() 33 | if !certPool.AppendCertsFromPEM(certBytes) { 34 | return nil, errors.New("Invalid certificate (service)") 35 | } 36 | if !certPool.AppendCertsFromPEM(caBytes) { 37 | return nil, errors.New("Invalid certificate (authority)") 38 | } 39 | 40 | creds := credentials.NewClientTLSFromCert(certPool, "") 41 | 42 | return grpc.Dial(addrConnect, 43 | grpc.WithTransportCredentials(creds), 44 | grpc.WithUnaryInterceptor(grpc_prometheus.UnaryClientInterceptor), 45 | grpc.WithStreamInterceptor(grpc_prometheus.StreamClientInterceptor)) 46 | } 47 | 48 | func DialTLSInsecure(addrConnect string) (*grpc.ClientConn, error) { 49 | config := &tls.Config{ 50 | InsecureSkipVerify: true, 51 | } 52 | creds := credentials.NewTLS(config) 53 | return grpc.Dial(addrConnect, 54 | grpc.WithTransportCredentials(creds), 55 | grpc.WithUnaryInterceptor(grpc_prometheus.UnaryClientInterceptor), 56 | grpc.WithStreamInterceptor(grpc_prometheus.StreamClientInterceptor)) 57 | } 58 | 59 | func ServerTLS(dirConfig string) (*grpc.Server, error) { 60 | var certBytes, keyBytes []byte 61 | var err error 62 | 63 | if certBytes, err = ioutil.ReadFile(dirConfig + "/service.pem"); err != nil { 64 | return nil, err 65 | } 66 | if keyBytes, err = ioutil.ReadFile(dirConfig + "/service.key"); err != nil { 67 | return nil, err 68 | } 69 | 70 | certPool := x509.NewCertPool() 71 | ok := certPool.AppendCertsFromPEM(certBytes) 72 | if !ok { 73 | return nil, errors.New("Invalid certificates") 74 | } 75 | 76 | cert, err := tls.X509KeyPair(certBytes, keyBytes) 77 | if err != nil { 78 | return nil, err 79 | } 80 | 81 | creds := credentials.NewServerTLSFromCert(&cert) 82 | srv := grpc.NewServer( 83 | grpc.Creds(creds), 84 | grpc.StreamInterceptor(grpc_prometheus.StreamServerInterceptor), 85 | grpc.UnaryInterceptor(grpc_prometheus.UnaryServerInterceptor)) 86 | return srv, nil 87 | } 88 | -------------------------------------------------------------------------------- /pkg/gunkan/catalog_consul.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package gunkan 7 | 8 | import ( 9 | "context" 10 | "errors" 11 | consulapi "github.com/armon/consul-api" 12 | "net" 13 | "strconv" 14 | ) 15 | 16 | func GetConsulEndpoint() (string, error) { 17 | return "127.0.0.1", nil 18 | } 19 | 20 | func NewCatalogConsul(ip string) (Catalog, error) { 21 | d := consulDiscovery{} 22 | if err := d.init(ip); err != nil { 23 | return nil, err 24 | } else { 25 | return &d, nil 26 | } 27 | } 28 | 29 | type consulDiscovery struct { 30 | resolver net.Resolver 31 | consul *consulapi.Client 32 | } 33 | 34 | func (self *consulDiscovery) init(ip string) error { 35 | self.resolver.PreferGo = false 36 | self.resolver.Dial = func(ctx context.Context, network, address string) (net.Conn, error) { 37 | local, e0 := net.ResolveUDPAddr("udp", ip+":0") 38 | remote, e1 := net.ResolveUDPAddr("udp", ip+":8600") 39 | if e0 != nil || e1 != nil { 40 | return nil, errors.New("Resolution error") 41 | } 42 | return net.DialUDP("udp", local, remote) 43 | } 44 | 45 | cfg := consulapi.Config{} 46 | if endpoint, err := GetConsulEndpoint(); err != nil { 47 | return err 48 | } else { 49 | cfg.Address = endpoint + ":8500" 50 | } 51 | var err error 52 | self.consul, err = consulapi.NewClient(&cfg) 53 | return err 54 | } 55 | 56 | func (self *consulDiscovery) ListIndexGate() ([]string, error) { 57 | return self.listServices(ConsulSrvIndexGate) 58 | } 59 | 60 | func (self *consulDiscovery) ListDataGate() ([]string, error) { 61 | return self.listServices(ConsulSrvDataGate) 62 | } 63 | 64 | func (self *consulDiscovery) ListIndexStore() ([]string, error) { 65 | return self.listServices(ConsulSrvIndexStore) 66 | } 67 | 68 | func (self *consulDiscovery) ListBlobStore() ([]string, error) { 69 | return self.listServices(ConsulSrvBlobStore) 70 | } 71 | 72 | func (self *consulDiscovery) listServices(srvtype string) ([]string, error) { 73 | var result []string 74 | args := consulapi.QueryOptions{} 75 | args.Datacenter = "" 76 | args.AllowStale = true 77 | args.RequireConsistent = false 78 | catalog := self.consul.Catalog() 79 | if srvtab, _, err := catalog.Services(&args); err != nil { 80 | return result, err 81 | } else { 82 | for srvid, tags := range srvtab { 83 | if !arrayHas(srvtype, tags[:]) { 84 | continue 85 | } 86 | allsrv, _, err := catalog.Service(srvid, srvtype, &args) 87 | if err != nil { 88 | Logger.Info().Str("id", srvid).Str("type", srvtype).Err(err).Msg("Service resolution error") 89 | } else { 90 | for _, srv := range allsrv { 91 | result = append(result, srv.Address+":"+strconv.Itoa(srv.ServicePort)) 92 | } 93 | } 94 | } 95 | Logger.Debug().Str("type", srvtype).Int("nb", len(result)).Msg("Loaded") 96 | return result, nil 97 | } 98 | } 99 | 100 | func arrayHas(needle string, haystack []string) bool { 101 | for _, hay := range haystack { 102 | if hay == needle { 103 | return true 104 | } 105 | } 106 | return false 107 | } 108 | -------------------------------------------------------------------------------- /internal/cmd-blob-store-fs/handlers.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package cmd_blob_store_fs 7 | 8 | import ( 9 | "fmt" 10 | ghttp "github.com/jfsmig/object-storage/internal/helpers-http" 11 | "github.com/jfsmig/object-storage/pkg/gunkan" 12 | "golang.org/x/sys/unix" 13 | "io" 14 | "net/http" 15 | "time" 16 | ) 17 | 18 | func (srv *service) handleBlob() ghttp.RequestHandler { 19 | return func(ctx *ghttp.RequestContext) { 20 | pre := time.Now() 21 | id := ctx.Req.URL.Path[len(prefixData):] 22 | switch ctx.Method() { 23 | case "GET", "HEAD": 24 | srv.handleBlobGet(ctx, id) 25 | srv.timeGet.Observe(time.Since(pre).Seconds()) 26 | case "PUT": 27 | srv.handleBlobPut(ctx, id) 28 | srv.timePut.Observe(time.Since(pre).Seconds()) 29 | case "DELETE": 30 | srv.handleBlobDel(ctx, id) 31 | srv.timeDel.Observe(time.Since(pre).Seconds()) 32 | default: 33 | ctx.WriteHeader(http.StatusMethodNotAllowed) 34 | } 35 | } 36 | } 37 | 38 | func (srv *service) handleList() ghttp.RequestHandler { 39 | h := func(ctx *ghttp.RequestContext) { 40 | ctx.WriteHeader(http.StatusNotImplemented) 41 | } 42 | return func(ctx *ghttp.RequestContext) { 43 | pre := time.Now() 44 | h(ctx) 45 | srv.timeList.Observe(time.Since(pre).Seconds()) 46 | } 47 | } 48 | 49 | func (srv *service) handleBlobDel(ctx *ghttp.RequestContext, blobid string) { 50 | err := srv.repo.Delete(blobid) 51 | if err != nil { 52 | ctx.ReplyError(err) 53 | } else { 54 | ctx.ReplySuccess() 55 | } 56 | } 57 | 58 | func (srv *service) handleBlobGet(ctx *ghttp.RequestContext, blobid string) { 59 | var st unix.Stat_t 60 | var f BlobReader 61 | var err error 62 | 63 | f, err = srv.repo.Open(blobid) 64 | if err != nil { 65 | ctx.ReplyError(err) 66 | return 67 | } else { 68 | defer f.Close() 69 | } 70 | 71 | err = unix.Fstat(int(f.Stream().Fd()), &st) 72 | if err == nil { 73 | if st.Size == 0 { 74 | ctx.SetHeader("Content-Length", "0") 75 | ctx.WriteHeader(http.StatusNoContent) 76 | } else { 77 | ctx.SetHeader("Content-Length", fmt.Sprintf("%d", st.Size)) 78 | ctx.WriteHeader(http.StatusOK) 79 | } 80 | ctx.SetHeader("Content-Type", "octet/stream") 81 | _, err = io.Copy(ctx.Output(), &io.LimitedReader{R: f.Stream(), N: st.Size}) 82 | } 83 | if err != nil { 84 | ctx.ReplyError(err) 85 | } 86 | } 87 | 88 | func (srv *service) handleBlobPut(ctx *ghttp.RequestContext, encoded string) { 89 | var err error 90 | var id gunkan.BlobId 91 | 92 | if id, err = gunkan.DecodeBlobId(string(encoded)); err != nil { 93 | ctx.ReplyCodeError(http.StatusBadRequest, err) 94 | return 95 | } 96 | 97 | f, err := srv.repo.Create(id) 98 | if err != nil { 99 | ctx.ReplyError(err) 100 | return 101 | } 102 | 103 | var final string 104 | _, err = io.Copy(f.Stream(), ctx.Input()) 105 | if err != nil { 106 | f.Abort() 107 | ctx.ReplyError(err) 108 | } else if final, err = f.Commit(); err != nil { 109 | ctx.ReplyError(err) 110 | } else { 111 | ctx.SetHeader("Location", final) 112 | ctx.WriteHeader(http.StatusCreated) 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /pkg/gunkan/index_client_pooled.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package gunkan 7 | 8 | import ( 9 | "context" 10 | "errors" 11 | ) 12 | 13 | const ( 14 | IndexClientPoolSize = 2 15 | ) 16 | 17 | type IndexPooledClient struct { 18 | dirConfig string 19 | lb Balancer 20 | pool chan IndexClient 21 | remaining chan bool 22 | } 23 | 24 | func DialIndexPooled(dirConfig string) (IndexClient, error) { 25 | var err error 26 | var client IndexPooledClient 27 | 28 | client.lb, err = NewBalancerDefault() 29 | if err != nil { 30 | return nil, err 31 | } 32 | 33 | client.dirConfig = dirConfig 34 | client.pool = make(chan IndexClient, IndexClientPoolSize) 35 | client.remaining = make(chan bool, IndexClientPoolSize) 36 | for i := 0; i < IndexClientPoolSize; i++ { 37 | client.remaining <- true 38 | } 39 | close(client.remaining) 40 | return &client, nil 41 | } 42 | 43 | func (self *IndexPooledClient) Get(ctx context.Context, key BaseKey) (string, error) { 44 | client, err := self.acquire(ctx) 45 | defer self.release(client) 46 | if err != nil { 47 | return "", err 48 | } else { 49 | return client.Get(ctx, key) 50 | } 51 | } 52 | 53 | func (self *IndexPooledClient) List(ctx context.Context, key BaseKey, max uint32) ([]string, error) { 54 | client, err := self.acquire(ctx) 55 | defer self.release(client) 56 | if err != nil { 57 | return nil, err 58 | } else { 59 | return client.List(ctx, key, max) 60 | } 61 | } 62 | 63 | func (self *IndexPooledClient) Put(ctx context.Context, key BaseKey, value string) error { 64 | client, err := self.acquire(ctx) 65 | defer self.release(client) 66 | if err != nil { 67 | return err 68 | } else { 69 | return client.Put(ctx, key, value) 70 | } 71 | } 72 | 73 | func (self *IndexPooledClient) Delete(ctx context.Context, key BaseKey) error { 74 | client, err := self.acquire(ctx) 75 | defer self.release(client) 76 | if err != nil { 77 | return err 78 | } else { 79 | return client.Delete(ctx, key) 80 | } 81 | } 82 | 83 | func (self *IndexPooledClient) dial(ctx context.Context) (IndexClient, error) { 84 | url, err := self.lb.PollIndexGate() 85 | if err != nil { 86 | return nil, err 87 | } 88 | 89 | return DialIndexGrpc(url, self.dirConfig) 90 | } 91 | 92 | func (self *IndexPooledClient) acquire(ctx context.Context) (IndexClient, error) { 93 | // Item immediately ready from the pool 94 | select { 95 | case client := <-self.pool: 96 | Logger.Debug().Msg("Reusing a direct client") 97 | return client, nil 98 | default: 99 | } 100 | 101 | // Permission to allocate a new item 102 | ok, _ := <-self.remaining 103 | if ok { 104 | Logger.Debug().Msg("Dialing a new direct client") 105 | return self.dial(ctx) 106 | } 107 | 108 | // No item, No permission ... wait for an item to be released 109 | done := ctx.Done() 110 | Logger.Debug().Msg("Waiting for an idle client") 111 | select { 112 | case client := <-self.pool: 113 | return client, nil 114 | case <-done: 115 | return nil, errors.New("No client ready") 116 | } 117 | } 118 | 119 | func (self *IndexPooledClient) release(c IndexClient) { 120 | if c != nil { 121 | self.pool <- c 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /internal/cmd-data-gate/handlers.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package cmd_data_gate 7 | 8 | import ( 9 | "errors" 10 | "fmt" 11 | ghttp "github.com/jfsmig/object-storage/internal/helpers-http" 12 | "github.com/jfsmig/object-storage/pkg/gunkan" 13 | "net/http" 14 | "strconv" 15 | "strings" 16 | "time" 17 | ) 18 | 19 | func (srv *service) handlePart() ghttp.RequestHandler { 20 | return func(ctx *ghttp.RequestContext) { 21 | pre := time.Now() 22 | id := ctx.Req.URL.Path[len(prefixData):] 23 | switch ctx.Method() { 24 | case "GET", "HEAD": 25 | srv.handleBlobGet(ctx, id) 26 | srv.timeGet.Observe(time.Since(pre).Seconds()) 27 | case "PUT": 28 | srv.handleBlobPut(ctx, id) 29 | srv.timePut.Observe(time.Since(pre).Seconds()) 30 | case "DELETE": 31 | srv.handleBlobDel(ctx, id) 32 | srv.timeDel.Observe(time.Since(pre).Seconds()) 33 | default: 34 | ctx.WriteHeader(http.StatusMethodNotAllowed) 35 | } 36 | } 37 | } 38 | 39 | func (srv *service) handleList() ghttp.RequestHandler { 40 | h := func(ctx *ghttp.RequestContext) { 41 | // Unpack request attributes 42 | q := ctx.Req.URL.Query() 43 | bucket := q.Get("b") 44 | smax := q.Get("max") 45 | marker := q.Get("m") 46 | 47 | if !gunkan.ValidateBucketName(bucket) || !gunkan.ValidateContentName(marker) { 48 | ctx.WriteHeader(http.StatusBadRequest) 49 | return 50 | } 51 | max64, err := strconv.ParseUint(smax, 10, 32) 52 | if err != nil { 53 | ctx.WriteHeader(http.StatusBadRequest) 54 | return 55 | } 56 | max32 := uint32(max64) 57 | 58 | // Query the index about a slice of items 59 | addr, err := srv.lb.PollIndexGate() 60 | if err != nil { 61 | ctx.WriteHeader(http.StatusInternalServerError) 62 | return 63 | } 64 | 65 | client, err := gunkan.DialIndexGrpc(addr, srv.config.dirConfig) 66 | if err != nil { 67 | ctx.WriteHeader(http.StatusInternalServerError) 68 | return 69 | } 70 | 71 | tab, err := client.List(ctx.Req.Context(), gunkan.BK(bucket, marker), max32) 72 | if err != nil { 73 | ctx.ReplyError(err) 74 | return 75 | } 76 | 77 | if len(tab) <= 0 { 78 | ctx.WriteHeader(http.StatusNoContent) 79 | } else { 80 | for _, item := range tab { 81 | fmt.Println(item) 82 | } 83 | } 84 | } 85 | return func(ctx *ghttp.RequestContext) { 86 | pre := time.Now() 87 | h(ctx) 88 | srv.timeList.Observe(time.Since(pre).Seconds()) 89 | } 90 | } 91 | 92 | func (srv *service) handleBlobDel(ctx *ghttp.RequestContext, blobid string) { 93 | ctx.WriteHeader(http.StatusNotImplemented) 94 | } 95 | 96 | func (srv *service) handleBlobGet(ctx *ghttp.RequestContext, blobid string) { 97 | ctx.WriteHeader(http.StatusNotImplemented) 98 | } 99 | 100 | func (srv *service) handleBlobPut(ctx *ghttp.RequestContext, tail string) { 101 | var err error 102 | var policy string 103 | 104 | // Unpack the object name 105 | tokens := strings.Split(tail, "/") 106 | if len(tokens) != 3 { 107 | ctx.ReplyCodeError(http.StatusBadRequest, errors.New("3 tokens expected")) 108 | return 109 | } 110 | 111 | var id gunkan.BlobId 112 | id.Bucket = tokens[0] 113 | id.Content = tokens[1] 114 | id.PartId = tokens[2] 115 | 116 | // Locate the storage policy 117 | policy = ctx.Req.Header.Get(HeaderNameObjectPolicy) 118 | if policy == "" { 119 | policy = "single" 120 | } 121 | 122 | // Find a set of backends 123 | // FIXME(jfsmig): Dumb implementation that only accept the "SINGLE COPY" policy 124 | var url string 125 | url, err = srv.lb.PollBlobStore() 126 | if err != nil { 127 | ctx.ReplyCodeError(http.StatusServiceUnavailable, err) 128 | return 129 | } 130 | 131 | var realid string 132 | var client gunkan.BlobClient 133 | client, err = gunkan.DialBlob(url) 134 | if err != nil { 135 | ctx.ReplyCodeError(http.StatusInternalServerError, err) 136 | return 137 | } 138 | 139 | realid, err = client.Put(ctx.Req.Context(), id, ctx.Input()) 140 | if err != nil { 141 | ctx.ReplyCodeError(http.StatusServiceUnavailable, err) 142 | return 143 | } 144 | ctx.SetHeader(HeaderPrefixCommon+"part-read-id", realid) 145 | ctx.WriteHeader(http.StatusCreated) 146 | } 147 | -------------------------------------------------------------------------------- /pkg/gunkan/part_client_direct.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package gunkan 7 | 8 | import ( 9 | "bufio" 10 | "context" 11 | "io" 12 | "strings" 13 | ) 14 | 15 | type httpPartClient struct { 16 | client HttpSimpleClient 17 | } 18 | 19 | func DialPart(url string) (PartClient, error) { 20 | var err error 21 | var rc httpPartClient 22 | err = rc.client.Init(url) 23 | if err != nil { 24 | return nil, err 25 | } 26 | return &rc, nil 27 | } 28 | 29 | func (self *httpPartClient) Delete(ctx context.Context, id PartId) error { 30 | url := self.client.BuildUrl("/v1/blob") 31 | 32 | req, err := self.client.makeRequest(ctx, "DELETE", url, nil) 33 | if err != nil { 34 | return err 35 | } 36 | 37 | rep, err := self.client.Http.Do(req) 38 | if err != nil { 39 | return err 40 | } 41 | 42 | defer rep.Body.Close() 43 | return MapCodeToError(rep.StatusCode) 44 | } 45 | 46 | func (self *httpPartClient) Get(ctx context.Context, id PartId) (io.ReadCloser, error) { 47 | url := self.client.BuildUrl("/v1/blob") 48 | 49 | req, err := self.client.makeRequest(ctx, "GET", url, nil) 50 | if err != nil { 51 | return nil, err 52 | } 53 | 54 | rep, err := self.client.Http.Do(req) 55 | if err != nil { 56 | return nil, err 57 | } 58 | 59 | switch rep.StatusCode { 60 | case 200, 201, 204: 61 | return rep.Body, nil 62 | default: 63 | return nil, MapCodeToError(rep.StatusCode) 64 | } 65 | } 66 | 67 | func (self *httpPartClient) PutN(ctx context.Context, id PartId, data io.Reader, size int64) error { 68 | url := self.client.BuildUrl("/v1/blob") 69 | 70 | req, err := self.client.makeRequest(ctx, "PUT", url, data) 71 | if err != nil { 72 | return err 73 | } 74 | 75 | req.ContentLength = size 76 | rep, err := self.client.Http.Do(req) 77 | if err != nil { 78 | return err 79 | } 80 | 81 | defer rep.Body.Close() 82 | return MapCodeToError(rep.StatusCode) 83 | } 84 | 85 | func (self *httpPartClient) Put(ctx context.Context, id PartId, data io.Reader) error { 86 | url := self.client.BuildUrl("/v1/blob") 87 | 88 | req, err := self.client.makeRequest(ctx, "PUT", url, data) 89 | if err != nil { 90 | return err 91 | } 92 | 93 | req.ContentLength = -1 94 | rep, err := self.client.Http.Do(req) 95 | if err != nil { 96 | return err 97 | } 98 | 99 | defer rep.Body.Close() 100 | return MapCodeToError(rep.StatusCode) 101 | } 102 | 103 | func (self *httpPartClient) List(ctx context.Context, max uint32) ([]PartId, error) { 104 | return self.listRaw(ctx, max, "") 105 | } 106 | 107 | func (self *httpPartClient) ListAfter(ctx context.Context, max uint32, id PartId) ([]PartId, error) { 108 | return self.listRaw(ctx, max, id.EncodeMarker()) 109 | } 110 | 111 | func (self *httpPartClient) listRaw(ctx context.Context, max uint32, marker string) ([]PartId, error) { 112 | url := self.client.BuildUrl("/v1/list") 113 | 114 | req, err := self.client.makeRequest(ctx, "GET", url, nil) 115 | if err != nil { 116 | return nil, err 117 | } 118 | 119 | rep, err := self.client.Http.Do(req) 120 | if err != nil { 121 | return nil, err 122 | } 123 | 124 | defer rep.Body.Close() 125 | switch rep.StatusCode { 126 | case 200, 201, 204: 127 | return unpackPartIdArray(rep.Body) 128 | default: 129 | return nil, MapCodeToError(rep.StatusCode) 130 | } 131 | } 132 | 133 | func unpackPartIdArray(body io.Reader) ([]PartId, error) { 134 | rc := make([]PartId, 0) 135 | r := bufio.NewReader(body) 136 | for { 137 | if line, err := r.ReadString('\n'); err != nil { 138 | if err == io.EOF { 139 | return rc, nil 140 | } else { 141 | return nil, err 142 | } 143 | } else if len(line) > 0 { 144 | var id PartId 145 | line = strings.Trim(line, "\r\n") 146 | if err = id.Decode(line); err != nil { 147 | return nil, err 148 | } else { 149 | rc = append(rc, id) 150 | } 151 | } 152 | } 153 | } 154 | 155 | func (self *httpPartClient) Info(ctx context.Context) ([]byte, error) { 156 | return self.client.Info(ctx) 157 | } 158 | 159 | func (self *httpPartClient) Health(ctx context.Context) ([]byte, error) { 160 | return self.client.Health(ctx) 161 | } 162 | 163 | func (self *httpPartClient) Metrics(ctx context.Context) ([]byte, error) { 164 | return self.client.Metrics(ctx) 165 | } 166 | -------------------------------------------------------------------------------- /internal/helpers-http/ghttp.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package ghttp 7 | 8 | import ( 9 | "encoding/json" 10 | "github.com/jfsmig/object-storage/pkg/gunkan" 11 | "github.com/prometheus/client_golang/prometheus/promhttp" 12 | "io" 13 | "net/http" 14 | "os" 15 | ) 16 | 17 | type Service struct { 18 | Url string 19 | Info string 20 | 21 | mux *http.ServeMux 22 | } 23 | 24 | type RequestContext struct { 25 | Srv *Service 26 | Req *http.Request 27 | Rep http.ResponseWriter 28 | Err error 29 | Code int 30 | } 31 | 32 | type RequestHandler func(ctx *RequestContext) 33 | 34 | func NewHttpApi(url, info string) *Service { 35 | var srv Service 36 | srv.Url = url 37 | srv.Info = info 38 | 39 | srv.mux = http.NewServeMux() 40 | srv.mux.HandleFunc(gunkan.RouteInfo, getF(srv.handleInfo())) 41 | srv.mux.HandleFunc(gunkan.RouteHealth, getF(srv.handleHealth())) 42 | srv.mux.Handle(gunkan.RouteMetrics, promhttp.Handler()) 43 | return &srv 44 | } 45 | 46 | func (srv *Service) Handler() http.Handler { 47 | return srv.mux 48 | } 49 | 50 | func (srv *Service) Route(pattern string, h RequestHandler) { 51 | srv.mux.HandleFunc(pattern, srv.wrap(h)) 52 | } 53 | 54 | func (srv *Service) handleInfo() http.HandlerFunc { 55 | return func(rep http.ResponseWriter, req *http.Request) { 56 | rep.Header().Set("Content-Type", "text/plain") 57 | _, _ = rep.Write([]byte(srv.Info)) 58 | } 59 | } 60 | 61 | func (srv *Service) handleHealth() http.HandlerFunc { 62 | return func(rep http.ResponseWriter, req *http.Request) { 63 | rep.WriteHeader(http.StatusNoContent) 64 | } 65 | } 66 | 67 | func (srv *Service) wrap(h RequestHandler) http.HandlerFunc { 68 | return func(rep http.ResponseWriter, req *http.Request) { 69 | ctx := RequestContext{Req: req, Rep: rep} 70 | h(&ctx) 71 | gunkan.Logger.Info(). 72 | Str("local", srv.Url). 73 | Str("peer", req.RemoteAddr). 74 | Str("method", req.Method). 75 | Str("url", req.URL.String()). 76 | Err(ctx.Err). 77 | Int("rc", ctx.Code).Msg("access") 78 | } 79 | } 80 | 81 | func (ctx *RequestContext) WriteHeader(code int) { 82 | ctx.Code = code 83 | ctx.Rep.WriteHeader(code) 84 | } 85 | 86 | func (ctx *RequestContext) Write(b []byte) (int, error) { 87 | return ctx.Rep.Write(b) 88 | } 89 | 90 | func (ctx *RequestContext) SetHeader(k, v string) { 91 | ctx.Rep.Header().Set(k, v) 92 | } 93 | 94 | func (ctx *RequestContext) Method() string { 95 | return ctx.Req.Method 96 | } 97 | 98 | func (ctx *RequestContext) Input() io.Reader { 99 | return ctx.Req.Body 100 | } 101 | 102 | func (ctx *RequestContext) Output() io.Writer { 103 | return ctx.Rep 104 | } 105 | 106 | func (ctx *RequestContext) ReplyCodeErrorMsg(code int, err string) { 107 | ctx.Code = code 108 | replySetErrorMsg(ctx.Rep, code, err) 109 | } 110 | 111 | func (ctx *RequestContext) ReplyCodeError(code int, err error) { 112 | ctx.ReplyCodeErrorMsg(code, err.Error()) 113 | } 114 | 115 | func (ctx *RequestContext) ReplyError(err error) { 116 | code := http.StatusInternalServerError 117 | if os.IsNotExist(err) { 118 | code = http.StatusNotFound 119 | } else if os.IsExist(err) { 120 | code = http.StatusConflict 121 | } else if os.IsPermission(err) { 122 | code = http.StatusForbidden 123 | } else if os.IsTimeout(err) { 124 | code = http.StatusRequestTimeout 125 | } 126 | ctx.ReplyCodeErrorMsg(code, err.Error()) 127 | } 128 | 129 | func (ctx *RequestContext) JSON(o interface{}) { 130 | ctx.SetHeader("Content-Type", "text/plain") 131 | json.NewEncoder(ctx.Output()).Encode(o) 132 | } 133 | 134 | func (ctx *RequestContext) ReplySuccess() { 135 | ctx.WriteHeader(http.StatusNoContent) 136 | } 137 | 138 | func replySetErrorMsg(rep http.ResponseWriter, code int, err string) { 139 | rep.Header().Set("X-Error", err) 140 | rep.WriteHeader(code) 141 | } 142 | 143 | func getF(h http.HandlerFunc) http.HandlerFunc { 144 | return func(rep http.ResponseWriter, req *http.Request) { 145 | switch req.Method { 146 | case "GET", "HEAD": 147 | h(rep, req) 148 | default: 149 | replySetErrorMsg(rep, http.StatusMethodNotAllowed, "Only GET or HEAD") 150 | } 151 | } 152 | } 153 | 154 | func Get(h RequestHandler) RequestHandler { 155 | return func(ctx *RequestContext) { 156 | switch ctx.Method() { 157 | case "GET", "HEAD": 158 | h(ctx) 159 | default: 160 | ctx.ReplyCodeErrorMsg(http.StatusMethodNotAllowed, "Only GET or HEAD") 161 | } 162 | } 163 | } 164 | -------------------------------------------------------------------------------- /internal/cmd-index-store-rocksdb/service.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package cmd_index_store_rocksdb 7 | 8 | import ( 9 | "bytes" 10 | "context" 11 | "errors" 12 | "github.com/jfsmig/object-storage/pkg/gunkan" 13 | proto "github.com/jfsmig/object-storage/pkg/gunkan-index-proto" 14 | "github.com/tecbot/gorocksdb" 15 | "google.golang.org/grpc/codes" 16 | "google.golang.org/grpc/status" 17 | "time" 18 | ) 19 | 20 | type serviceConfig struct { 21 | uuid string 22 | addrBind string 23 | addrAnnounce string 24 | dirConfig string 25 | dirBase string 26 | 27 | delayIoError time.Duration 28 | delayFullError time.Duration 29 | } 30 | 31 | type service struct { 32 | cfg serviceConfig 33 | db *gorocksdb.DB 34 | } 35 | 36 | func NewService(cfg serviceConfig) (*service, error) { 37 | options := gorocksdb.NewDefaultOptions() 38 | options.SetCreateIfMissing(true) 39 | db, err := gorocksdb.OpenDb(options, cfg.dirBase) 40 | if err != nil { 41 | return nil, err 42 | } 43 | srv := service{cfg: cfg, db: db} 44 | return &srv, nil 45 | } 46 | 47 | func (srv *service) Put(ctx context.Context, req *proto.PutRequest) (*proto.None, error) { 48 | key := gunkan.BK(req.Base, req.Key) 49 | 50 | encoded := []byte(key.Encode()) 51 | 52 | // FIXME(jfs): check if the KV is present 53 | 54 | opts := gorocksdb.NewDefaultWriteOptions() 55 | defer opts.Destroy() 56 | opts.SetSync(false) 57 | err := srv.db.Put(opts, encoded, []byte(req.Value)) 58 | if err != nil { 59 | return nil, err 60 | } else { 61 | return &proto.None{}, nil 62 | } 63 | } 64 | 65 | func (srv *service) Delete(ctx context.Context, req *proto.DeleteRequest) (*proto.None, error) { 66 | key := gunkan.BK(req.Base, req.Key) 67 | encoded := []byte(key.Encode()) 68 | 69 | // FIXME(jfs): check if the KV is present 70 | 71 | opts := gorocksdb.NewDefaultWriteOptions() 72 | opts.SetSync(false) 73 | defer opts.Destroy() 74 | err := srv.db.Put(opts, encoded, []byte{}) 75 | if err != nil { 76 | return nil, err 77 | } else { 78 | return &proto.None{}, nil 79 | } 80 | } 81 | 82 | func (srv *service) Get(ctx context.Context, req *proto.GetRequest) (*proto.GetReply, error) { 83 | needle := gunkan.BK(req.Base, req.Key) 84 | encoded := []byte(needle.Encode()) 85 | 86 | opts := gorocksdb.NewDefaultReadOptions() 87 | defer opts.Destroy() 88 | opts.SetFillCache(true) 89 | iterator := srv.db.NewIterator(opts) 90 | iterator.Seek(encoded) 91 | if !iterator.Valid() { 92 | return nil, errors.New("Not found") 93 | } 94 | 95 | var got gunkan.BaseKey 96 | sk := iterator.Key() 97 | if err := got.DecodeBytes(sk.Data()); err != nil { 98 | return nil, err 99 | } 100 | 101 | // Latest item wanted 102 | if got.Base != needle.Base || got.Key != needle.Key { 103 | return nil, errors.New("Not found") 104 | } 105 | 106 | return &proto.GetReply{Value: string(iterator.Value().Data())}, nil 107 | } 108 | 109 | func (srv *service) List(ctx context.Context, req *proto.ListRequest) (*proto.ListReply, error) { 110 | if req.Max < 0 { 111 | req.Max = 1 112 | } else if req.Max > gunkan.ListHardMax { 113 | req.Max = gunkan.ListHardMax 114 | } 115 | 116 | if req.Base == "" { 117 | return nil, status.Errorf(codes.InvalidArgument, "Missing base") 118 | } 119 | 120 | var needle []byte 121 | if len(req.Base) > 0 { 122 | if len(req.Marker) > 0 { 123 | needle = []byte(gunkan.BK(req.Base, req.Marker).Encode()) 124 | } else { 125 | needle = []byte(req.Base + ",") 126 | } 127 | } 128 | 129 | opts := gorocksdb.NewDefaultReadOptions() 130 | defer opts.Destroy() 131 | opts.SetFillCache(true) 132 | iterator := srv.db.NewIterator(opts) 133 | 134 | rep := proto.ListReply{} 135 | 136 | iterator.Seek(needle) 137 | for ; iterator.Valid(); iterator.Next() { 138 | if bytes.Compare(iterator.Key().Data(), needle) > 0 { 139 | break 140 | } 141 | } 142 | for ; iterator.Valid(); iterator.Next() { 143 | // Check we didn't reach the max elements 144 | if uint32(len(rep.Items)) > req.Max { 145 | break 146 | } 147 | 148 | // Check the base matches 149 | sk := iterator.Key() 150 | var k gunkan.BaseKey 151 | err := k.DecodeBytes(sk.Data()) 152 | if err != nil { 153 | return nil, status.Errorf(codes.DataLoss, "Malformed DB entry") 154 | } 155 | if k.Base != req.Base { 156 | break 157 | } 158 | 159 | rep.Items = append(rep.Items, k.Key) 160 | } 161 | 162 | return &rep, nil 163 | } 164 | -------------------------------------------------------------------------------- /internal/cmd-blob-store-fs/repository.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package cmd_blob_store_fs 7 | 8 | import ( 9 | "fmt" 10 | "github.com/jfsmig/object-storage/pkg/gunkan" 11 | "golang.org/x/sys/unix" 12 | "math/rand" 13 | "os" 14 | "path/filepath" 15 | "strings" 16 | "sync" 17 | "syscall" 18 | "time" 19 | ) 20 | 21 | type Repo interface { 22 | Create(id gunkan.BlobId) (BlobBuilder, error) 23 | Open(blobId string) (BlobReader, error) 24 | Delete(blobId string) error 25 | } 26 | 27 | type BlobReader interface { 28 | Stream() *os.File 29 | Close() 30 | } 31 | 32 | type BlobBuilder interface { 33 | Stream() *os.File 34 | Commit() (string, error) 35 | Abort() error 36 | } 37 | 38 | type fsPostRepo struct { 39 | fdBase int 40 | pathBase string 41 | 42 | rwRand sync.RWMutex 43 | idRand *rand.Rand 44 | 45 | // Control the way a filename is hashed to get the directory hierarchy 46 | hashWidth uint 47 | 48 | // Control the guarantees given before replying to the client 49 | syncFile bool 50 | syncDir bool 51 | } 52 | 53 | type fsPostRW struct { 54 | file *os.File 55 | repo *fsPostRepo 56 | id gunkan.BlobId 57 | } 58 | 59 | type fsPostRO struct { 60 | file *os.File 61 | repo *fsPostRepo 62 | } 63 | 64 | func MakePostNamed(basedir string) (Repo, error) { 65 | var err error 66 | r := fsPostRepo{ 67 | fdBase: -1, 68 | pathBase: basedir, 69 | hashWidth: 4, 70 | syncFile: false, 71 | syncDir: false} 72 | 73 | r.fdBase, err = syscall.Open(r.pathBase, flagsOpenDir, 0) 74 | if err != nil { 75 | return nil, err 76 | } 77 | 78 | r.idRand = rand.New(rand.NewSource(time.Now().UnixNano())) 79 | 80 | return &r, nil 81 | } 82 | 83 | func (r *fsPostRepo) relpath(objname string) (string, error) { 84 | sb := strings.Builder{} 85 | sb.Grow(16) 86 | if r.hashWidth > 0 { 87 | sb.WriteString(objname[0:r.hashWidth]) 88 | sb.WriteRune('/') 89 | } 90 | sb.WriteString(objname[r.hashWidth:]) 91 | return sb.String(), nil 92 | } 93 | 94 | func (r *fsPostRepo) mkdir(path string, retry bool) error { 95 | err := unix.Mkdirat(r.fdBase, path, 0755) 96 | if err == nil || os.IsExist(err) { 97 | return nil 98 | } 99 | if os.IsNotExist(err) { 100 | if err = r.mkdir(filepath.Dir(path), true); err == nil { 101 | return r.mkdir(path, false) 102 | } 103 | } 104 | return err 105 | } 106 | 107 | func (r *fsPostRepo) createOrRetry(path string, retry bool) (*os.File, error) { 108 | fd, err := unix.Openat(r.fdBase, path, flagsCreate, 0644) 109 | if err != nil { 110 | if retry && os.IsNotExist(err) { 111 | err = r.mkdir(filepath.Dir(path), true) 112 | if err == nil { 113 | return r.createOrRetry(path, false) 114 | } 115 | } 116 | return nil, err 117 | } 118 | 119 | return os.NewFile(uintptr(fd), path), nil 120 | } 121 | 122 | func (r *fsPostRepo) Delete(relpath string) error { 123 | return unix.Unlinkat(r.fdBase, relpath, 0) 124 | } 125 | 126 | func (r *fsPostRepo) nextId() string { 127 | d := (time.Now().UnixNano() / (1024 * 1024 * 256)) % 65536 128 | f := uint32(r.idRand.Int31n(1024 * 1024)) 129 | return fmt.Sprintf("%04X%05X", d, f) 130 | } 131 | 132 | func (r *fsPostRepo) Create(id gunkan.BlobId) (BlobBuilder, error) { 133 | cid := r.nextId() 134 | 135 | pathFinal, err := r.relpath(cid) 136 | if err != nil { 137 | return nil, err 138 | } 139 | 140 | var f *os.File 141 | f, err = r.createOrRetry(pathFinal, true) 142 | return &fsPostRW{file: f, repo: r, id: id}, err 143 | } 144 | 145 | func (r *fsPostRepo) Open(realid string) (BlobReader, error) { 146 | var err error 147 | relpath, err := r.relpath(realid) 148 | if err != nil { 149 | return nil, err 150 | } 151 | 152 | var fd int 153 | fd, err = unix.Openat(r.fdBase, relpath, flagsOpenRead, 0) 154 | if err != nil { 155 | return nil, err 156 | } 157 | 158 | return &fsPostRO{file: os.NewFile(uintptr(fd), relpath), repo: r}, nil 159 | } 160 | 161 | func (f *fsPostRW) Stream() *os.File { 162 | return f.file 163 | } 164 | 165 | func (f *fsPostRW) Abort() error { 166 | if f == nil || f.file == nil { 167 | return nil 168 | } 169 | err := unix.Unlinkat(f.repo.fdBase, f.file.Name(), 0) 170 | _ = f.file.Close() 171 | return err 172 | } 173 | 174 | func (f *fsPostRW) Commit() (string, error) { 175 | if f == nil || f.file == nil { 176 | panic("Invalid file being commited") 177 | } 178 | 179 | var err error 180 | if f.repo.syncFile { 181 | err = f.file.Sync() 182 | } 183 | if f.repo.syncDir { 184 | err = unix.Fdatasync(int(f.file.Fd())) 185 | } 186 | 187 | _ = f.file.Close() 188 | return f.file.Name(), err 189 | } 190 | 191 | func (f *fsPostRO) Stream() *os.File { 192 | return f.file 193 | } 194 | 195 | func (f *fsPostRO) Close() { 196 | _ = f.file.Close() 197 | } 198 | -------------------------------------------------------------------------------- /pkg/gunkan/blob_client_direct.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package gunkan 7 | 8 | import ( 9 | "bufio" 10 | "context" 11 | "io" 12 | "io/ioutil" 13 | "strings" 14 | ) 15 | 16 | type httpBlobClient struct { 17 | client HttpSimpleClient 18 | } 19 | 20 | func DialBlob(url string) (BlobClient, error) { 21 | var err error 22 | var rc httpBlobClient 23 | err = rc.client.Init(url) 24 | if err != nil { 25 | return nil, err 26 | } 27 | return &rc, nil 28 | } 29 | 30 | func (self *httpBlobClient) Delete(ctx context.Context, realid string) error { 31 | b := strings.Builder{} 32 | b.WriteString("http://") 33 | b.WriteString(self.client.Endpoint) 34 | b.WriteString("/v1/blob/") 35 | b.WriteString(realid) 36 | 37 | req, err := self.client.makeRequest(ctx, "DELETE", b.String(), nil) 38 | if err != nil { 39 | return err 40 | } 41 | 42 | rep, err := self.client.Http.Do(req) 43 | if err != nil { 44 | return err 45 | } 46 | 47 | defer rep.Body.Close() 48 | return MapCodeToError(rep.StatusCode) 49 | } 50 | 51 | func (self *httpBlobClient) Get(ctx context.Context, realid string) (io.ReadCloser, error) { 52 | b := strings.Builder{} 53 | b.WriteString("http://") 54 | b.WriteString(self.client.Endpoint) 55 | b.WriteString("/v1/blob/") 56 | b.WriteString(realid) 57 | 58 | req, err := self.client.makeRequest(ctx, "GET", b.String(), nil) 59 | if err != nil { 60 | return nil, err 61 | } 62 | 63 | rep, err := self.client.Http.Do(req) 64 | if err != nil { 65 | return nil, err 66 | } 67 | 68 | switch rep.StatusCode { 69 | case 200, 201, 204: 70 | return rep.Body, nil 71 | default: 72 | return nil, MapCodeToError(rep.StatusCode) 73 | } 74 | } 75 | 76 | func (self *httpBlobClient) PutN(ctx context.Context, id BlobId, data io.Reader, size int64) (string, error) { 77 | b := strings.Builder{} 78 | b.WriteString("http://") 79 | b.WriteString(self.client.Endpoint) 80 | b.WriteString("/v1/blob/") 81 | id.EncodeIn(&b) 82 | 83 | req, err := self.client.makeRequest(ctx, "PUT", b.String(), data) 84 | if err != nil { 85 | return "", err 86 | } 87 | 88 | req.ContentLength = size 89 | rep, err := self.client.Http.Do(req) 90 | if err != nil { 91 | return "", err 92 | } 93 | 94 | defer rep.Body.Close() 95 | return "", MapCodeToError(rep.StatusCode) 96 | } 97 | 98 | func (self *httpBlobClient) Put(ctx context.Context, id BlobId, data io.Reader) (string, error) { 99 | b := strings.Builder{} 100 | b.WriteString("http://") 101 | b.WriteString(self.client.Endpoint) 102 | b.WriteString("/v1/blob/") 103 | id.EncodeIn(&b) 104 | 105 | req, err := self.client.makeRequest(ctx, "PUT", b.String(), data) 106 | if err != nil { 107 | return "", err 108 | } 109 | 110 | req.ContentLength = -1 111 | rep, err := self.client.Http.Do(req) 112 | if err != nil { 113 | return "", err 114 | } 115 | 116 | defer rep.Body.Close() 117 | return "", MapCodeToError(rep.StatusCode) 118 | } 119 | 120 | func (self *httpBlobClient) List(ctx context.Context, max uint) ([]BlobListItem, error) { 121 | return self.listRaw(ctx, max, "") 122 | } 123 | 124 | func (self *httpBlobClient) ListAfter(ctx context.Context, max uint, marker string) ([]BlobListItem, error) { 125 | return self.listRaw(ctx, max, marker) 126 | } 127 | 128 | func (self *httpBlobClient) listRaw(ctx context.Context, max uint, marker string) ([]BlobListItem, error) { 129 | b := strings.Builder{} 130 | b.WriteString("http://") 131 | b.WriteString(self.client.Endpoint) 132 | b.WriteString("/v1/list") 133 | if len(marker) > 0 { 134 | b.WriteRune('/') 135 | b.WriteString(marker) 136 | } 137 | 138 | req, err := self.client.makeRequest(ctx, "GET", b.String(), nil) 139 | if err != nil { 140 | return nil, err 141 | } 142 | 143 | rep, err := self.client.Http.Do(req) 144 | if err != nil { 145 | return nil, err 146 | } 147 | 148 | defer rep.Body.Close() 149 | switch rep.StatusCode { 150 | case 200, 201, 204: 151 | return unpackBlobIdArray(rep.Body) 152 | default: 153 | return nil, MapCodeToError(rep.StatusCode) 154 | } 155 | } 156 | 157 | func unpackBlobIdArray(body io.Reader) ([]BlobListItem, error) { 158 | rc := make([]BlobListItem, 0) 159 | r := bufio.NewReader(body) 160 | for { 161 | if line, err := r.ReadString('\n'); err != nil { 162 | if err == io.EOF { 163 | return rc, nil 164 | } else { 165 | return nil, err 166 | } 167 | } else if len(line) > 0 { 168 | var id BlobId 169 | line = strings.Trim(line, "\r\n") 170 | if id, err = DecodeBlobId(line); err != nil { 171 | return nil, err 172 | } else { 173 | rc = append(rc, BlobListItem{"", id}) 174 | } 175 | } 176 | } 177 | } 178 | 179 | func (self *httpBlobClient) srvGet(ctx context.Context, tag string) ([]byte, error) { 180 | b := strings.Builder{} 181 | b.WriteString("http://") 182 | b.WriteString(self.client.Endpoint) 183 | b.WriteString(tag) 184 | 185 | req, err := self.client.makeRequest(ctx, "GET", b.String(), nil) 186 | if err != nil { 187 | return []byte{}, err 188 | } 189 | 190 | rep, err := self.client.Http.Do(req) 191 | if err != nil { 192 | return []byte{}, err 193 | } 194 | 195 | defer rep.Body.Close() 196 | return ioutil.ReadAll(rep.Body) 197 | } 198 | 199 | func (self *httpBlobClient) Info(ctx context.Context) ([]byte, error) { 200 | return self.client.Info(ctx) 201 | } 202 | 203 | func (self *httpBlobClient) Health(ctx context.Context) ([]byte, error) { 204 | return self.client.Health(ctx) 205 | } 206 | 207 | func (self *httpBlobClient) Metrics(ctx context.Context) ([]byte, error) { 208 | return self.client.Metrics(ctx) 209 | } 210 | -------------------------------------------------------------------------------- /internal/cmd-index-client/cmd.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package cmd_index_client 7 | 8 | import ( 9 | "bufio" 10 | "errors" 11 | "fmt" 12 | "github.com/jfsmig/object-storage/pkg/gunkan" 13 | "github.com/spf13/cobra" 14 | "github.com/spf13/pflag" 15 | "io" 16 | "os" 17 | ) 18 | 19 | func MainCommand() *cobra.Command { 20 | cmd := &cobra.Command{ 21 | Use: "cli", 22 | Aliases: []string{"client"}, 23 | Short: "Query a KV server", 24 | RunE: func(cmd *cobra.Command, args []string) error { 25 | return cobra.ErrSubCommandRequired 26 | }, 27 | } 28 | cmd.AddCommand(PutCommand()) 29 | cmd.AddCommand(GetCommand()) 30 | cmd.AddCommand(DeleteCommand()) 31 | cmd.AddCommand(ListCommand()) 32 | return cmd 33 | } 34 | 35 | type config struct { 36 | url string 37 | dirConfig string 38 | } 39 | 40 | func (cfg *config) prepare(fs *pflag.FlagSet) { 41 | fs.StringVar(&cfg.url, "url", "", "IP:PORT endpoint of the service to contact") 42 | fs.StringVar(&cfg.dirConfig, "f", "", "IP:PORT endpoint of the service to contact") 43 | } 44 | 45 | func (cfg *config) dial() (gunkan.IndexClient, error) { 46 | if cfg.url != "" { 47 | gunkan.Logger.Debug().Msg("Explicit Index service endpoint") 48 | return gunkan.DialIndexGrpc(cfg.url, cfg.dirConfig) 49 | } else { 50 | gunkan.Logger.Debug().Msg("Polling an Index gate service endpoint") 51 | return gunkan.DialIndexPooled(cfg.dirConfig) 52 | } 53 | } 54 | 55 | func PutCommand() *cobra.Command { 56 | var cfg config 57 | var flagStdIn bool 58 | 59 | cmd := &cobra.Command{ 60 | Use: "put", 61 | Aliases: []string{"set"}, 62 | Short: "Check a service is up", 63 | RunE: func(cmd *cobra.Command, args []string) error { 64 | client, err := cfg.dial() 65 | if err != nil { 66 | return err 67 | } 68 | if flagStdIn { 69 | r := bufio.NewReader(os.Stdin) 70 | var base, key, value string 71 | for { 72 | n, err := fmt.Fscanln(r, &base, &key, &value) 73 | if err == io.EOF { 74 | return nil 75 | } 76 | if err != nil { 77 | return err 78 | } 79 | if n != 3 { 80 | return errors.New("Invalid line") 81 | } 82 | 83 | k := gunkan.BK(base, key) 84 | err = client.Put(cmd.Context(), k, value) 85 | if err != nil { 86 | gunkan.Logger.Warn(). 87 | Str("base", base).Str("key", key).Str("value", value). 88 | Err(err) 89 | } 90 | } 91 | } else { 92 | if len(args) != 3 { 93 | return errors.New("Missing BASE, KEY or VALUE") 94 | } 95 | key := gunkan.BK(args[0], args[1]) 96 | value := args[2] 97 | return client.Put(cmd.Context(), key, value) 98 | } 99 | }, 100 | } 101 | 102 | cfg.prepare(cmd.Flags()) 103 | cmd.Flags().BoolVarP(&flagStdIn, "stdin", "i", flagStdIn, "Consume triples from stdin") 104 | return cmd 105 | } 106 | 107 | func ListCommand() *cobra.Command { 108 | var cfg config 109 | var maxItems uint32 = gunkan.ListHardMax 110 | var flagFull bool 111 | 112 | cmd := &cobra.Command{ 113 | Use: "list", 114 | Aliases: []string{"ls"}, 115 | Short: "Get a slice of keys from a KV server", 116 | RunE: func(cmd *cobra.Command, args []string) error { 117 | client, err := cfg.dial() 118 | if err != nil { 119 | return err 120 | } 121 | if len(args) < 1 { 122 | return errors.New("Missing BASE") 123 | } 124 | if len(args) > 3 { 125 | return errors.New("Too many MARKER") 126 | } 127 | 128 | var base, marker string 129 | base = args[0] 130 | if len(args) == 2 { 131 | marker = args[1] 132 | } 133 | 134 | for { 135 | key := gunkan.BK(base, marker) 136 | items, err := client.List(cmd.Context(), key, maxItems) 137 | if err != nil { 138 | return err 139 | } 140 | if len(items) <= 0 { 141 | break 142 | } 143 | for _, item := range items { 144 | fmt.Println(item) 145 | } 146 | if flagFull { 147 | marker = items[len(items)-1] 148 | } else { 149 | break 150 | } 151 | } 152 | return nil 153 | }, 154 | } 155 | 156 | cmd.Flags().BoolVarP(&flagFull, "full", "f", flagFull, "Iterate to the end of the list") 157 | cmd.Flags().Uint32VarP(&maxItems, "max", "n", maxItems, "Hint on the number of items received") 158 | cfg.prepare(cmd.Flags()) 159 | return cmd 160 | } 161 | 162 | func GetCommand() *cobra.Command { 163 | var cfg config 164 | 165 | cmd := &cobra.Command{ 166 | Use: "get", 167 | Aliases: []string{"fetch", "retrieve"}, 168 | Short: "Get a value from a KV server", 169 | RunE: func(cmd *cobra.Command, args []string) error { 170 | client, err := cfg.dial() 171 | if err != nil { 172 | return err 173 | } 174 | 175 | // Unpack the positional arguments 176 | if len(args) < 2 { 177 | return errors.New("Missing BASE or KEY") 178 | } 179 | base := args[0] 180 | 181 | for _, key := range args[1:] { 182 | value, err := client.Get(cmd.Context(), gunkan.BK(base, key)) 183 | if err != nil { 184 | return err 185 | } 186 | fmt.Printf("%s %s", key, value) 187 | } 188 | return nil 189 | }, 190 | } 191 | 192 | cfg.prepare(cmd.Flags()) 193 | return cmd 194 | } 195 | 196 | func DeleteCommand() *cobra.Command { 197 | var cfg config 198 | 199 | cmd := &cobra.Command{ 200 | Use: "del", 201 | Aliases: []string{"delete", "remove", "erase", "rm"}, 202 | Short: "Delete an entry from a KV service", 203 | RunE: func(cmd *cobra.Command, args []string) error { 204 | client, err := cfg.dial() 205 | if err != nil { 206 | return err 207 | } 208 | if len(args) != 2 { 209 | return errors.New("Missing BASE and/or KEY") 210 | } 211 | 212 | return client.Delete(cmd.Context(), gunkan.BK(args[0], args[1])) 213 | }, 214 | } 215 | 216 | cfg.prepare(cmd.Flags()) 217 | return cmd 218 | } 219 | -------------------------------------------------------------------------------- /ci/run.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright (C) 2019-2020 OpenIO SAS 4 | # This Source Code Form is subject to the terms of the Mozilla Public 5 | # License, v. 2.0. If a copy of the MPL was not distributed with this 6 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 7 | 8 | import os 9 | import subprocess 10 | from string import Template 11 | import time 12 | import tempfile 13 | 14 | BASEDIR = tempfile.mkdtemp(suffix='-test', prefix='gunkan-') 15 | DATADIR = BASEDIR + "/data" 16 | CFGDIR = BASEDIR + "/etc" 17 | 18 | DC = "dc0" 19 | ip="127.0.0.1" 20 | 21 | consul_tpl = Template("""{ 22 | "node_name": "test-node", 23 | "datacenter": "test-dc", 24 | "data_dir": "$vol", 25 | "log_level": "INFO", 26 | "server": true, 27 | "enable_syslog": true, 28 | "syslog_facility": "LOCAL0", 29 | "ui": true, 30 | "serf_lan": "$ip", 31 | "serf_wan": "$ip", 32 | "bind_addr": "$ip", 33 | "client_addr": "$ip" 34 | }""") 35 | 36 | 37 | consul_srv_http_tpl = Template("""{ 38 | "service": { 39 | "check": { 40 | "id": "$id", 41 | "interval": "2s", 42 | "timeout": "1s", 43 | "http": "http://$ip:$port/health" 44 | }, 45 | "id": "$id", 46 | "name": "$type", 47 | "tags": [ "$tag" ], 48 | "address": "$ip", 49 | "port": $port 50 | } 51 | }""") 52 | 53 | consul_srv_grpc_tpl = Template("""{ 54 | "service": { 55 | "checks": [ 56 | { "interval": "2s", "timeout": "1s", "tcp": "$ip:$port" } 57 | ], 58 | "id": "$id", 59 | "name": "$type", 60 | "tags": [ "$tag" ], 61 | "address": "$ip", 62 | "port": $port 63 | } 64 | }""") 65 | 66 | 67 | def stateless(t, num, e): 68 | uid = t + '-' + str(num) 69 | return {"tag": t, "type": t, "id": uid, 70 | "ip": ip, "port": 6000 + num, "exe": e, 71 | "vol": None, "cfg": CFGDIR} 72 | 73 | 74 | def statefull(t, num, e): 75 | uid = t + '-' + str(num) 76 | return {"tag": t, "type": uid, "id": uid, 77 | "ip": ip, "port": 6000 + num, "exe": e, 78 | "vol": DATADIR + '/' + t + '-' + str(num), "cfg": CFGDIR} 79 | 80 | 81 | def do(*args): 82 | subprocess.check_call(args) 83 | 84 | 85 | def generate_certificate(path): 86 | def rel(s): 87 | return path + '/' + s 88 | with open(rel('certificate.conf'), 'w') as f: 89 | f.write(''' 90 | [ req ] 91 | prompt = no 92 | default_bits = 4096 93 | distinguished_name = req_distinguished_name 94 | req_extensions = req_ext 95 | 96 | [ req_distinguished_name ] 97 | C=FR 98 | ST=Nord 99 | L=Hem 100 | O=OpenIO 101 | OU=R&D 102 | CN=localhost 103 | 104 | [ req_ext ] 105 | subjectAltName = @alt_names 106 | 107 | [alt_names] 108 | DNS.1 = hostname.domain.tld 109 | DNS.2 = hostname 110 | IP.1 = 127.0.0.1 111 | ''') 112 | do('openssl', 'genrsa', 113 | '-out', rel('ca.key'), '4096') 114 | do('openssl', 'req', '-new', 115 | '-x509', '-key', rel('ca.key'), '-sha256', 116 | '-subj', "/C=FR/ST=Nord/O=CA, Inc./CN=localhost", '-days', '365', 117 | '-out', rel('ca.cert')) 118 | do('openssl', 'genrsa', 119 | '-out', rel('service.key'), '4096') 120 | do('openssl', 'req', '-new', 121 | '-key', rel('service.key'), '-out', rel('service.csr'), 122 | '-config', rel('certificate.conf')) 123 | do('openssl', 'x509', '-req', 124 | '-in', rel('service.csr'), 125 | '-CA', rel('ca.cert'), '-CAkey', rel('ca.key'), '-CAcreateserial', 126 | '-out', rel('service.pem'), '-days', '365', '-sha256', 127 | '-extfile', rel('certificate.conf'), 128 | '-extensions', 'req_ext') 129 | do('openssl', 'x509', 130 | '-in', rel('service.pem'), '-text', '-noout') 131 | 132 | 133 | def sequence(start=0): 134 | while True: 135 | yield start 136 | start+=1 137 | 138 | 139 | def services(): 140 | port = sequence() 141 | for _i in range(3): 142 | yield "grpc", statefull("gkindex-store", next(port), "gunkan-index-store-rocksdb") 143 | for _i in range(3): 144 | yield "http", statefull("gkblob-store", next(port), "gunkan-blob-store-fs") 145 | for _i in range(3): 146 | yield "grpc", stateless("gkindex-gate", next(port), "gunkan-index-gate") 147 | for _i in range(3): 148 | yield "http", stateless("gkdata-gate", next(port), "gunkan-data-gate") 149 | 150 | 151 | # Create the working directories 152 | for kind, srv in services(): 153 | try: 154 | if srv['vol']: 155 | os.makedirs(srv['vol']) 156 | except OSError: 157 | pass 158 | try: 159 | if srv['cfg']: 160 | os.makedirs(srv['cfg']) 161 | except OSError: 162 | pass 163 | 164 | # Populate the consul configuration 165 | try: 166 | os.makedirs(CFGDIR + '/consul-0.d') 167 | except OSError: 168 | pass 169 | 170 | with open(CFGDIR + '/consul-0.json', 'w') as f: 171 | f.write(consul_tpl.safe_substitute(**{'vol': DATADIR + '/consul-0', 'ip': ip})) 172 | 173 | for kind, srv in services(): 174 | with open(CFGDIR + '/consul-0.d/srv-' + srv['id'] + '.json', 'w') as f: 175 | if kind =='http': 176 | f.write(consul_srv_http_tpl.safe_substitute(**srv)) 177 | elif kind == 'grpc': 178 | f.write(consul_srv_grpc_tpl.safe_substitute(**srv)) 179 | else: 180 | raise ValueError("Invalid service kind") 181 | 182 | # Generate a certificate that will be used by all the services. 183 | generate_certificate(CFGDIR) 184 | 185 | # Start the services 186 | children = list() 187 | for kind, srv in services(): 188 | endpoint = srv['ip'] + ':' + str(srv['port']) 189 | cmd = [srv['exe'], '--tls', srv['cfg'], endpoint] 190 | if srv['vol']: 191 | cmd.append(srv['vol']) 192 | print(repr(cmd)) 193 | child = subprocess.Popen(cmd) 194 | children.append(child) 195 | 196 | consul = subprocess.Popen(( 197 | 'consul', 'agent', '-server', '-bootstrap', '-dev', '-ui', 198 | '-config-file', CFGDIR + '/consul-0.json', 199 | '-config-dir', CFGDIR + '/consul-0.d')) 200 | children.append(consul) 201 | 202 | # Wait for a termination event 203 | try: 204 | while True: 205 | time.sleep(1.0) 206 | except: 207 | pass 208 | 209 | # Final cleanup 210 | for child in children: 211 | child.terminate() 212 | for child in children: 213 | child.wait() 214 | -------------------------------------------------------------------------------- /internal/cmd-index-gate/service.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2019-2020 OpenIO SAS 2 | // This Source Code Form is subject to the terms of the Mozilla Public 3 | // License, v. 2.0. If a copy of the MPL was not distributed with this 4 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 | 6 | package cmd_index_gate 7 | 8 | import ( 9 | "context" 10 | "errors" 11 | helpers_grpc "github.com/jfsmig/object-storage/internal/helpers-grpc" 12 | "github.com/jfsmig/object-storage/pkg/gunkan" 13 | proto "github.com/jfsmig/object-storage/pkg/gunkan-index-proto" 14 | "google.golang.org/grpc" 15 | "sync" 16 | "time" 17 | ) 18 | 19 | const ( 20 | parallelismPut = 5 21 | parallelismGet = 5 22 | parallelismDelete = 5 23 | parallelismList = 5 24 | ) 25 | 26 | type serviceConfig struct { 27 | uuid string 28 | addrBind string 29 | addrAnnounce string 30 | dirConfig string 31 | } 32 | 33 | type service struct { 34 | cfg serviceConfig 35 | 36 | balancer gunkan.Balancer 37 | catalog gunkan.Catalog 38 | 39 | wg sync.WaitGroup 40 | rw sync.RWMutex 41 | back map[string]*grpc.ClientConn 42 | flag_running bool 43 | } 44 | 45 | func NewService(config serviceConfig) (*service, error) { 46 | var err error 47 | 48 | srv := service{} 49 | srv.cfg = config 50 | srv.flag_running = true 51 | srv.back = make(map[string]*grpc.ClientConn) 52 | 53 | srv.catalog, err = gunkan.NewCatalogDefault() 54 | if err != nil { 55 | return nil, err 56 | } 57 | 58 | srv.balancer, err = gunkan.NewBalancerSimple(srv.catalog) 59 | if err != nil { 60 | return nil, err 61 | } 62 | 63 | srv.wg.Add(1) 64 | go func() { 65 | defer srv.wg.Done() 66 | for srv.flag_running { 67 | tick := time.After(1 * time.Second) 68 | <-tick 69 | srv.reload() 70 | } 71 | }() 72 | return &srv, nil 73 | } 74 | 75 | func (srv *service) reload() { 76 | srv.rw.Lock() 77 | defer srv.rw.Unlock() 78 | 79 | // Get all the declared backends 80 | addrs, err := srv.catalog.ListIndexStore() 81 | if err != nil { 82 | gunkan.Logger.Warn().Err(err).Msg("Discovery: index stores") 83 | return 84 | } 85 | 86 | // Open a connection to each new declared backend. 87 | // We avoid closing/reopening connections to stable backends 88 | for _, a := range addrs { 89 | if c, ok := srv.back[a]; ok && c != nil { 90 | continue 91 | } 92 | c, err := helpers_grpc.DialTLSInsecure(a) 93 | if err != nil { 94 | gunkan.Logger.Warn().Err(err).Str("to", a).Msg("Connection error to index") 95 | srv.back[a] = nil 96 | } else { 97 | srv.back[a] = c 98 | } 99 | } 100 | } 101 | 102 | func (srv *service) Join() { 103 | srv.flag_running = false 104 | srv.wg.Wait() 105 | } 106 | 107 | type targetError struct { 108 | addr string 109 | err error 110 | } 111 | 112 | type targetErrorValue struct { 113 | targetError 114 | value string 115 | version uint64 116 | } 117 | 118 | type targetErrorList struct { 119 | targetError 120 | items []string 121 | } 122 | 123 | type targetInput struct { 124 | addr string 125 | cnx *grpc.ClientConn 126 | } 127 | 128 | func mergeTargetError(chans ...<-chan targetError) <-chan targetError { 129 | var wg sync.WaitGroup 130 | out := make(chan targetError) 131 | consume := func(input <-chan targetError) { 132 | for i := range input { 133 | out <- i 134 | } 135 | wg.Done() 136 | } 137 | 138 | wg.Add(len(chans)) 139 | for _, c := range chans { 140 | go consume(c) 141 | } 142 | 143 | go func() { 144 | wg.Wait() 145 | close(out) 146 | }() 147 | 148 | return out 149 | } 150 | 151 | func mergeTargetValueError(chans ...<-chan targetErrorValue) <-chan targetErrorValue { 152 | var wg sync.WaitGroup 153 | out := make(chan targetErrorValue) 154 | consume := func(input <-chan targetErrorValue) { 155 | for i := range input { 156 | out <- i 157 | } 158 | wg.Done() 159 | } 160 | 161 | wg.Add(len(chans)) 162 | for _, c := range chans { 163 | go consume(c) 164 | } 165 | 166 | go func() { 167 | wg.Wait() 168 | close(out) 169 | }() 170 | 171 | return out 172 | } 173 | 174 | func mergeTargetValueList(chans ...<-chan targetErrorList) <-chan targetErrorList { 175 | var wg sync.WaitGroup 176 | out := make(chan targetErrorList) 177 | consume := func(input <-chan targetErrorList) { 178 | for i := range input { 179 | out <- i 180 | } 181 | wg.Done() 182 | } 183 | 184 | wg.Add(len(chans)) 185 | for _, c := range chans { 186 | go consume(c) 187 | } 188 | 189 | go func() { 190 | wg.Wait() 191 | close(out) 192 | }() 193 | 194 | return out 195 | } 196 | 197 | func (srv *service) Put(ctx context.Context, req *proto.PutRequest) (*proto.None, error) { 198 | work := func(input <-chan targetInput) <-chan targetError { 199 | out := make(chan targetError, 1) 200 | go func() { 201 | for i := range input { 202 | cli := proto.NewIndexClient(i.cnx) 203 | _, err := cli.Put(ctx, req) 204 | out <- targetError{i.addr, err} 205 | } 206 | close(out) 207 | }() 208 | return out 209 | } 210 | 211 | srv.rw.RLock() 212 | defer srv.rw.RUnlock() 213 | 214 | in := make(chan targetInput, len(srv.back)) 215 | outv := make([]<-chan targetError, 0) 216 | for i := 0; i < parallelismPut; i++ { 217 | outv = append(outv, work(in)) 218 | } 219 | out := mergeTargetError(outv...) 220 | 221 | for addr, cnx := range srv.back { 222 | in <- targetInput{addr: addr, cnx: cnx} 223 | } 224 | close(in) 225 | any := false 226 | for err := range out { 227 | if err.err == nil { 228 | gunkan.Logger.Debug(). 229 | Str("op", "PUT").Str("k", req.Key).Str("srv", err.addr) 230 | any = true 231 | } else { 232 | gunkan.Logger.Warn(). 233 | Str("op", "PUT").Str("k", req.Key).Str("srv", err.addr).Err(err.err) 234 | } 235 | } 236 | 237 | if !any { 238 | return nil, errors.New("No backend replied") 239 | } else { 240 | return &proto.None{}, nil 241 | } 242 | } 243 | 244 | func (srv *service) Delete(ctx context.Context, req *proto.DeleteRequest) (*proto.None, error) { 245 | work := func(input <-chan targetInput) <-chan targetError { 246 | out := make(chan targetError) 247 | go func() { 248 | for i := range input { 249 | cli := proto.NewIndexClient(i.cnx) 250 | _, err := cli.Delete(ctx, req) 251 | out <- targetError{i.addr, err} 252 | } 253 | close(out) 254 | }() 255 | return out 256 | } 257 | 258 | srv.rw.RLock() 259 | in := make(chan targetInput, len(srv.back)) 260 | outv := make([]<-chan targetError, 0) 261 | for i := 0; i < parallelismDelete; i++ { 262 | outv = append(outv, work(in)) 263 | } 264 | out := mergeTargetError(outv...) 265 | for addr, cnx := range srv.back { 266 | in <- targetInput{addr: addr, cnx: cnx} 267 | } 268 | close(in) 269 | any := false 270 | for err := range out { 271 | if err.err == nil { 272 | gunkan.Logger.Debug(). 273 | Str("op", "DEL").Str("k", req.Key).Str("srv", err.addr) 274 | any = true 275 | } else { 276 | gunkan.Logger.Debug(). 277 | Str("op", "DEL").Str("k", req.Key).Str("srv", err.addr).Err(err.err) 278 | } 279 | } 280 | srv.rw.RUnlock() 281 | 282 | if !any { 283 | return nil, errors.New("No backend replied") 284 | } else { 285 | return &proto.None{}, nil 286 | } 287 | } 288 | 289 | func (srv *service) Get(ctx context.Context, req *proto.GetRequest) (*proto.GetReply, error) { 290 | work := func(input <-chan targetInput) <-chan targetErrorValue { 291 | out := make(chan targetErrorValue, 1) 292 | go func() { 293 | for i := range input { 294 | cli := proto.NewIndexClient(i.cnx) 295 | rep, err := cli.Get(ctx, req) 296 | rc := targetErrorValue{} 297 | rc.addr = i.addr 298 | rc.err = err 299 | if err == nil { 300 | rc.value = rep.Value 301 | rc.version = rep.Version 302 | } 303 | out <- rc 304 | } 305 | close(out) 306 | }() 307 | return out 308 | } 309 | 310 | srv.rw.RLock() 311 | srv.rw.RUnlock() 312 | 313 | in := make(chan targetInput, len(srv.back)) 314 | outv := make([]<-chan targetErrorValue, 0) 315 | for i := 0; i < parallelismGet; i++ { 316 | outv = append(outv, work(in)) 317 | } 318 | out := mergeTargetValueError(outv...) 319 | 320 | for addr, cnx := range srv.back { 321 | in <- targetInput{addr: addr, cnx: cnx} 322 | } 323 | close(in) 324 | 325 | any := false 326 | rep := proto.GetReply{Value: "", Version: 0} 327 | for x := range out { 328 | if x.err != nil { 329 | gunkan.Logger.Warn().Str("op", "GET").Str("k", req.Key).Str("srv", x.addr).Err(x.err) 330 | } else { 331 | any = true 332 | if x.version > rep.Version { 333 | rep.Version = x.version 334 | rep.Value = x.value 335 | } 336 | } 337 | } 338 | 339 | if !any { 340 | return nil, errors.New("No backend replied") 341 | } else { 342 | return &rep, nil 343 | } 344 | } 345 | 346 | func (srv *service) List(ctx context.Context, req *proto.ListRequest) (*proto.ListReply, error) { 347 | work := func(input <-chan targetInput) <-chan targetErrorList { 348 | out := make(chan targetErrorList, 1) 349 | go func() { 350 | for i := range input { 351 | cli := proto.NewIndexClient(i.cnx) 352 | rep, err := cli.List(ctx, req) 353 | rc := targetErrorList{} 354 | rc.addr = i.addr 355 | rc.err = err 356 | if err == nil { 357 | rc.items = rep.Items[:] 358 | } 359 | out <- rc 360 | } 361 | close(out) 362 | }() 363 | return out 364 | } 365 | 366 | if req.Max <= 0 { 367 | req.Max = 1 368 | } else if req.Max > gunkan.ListHardMax { 369 | req.Max = gunkan.ListHardMax 370 | } 371 | 372 | srv.rw.RLock() 373 | srv.rw.RUnlock() 374 | 375 | in := make(chan targetInput, len(srv.back)) 376 | outv := make([]<-chan targetErrorList, 0) 377 | for i := 0; i < parallelismList; i++ { 378 | outv = append(outv, work(in)) 379 | } 380 | out := mergeTargetValueList(outv...) 381 | 382 | for addr, cnx := range srv.back { 383 | in <- targetInput{addr: addr, cnx: cnx} 384 | } 385 | close(in) 386 | 387 | any := false 388 | rep := proto.ListReply{} 389 | 390 | tabs := make([][]string, 0) 391 | for x := range out { 392 | if x.err != nil { 393 | gunkan.Logger.Info(). 394 | Str("op", "LIST"). 395 | Str("k", req.Marker).Str("srv", x.addr). 396 | Err(x.err) 397 | } else { 398 | any = true 399 | tabs = append(tabs, x.items) 400 | } 401 | } 402 | 403 | for x := range keepSingleDedup(tabs, req.Max) { 404 | rep.Items = append(rep.Items, x) 405 | } 406 | if !any { 407 | return nil, errors.New("No backend replied") 408 | } else { 409 | return &rep, nil 410 | } 411 | } 412 | 413 | func keepSingleDedup(tabs [][]string, max uint32) <-chan string { 414 | nbTabs := len(tabs) 415 | maxes := make([]int, nbTabs, nbTabs) 416 | indices := make([]int, nbTabs, nbTabs) 417 | for i, tab := range tabs { 418 | maxes[i] = len(tab) 419 | } 420 | out := make(chan string, 1024) 421 | 422 | min := func() string { 423 | var min string 424 | var iMin int 425 | for i, tab := range tabs { 426 | if indices[i] >= maxes[i] { 427 | continue 428 | } 429 | obj := tab[indices[i]] 430 | if min == "" || obj < min { 431 | iMin = i 432 | min = obj 433 | } 434 | } 435 | if min != "" { 436 | indices[iMin]++ 437 | } 438 | return min 439 | } 440 | 441 | var last string 442 | poll := func() string { 443 | for { 444 | obj := min() 445 | if obj == "" { 446 | return "" 447 | } 448 | if last == "" { 449 | last = obj 450 | return last 451 | } 452 | if last == obj { 453 | continue 454 | } 455 | last = obj 456 | return last 457 | } 458 | } 459 | 460 | go func() { 461 | sent := uint32(0) 462 | for obj := poll(); obj != ""; obj = poll() { 463 | if sent >= max { 464 | break 465 | } 466 | out <- obj 467 | sent++ 468 | } 469 | close(out) 470 | }() 471 | 472 | return out 473 | } 474 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Mozilla Public License Version 2.0 2 | ================================== 3 | 4 | 1. Definitions 5 | -------------- 6 | 7 | 1.1. "Contributor" 8 | means each individual or legal entity that creates, contributes to 9 | the creation of, or owns Covered Software. 10 | 11 | 1.2. "Contributor Version" 12 | means the combination of the Contributions of others (if any) used 13 | by a Contributor and that particular Contributor's Contribution. 14 | 15 | 1.3. "Contribution" 16 | means Covered Software of a particular Contributor. 17 | 18 | 1.4. "Covered Software" 19 | means Source Code Form to which the initial Contributor has attached 20 | the notice in Exhibit A, the Executable Form of such Source Code 21 | Form, and Modifications of such Source Code Form, in each case 22 | including portions thereof. 23 | 24 | 1.5. "Incompatible With Secondary Licenses" 25 | means 26 | 27 | (a) that the initial Contributor has attached the notice described 28 | in Exhibit B to the Covered Software; or 29 | 30 | (b) that the Covered Software was made available under the terms of 31 | version 1.1 or earlier of the License, but not also under the 32 | terms of a Secondary License. 33 | 34 | 1.6. "Executable Form" 35 | means any form of the work other than Source Code Form. 36 | 37 | 1.7. "Larger Work" 38 | means a work that combines Covered Software with other material, in 39 | a separate file or files, that is not Covered Software. 40 | 41 | 1.8. "License" 42 | means this document. 43 | 44 | 1.9. "Licensable" 45 | means having the right to grant, to the maximum extent possible, 46 | whether at the time of the initial grant or subsequently, any and 47 | all of the rights conveyed by this License. 48 | 49 | 1.10. "Modifications" 50 | means any of the following: 51 | 52 | (a) any file in Source Code Form that results from an addition to, 53 | deletion from, or modification of the contents of Covered 54 | Software; or 55 | 56 | (b) any new file in Source Code Form that contains any Covered 57 | Software. 58 | 59 | 1.11. "Patent Claims" of a Contributor 60 | means any patent claim(s), including without limitation, method, 61 | process, and apparatus claims, in any patent Licensable by such 62 | Contributor that would be infringed, but for the grant of the 63 | License, by the making, using, selling, offering for sale, having 64 | made, import, or transfer of either its Contributions or its 65 | Contributor Version. 66 | 67 | 1.12. "Secondary License" 68 | means either the GNU General Public License, Version 2.0, the GNU 69 | Lesser General Public License, Version 2.1, the GNU Affero General 70 | Public License, Version 3.0, or any later versions of those 71 | licenses. 72 | 73 | 1.13. "Source Code Form" 74 | means the form of the work preferred for making modifications. 75 | 76 | 1.14. "You" (or "Your") 77 | means an individual or a legal entity exercising rights under this 78 | License. For legal entities, "You" includes any entity that 79 | controls, is controlled by, or is under common control with You. For 80 | purposes of this definition, "control" means (a) the power, direct 81 | or indirect, to cause the direction or management of such entity, 82 | whether by contract or otherwise, or (b) ownership of more than 83 | fifty percent (50%) of the outstanding shares or beneficial 84 | ownership of such entity. 85 | 86 | 2. License Grants and Conditions 87 | -------------------------------- 88 | 89 | 2.1. Grants 90 | 91 | Each Contributor hereby grants You a world-wide, royalty-free, 92 | non-exclusive license: 93 | 94 | (a) under intellectual property rights (other than patent or trademark) 95 | Licensable by such Contributor to use, reproduce, make available, 96 | modify, display, perform, distribute, and otherwise exploit its 97 | Contributions, either on an unmodified basis, with Modifications, or 98 | as part of a Larger Work; and 99 | 100 | (b) under Patent Claims of such Contributor to make, use, sell, offer 101 | for sale, have made, import, and otherwise transfer either its 102 | Contributions or its Contributor Version. 103 | 104 | 2.2. Effective Date 105 | 106 | The licenses granted in Section 2.1 with respect to any Contribution 107 | become effective for each Contribution on the date the Contributor first 108 | distributes such Contribution. 109 | 110 | 2.3. Limitations on Grant Scope 111 | 112 | The licenses granted in this Section 2 are the only rights granted under 113 | this License. No additional rights or licenses will be implied from the 114 | distribution or licensing of Covered Software under this License. 115 | Notwithstanding Section 2.1(b) above, no patent license is granted by a 116 | Contributor: 117 | 118 | (a) for any code that a Contributor has removed from Covered Software; 119 | or 120 | 121 | (b) for infringements caused by: (i) Your and any other third party's 122 | modifications of Covered Software, or (ii) the combination of its 123 | Contributions with other software (except as part of its Contributor 124 | Version); or 125 | 126 | (c) under Patent Claims infringed by Covered Software in the absence of 127 | its Contributions. 128 | 129 | This License does not grant any rights in the trademarks, service marks, 130 | or logos of any Contributor (except as may be necessary to comply with 131 | the notice requirements in Section 3.4). 132 | 133 | 2.4. Subsequent Licenses 134 | 135 | No Contributor makes additional grants as a result of Your choice to 136 | distribute the Covered Software under a subsequent version of this 137 | License (see Section 10.2) or under the terms of a Secondary License (if 138 | permitted under the terms of Section 3.3). 139 | 140 | 2.5. Representation 141 | 142 | Each Contributor represents that the Contributor believes its 143 | Contributions are its original creation(s) or it has sufficient rights 144 | to grant the rights to its Contributions conveyed by this License. 145 | 146 | 2.6. Fair Use 147 | 148 | This License is not intended to limit any rights You have under 149 | applicable copyright doctrines of fair use, fair dealing, or other 150 | equivalents. 151 | 152 | 2.7. Conditions 153 | 154 | Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted 155 | in Section 2.1. 156 | 157 | 3. Responsibilities 158 | ------------------- 159 | 160 | 3.1. Distribution of Source Form 161 | 162 | All distribution of Covered Software in Source Code Form, including any 163 | Modifications that You create or to which You contribute, must be under 164 | the terms of this License. You must inform recipients that the Source 165 | Code Form of the Covered Software is governed by the terms of this 166 | License, and how they can obtain a copy of this License. You may not 167 | attempt to alter or restrict the recipients' rights in the Source Code 168 | Form. 169 | 170 | 3.2. Distribution of Executable Form 171 | 172 | If You distribute Covered Software in Executable Form then: 173 | 174 | (a) such Covered Software must also be made available in Source Code 175 | Form, as described in Section 3.1, and You must inform recipients of 176 | the Executable Form how they can obtain a copy of such Source Code 177 | Form by reasonable means in a timely manner, at a charge no more 178 | than the cost of distribution to the recipient; and 179 | 180 | (b) You may distribute such Executable Form under the terms of this 181 | License, or sublicense it under different terms, provided that the 182 | license for the Executable Form does not attempt to limit or alter 183 | the recipients' rights in the Source Code Form under this License. 184 | 185 | 3.3. Distribution of a Larger Work 186 | 187 | You may create and distribute a Larger Work under terms of Your choice, 188 | provided that You also comply with the requirements of this License for 189 | the Covered Software. If the Larger Work is a combination of Covered 190 | Software with a work governed by one or more Secondary Licenses, and the 191 | Covered Software is not Incompatible With Secondary Licenses, this 192 | License permits You to additionally distribute such Covered Software 193 | under the terms of such Secondary License(s), so that the recipient of 194 | the Larger Work may, at their option, further distribute the Covered 195 | Software under the terms of either this License or such Secondary 196 | License(s). 197 | 198 | 3.4. Notices 199 | 200 | You may not remove or alter the substance of any license notices 201 | (including copyright notices, patent notices, disclaimers of warranty, 202 | or limitations of liability) contained within the Source Code Form of 203 | the Covered Software, except that You may alter any license notices to 204 | the extent required to remedy known factual inaccuracies. 205 | 206 | 3.5. Application of Additional Terms 207 | 208 | You may choose to offer, and to charge a fee for, warranty, support, 209 | indemnity or liability obligations to one or more recipients of Covered 210 | Software. However, You may do so only on Your own behalf, and not on 211 | behalf of any Contributor. You must make it absolutely clear that any 212 | such warranty, support, indemnity, or liability obligation is offered by 213 | You alone, and You hereby agree to indemnify every Contributor for any 214 | liability incurred by such Contributor as a result of warranty, support, 215 | indemnity or liability terms You offer. You may include additional 216 | disclaimers of warranty and limitations of liability specific to any 217 | jurisdiction. 218 | 219 | 4. Inability to Comply Due to Statute or Regulation 220 | --------------------------------------------------- 221 | 222 | If it is impossible for You to comply with any of the terms of this 223 | License with respect to some or all of the Covered Software due to 224 | statute, judicial order, or regulation then You must: (a) comply with 225 | the terms of this License to the maximum extent possible; and (b) 226 | describe the limitations and the code they affect. Such description must 227 | be placed in a text file included with all distributions of the Covered 228 | Software under this License. Except to the extent prohibited by statute 229 | or regulation, such description must be sufficiently detailed for a 230 | recipient of ordinary skill to be able to understand it. 231 | 232 | 5. Termination 233 | -------------- 234 | 235 | 5.1. The rights granted under this License will terminate automatically 236 | if You fail to comply with any of its terms. However, if You become 237 | compliant, then the rights granted under this License from a particular 238 | Contributor are reinstated (a) provisionally, unless and until such 239 | Contributor explicitly and finally terminates Your grants, and (b) on an 240 | ongoing basis, if such Contributor fails to notify You of the 241 | non-compliance by some reasonable means prior to 60 days after You have 242 | come back into compliance. Moreover, Your grants from a particular 243 | Contributor are reinstated on an ongoing basis if such Contributor 244 | notifies You of the non-compliance by some reasonable means, this is the 245 | first time You have received notice of non-compliance with this License 246 | from such Contributor, and You become compliant prior to 30 days after 247 | Your receipt of the notice. 248 | 249 | 5.2. If You initiate litigation against any entity by asserting a patent 250 | infringement claim (excluding declaratory judgment actions, 251 | counter-claims, and cross-claims) alleging that a Contributor Version 252 | directly or indirectly infringes any patent, then the rights granted to 253 | You by any and all Contributors for the Covered Software under Section 254 | 2.1 of this License shall terminate. 255 | 256 | 5.3. In the event of termination under Sections 5.1 or 5.2 above, all 257 | end user license agreements (excluding distributors and resellers) which 258 | have been validly granted by You or Your distributors under this License 259 | prior to termination shall survive termination. 260 | 261 | ************************************************************************ 262 | * * 263 | * 6. Disclaimer of Warranty * 264 | * ------------------------- * 265 | * * 266 | * Covered Software is provided under this License on an "as is" * 267 | * basis, without warranty of any kind, either expressed, implied, or * 268 | * statutory, including, without limitation, warranties that the * 269 | * Covered Software is free of defects, merchantable, fit for a * 270 | * particular purpose or non-infringing. The entire risk as to the * 271 | * quality and performance of the Covered Software is with You. * 272 | * Should any Covered Software prove defective in any respect, You * 273 | * (not any Contributor) assume the cost of any necessary servicing, * 274 | * repair, or correction. This disclaimer of warranty constitutes an * 275 | * essential part of this License. No use of any Covered Software is * 276 | * authorized under this License except under this disclaimer. * 277 | * * 278 | ************************************************************************ 279 | 280 | ************************************************************************ 281 | * * 282 | * 7. Limitation of Liability * 283 | * -------------------------- * 284 | * * 285 | * Under no circumstances and under no legal theory, whether tort * 286 | * (including negligence), contract, or otherwise, shall any * 287 | * Contributor, or anyone who distributes Covered Software as * 288 | * permitted above, be liable to You for any direct, indirect, * 289 | * special, incidental, or consequential damages of any character * 290 | * including, without limitation, damages for lost profits, loss of * 291 | * goodwill, work stoppage, computer failure or malfunction, or any * 292 | * and all other commercial damages or losses, even if such party * 293 | * shall have been informed of the possibility of such damages. This * 294 | * limitation of liability shall not apply to liability for death or * 295 | * personal injury resulting from such party's negligence to the * 296 | * extent applicable law prohibits such limitation. Some * 297 | * jurisdictions do not allow the exclusion or limitation of * 298 | * incidental or consequential damages, so this exclusion and * 299 | * limitation may not apply to You. * 300 | * * 301 | ************************************************************************ 302 | 303 | 8. Litigation 304 | ------------- 305 | 306 | Any litigation relating to this License may be brought only in the 307 | courts of a jurisdiction where the defendant maintains its principal 308 | place of business and such litigation shall be governed by laws of that 309 | jurisdiction, without reference to its conflict-of-law provisions. 310 | Nothing in this Section shall prevent a party's ability to bring 311 | cross-claims or counter-claims. 312 | 313 | 9. Miscellaneous 314 | ---------------- 315 | 316 | This License represents the complete agreement concerning the subject 317 | matter hereof. If any provision of this License is held to be 318 | unenforceable, such provision shall be reformed only to the extent 319 | necessary to make it enforceable. Any law or regulation which provides 320 | that the language of a contract shall be construed against the drafter 321 | shall not be used to construe this License against a Contributor. 322 | 323 | 10. Versions of the License 324 | --------------------------- 325 | 326 | 10.1. New Versions 327 | 328 | Mozilla Foundation is the license steward. Except as provided in Section 329 | 10.3, no one other than the license steward has the right to modify or 330 | publish new versions of this License. Each version will be given a 331 | distinguishing version number. 332 | 333 | 10.2. Effect of New Versions 334 | 335 | You may distribute the Covered Software under the terms of the version 336 | of the License under which You originally received the Covered Software, 337 | or under the terms of any subsequent version published by the license 338 | steward. 339 | 340 | 10.3. Modified Versions 341 | 342 | If you create software not governed by this License, and you want to 343 | create a new license for such software, you may create and use a 344 | modified version of this License if you rename the license and remove 345 | any references to the name of the license steward (except to note that 346 | such modified license differs from this License). 347 | 348 | 10.4. Distributing Source Code Form that is Incompatible With Secondary 349 | Licenses 350 | 351 | If You choose to distribute Source Code Form that is Incompatible With 352 | Secondary Licenses under the terms of this version of the License, the 353 | notice described in Exhibit B of this License must be attached. 354 | 355 | Exhibit A - Source Code Form License Notice 356 | ------------------------------------------- 357 | 358 | This Source Code Form is subject to the terms of the Mozilla Public 359 | License, v. 2.0. If a copy of the MPL was not distributed with this 360 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 361 | 362 | If it is not possible or desirable to put the notice in a particular 363 | file, then You may include the notice in a location (such as a LICENSE 364 | file in a relevant directory) where a recipient would be likely to look 365 | for such a notice. 366 | 367 | You may add additional accurate notices of copyright ownership. 368 | 369 | Exhibit B - "Incompatible With Secondary Licenses" Notice 370 | --------------------------------------------------------- 371 | 372 | This Source Code Form is "Incompatible With Secondary Licenses", as 373 | defined by the Mozilla Public License, v. 2.0. 374 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 3 | github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= 4 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 5 | github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 6 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 7 | github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 8 | github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6 h1:G1bPvciwNyF7IUmKXNt9Ak3m6u9DE1rF+RmtIkBpVdA= 9 | github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= 10 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= 11 | github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= 12 | github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= 13 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 14 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 15 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= 16 | github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= 17 | github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= 18 | github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= 19 | github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 20 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 21 | github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= 22 | github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= 23 | github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= 24 | github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= 25 | github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= 26 | github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= 27 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 28 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 29 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 30 | github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= 31 | github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= 32 | github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 33 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= 34 | github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0= 35 | github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= 36 | github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A= 37 | github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= 38 | github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpmbhCOZJ293Lz68O7PYrF2EzeiFMwCLk= 39 | github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= 40 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 41 | github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= 42 | github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0= 43 | github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 44 | github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 45 | github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= 46 | github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA= 47 | github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= 48 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 49 | github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 50 | github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= 51 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= 52 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 53 | github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 54 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 55 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 56 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 57 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 58 | github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= 59 | github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= 60 | github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= 61 | github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 62 | github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= 63 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 64 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 65 | github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= 66 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 67 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 68 | github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= 69 | github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= 70 | github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= 71 | github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= 72 | github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= 73 | github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= 74 | github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= 75 | github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= 76 | github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= 77 | github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= 78 | github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 79 | github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= 80 | github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= 81 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 82 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 83 | github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= 84 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 85 | github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= 86 | github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 87 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 88 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 89 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 90 | github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= 91 | github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= 92 | github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= 93 | github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= 94 | github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= 95 | github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= 96 | github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= 97 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 98 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 99 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 100 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 101 | github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= 102 | github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= 103 | github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= 104 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 105 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 106 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 107 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 108 | github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= 109 | github.com/prometheus/client_golang v0.9.3 h1:9iH4JKXLzFbOAdtqv/a+j8aewx2Y8lAjAydhbaScPF8= 110 | github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= 111 | github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= 112 | github.com/prometheus/client_golang v1.5.1 h1:bdHYieyGlH+6OLEk2YQha8THib30KP0/yD0YH9m6xcA= 113 | github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= 114 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 115 | github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 116 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= 117 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 118 | github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= 119 | github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 120 | github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= 121 | github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM= 122 | github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= 123 | github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= 124 | github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= 125 | github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= 126 | github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 127 | github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY= 128 | github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= 129 | github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= 130 | github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= 131 | github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= 132 | github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= 133 | github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= 134 | github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= 135 | github.com/rs/zerolog v1.18.0 h1:CbAm3kP2Tptby1i9sYy2MGRg0uxIN9cyDb59Ys7W8z8= 136 | github.com/rs/zerolog v1.18.0/go.mod h1:9nvC1axdVrAHcu/s9taAVfBuIdTZLVQmKQyvrUjF5+I= 137 | github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 138 | github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= 139 | github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 140 | github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= 141 | github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= 142 | github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= 143 | github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= 144 | github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= 145 | github.com/spf13/cobra v0.0.6 h1:breEStsVwemnKh2/s6gMvSdMEkwW0sK8vGStnlVBMCs= 146 | github.com/spf13/cobra v0.0.6/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= 147 | github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= 148 | github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= 149 | github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 150 | github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= 151 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 152 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 153 | github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= 154 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 155 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 156 | github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= 157 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 158 | github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c h1:g+WoO5jjkqGAzHWCjJB1zZfXPIAaDpzXIEJ0eS6B5Ok= 159 | github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c/go.mod h1:ahpPrc7HpcfEWDQRZEmnXMzHY03mLDYMCxeDzy46i+8= 160 | github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= 161 | github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= 162 | github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= 163 | github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= 164 | github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= 165 | go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= 166 | go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= 167 | go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= 168 | go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= 169 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 170 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= 171 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 172 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 173 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 174 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 175 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 176 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 177 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 178 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 179 | golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 180 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 181 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 182 | golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco= 183 | golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= 184 | golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 185 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= 186 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 187 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 188 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 189 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 190 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 191 | golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= 192 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 193 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 194 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 195 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 196 | golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 197 | golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 198 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 199 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 200 | golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 201 | golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8= 202 | golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 203 | golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= 204 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 205 | golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 206 | golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 207 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 208 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 209 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 210 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 211 | golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 212 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 213 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= 214 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 215 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 216 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 217 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 218 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= 219 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 220 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 221 | google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= 222 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 223 | google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk= 224 | google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 225 | gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= 226 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 227 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 228 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= 229 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 230 | gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= 231 | gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= 232 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 233 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 234 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 235 | gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 236 | gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= 237 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 238 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 239 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 240 | --------------------------------------------------------------------------------