├── doc ├── kglb.png └── kglb_open_arch.jpg ├── proto └── dropbox │ └── proto │ └── kglb │ ├── go.mod │ ├── state.proto │ ├── config.proto │ ├── common.proto │ └── healthchecker │ └── healthchecker.proto ├── kglb ├── common │ ├── const.go │ ├── all_test.go │ ├── utils.go │ ├── utils_test.go │ ├── data_types_test.go │ ├── pretty.go │ ├── pretty_test.go │ └── data_types.go ├── utils │ ├── fwmark │ │ ├── all_test.go │ │ ├── fwmark_helpers.go │ │ ├── fwmark_gen.go │ │ ├── fwmark_gen_test.go │ │ ├── fwmark_helpers_test.go │ │ ├── fwmark_manager_test.go │ │ ├── fwmark_manager.go │ │ └── fwmark_conn.go │ ├── comparable │ │ ├── all_test.go │ │ ├── arrays_test.go │ │ └── arrays.go │ ├── concurrency │ │ ├── all_test.go │ │ ├── concurrency.go │ │ └── concurrency_test.go │ ├── config_loader │ │ ├── all_test.go │ │ ├── config_loader.go │ │ ├── one_time_file_loader.go │ │ └── one_time_file_loader_test.go │ ├── health_manager │ │ ├── all_test.go │ │ ├── stats.go │ │ ├── health_status.go │ │ ├── health_manager_state.go │ │ ├── health_status_test.go │ │ └── health_manager_state_test.go │ ├── discovery │ │ ├── all_test.go │ │ ├── stats.go │ │ ├── discovery.go │ │ ├── host_ports.go │ │ ├── host_ports_test.go │ │ ├── static_resolver.go │ │ └── static_resolver_test.go │ ├── dns_resolver │ │ ├── all_test.go │ │ ├── dns_resolver_mock.go │ │ ├── system_resolver.go │ │ └── dns_resolver.go │ └── health_checker │ │ ├── dummy_checker_test.go │ │ ├── dns_handler.go │ │ ├── dummy_checker.go │ │ ├── tcp_checker.go │ │ ├── all_test.go │ │ ├── health_checker.go │ │ ├── dns_checker.go │ │ ├── syslog_checker.go │ │ ├── syslog_checker_test.go │ │ ├── tcp_checker_test.go │ │ └── dns_checker_test.go ├── data_plane │ ├── utils_test.go │ ├── bgp_module.go │ ├── netlink_address_module.go │ ├── resolver_module.go │ ├── all_test.go │ ├── ipvs_module.go │ ├── utils.go │ ├── config_provider.go │ ├── stats.go │ ├── resolver.go │ ├── manager_balancer_test.go │ ├── ipvs_mqliang_types_test.go │ ├── manager_dynamic_routing_test.go │ ├── netlink_address_impl.go │ ├── manager_balancer.go │ ├── mock_modules.go │ ├── manager_address.go │ └── cache_resolver.go └── control_plane │ ├── data_plane_interface.go │ ├── balancer_state.go │ ├── discovery_factory_test.go │ ├── stats.go │ ├── all_test.go │ ├── discovery_factory.go │ └── health_checker_factory.go ├── dlog └── dlog.go ├── exclog └── exclog_stdout.go ├── .github └── workflows │ ├── lint.yml │ ├── test.yml │ └── build.yml ├── go.mod ├── kglbd ├── bgp_module.go ├── config_loader.go ├── main.go └── service.go ├── vortex2 └── v2stats │ ├── util.go │ ├── gauge_group.go │ └── prometheus.go ├── examples └── example_config.json └── README.md /doc/kglb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dropbox/kglb/HEAD/doc/kglb.png -------------------------------------------------------------------------------- /proto/dropbox/proto/kglb/go.mod: -------------------------------------------------------------------------------- 1 | module dropbox/proto/kglb 2 | 3 | go 1.13 4 | -------------------------------------------------------------------------------- /doc/kglb_open_arch.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dropbox/kglb/HEAD/doc/kglb_open_arch.jpg -------------------------------------------------------------------------------- /kglb/common/const.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | const ( 4 | // default data plane listener port. 5 | DataPlanePort = 7654 6 | LoopbackLinkName = "lo" 7 | ) 8 | -------------------------------------------------------------------------------- /kglb/common/all_test.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "testing" 5 | 6 | . "gopkg.in/check.v1" 7 | ) 8 | 9 | func Test(t *testing.T) { 10 | TestingT(t) 11 | } 12 | -------------------------------------------------------------------------------- /kglb/utils/fwmark/all_test.go: -------------------------------------------------------------------------------- 1 | package fwmark 2 | 3 | import ( 4 | "testing" 5 | 6 | . "gopkg.in/check.v1" 7 | ) 8 | 9 | func Test(t *testing.T) { 10 | TestingT(t) 11 | } 12 | -------------------------------------------------------------------------------- /kglb/utils/comparable/all_test.go: -------------------------------------------------------------------------------- 1 | package comparable 2 | 3 | import ( 4 | "testing" 5 | 6 | . "gopkg.in/check.v1" 7 | ) 8 | 9 | func Test(t *testing.T) { 10 | TestingT(t) 11 | } 12 | -------------------------------------------------------------------------------- /kglb/utils/concurrency/all_test.go: -------------------------------------------------------------------------------- 1 | package concurrency 2 | 3 | import ( 4 | "testing" 5 | 6 | . "gopkg.in/check.v1" 7 | ) 8 | 9 | func Test(t *testing.T) { 10 | TestingT(t) 11 | } 12 | -------------------------------------------------------------------------------- /kglb/utils/config_loader/all_test.go: -------------------------------------------------------------------------------- 1 | package config_loader 2 | 3 | import ( 4 | "testing" 5 | 6 | . "gopkg.in/check.v1" 7 | ) 8 | 9 | func Test(t *testing.T) { 10 | TestingT(t) 11 | } 12 | -------------------------------------------------------------------------------- /kglb/utils/health_manager/all_test.go: -------------------------------------------------------------------------------- 1 | package health_manager 2 | 3 | import ( 4 | "testing" 5 | 6 | . "gopkg.in/check.v1" 7 | ) 8 | 9 | func Test(t *testing.T) { 10 | TestingT(t) 11 | } 12 | -------------------------------------------------------------------------------- /kglb/utils/discovery/all_test.go: -------------------------------------------------------------------------------- 1 | package discovery 2 | 3 | import ( 4 | "testing" 5 | 6 | . "gopkg.in/check.v1" 7 | ) 8 | 9 | // Bootstrap gocheck for this package. 10 | func Test(t *testing.T) { 11 | TestingT(t) 12 | } 13 | -------------------------------------------------------------------------------- /kglb/utils/dns_resolver/all_test.go: -------------------------------------------------------------------------------- 1 | package dns_resolver 2 | 3 | import ( 4 | "testing" 5 | 6 | . "gopkg.in/check.v1" 7 | ) 8 | 9 | // Bootstrap gocheck for this package. 10 | func Test(t *testing.T) { 11 | TestingT(t) 12 | } 13 | -------------------------------------------------------------------------------- /proto/dropbox/proto/kglb/state.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | import "dropbox/proto/kglb/common.proto"; 4 | 5 | package kglb; 6 | 7 | message DataPlaneState { 8 | repeated BalancerState balancers = 1; 9 | repeated DynamicRoute dynamic_routes = 2; 10 | repeated LinkAddress link_addresses = 3; 11 | } 12 | -------------------------------------------------------------------------------- /kglb/utils/fwmark/fwmark_helpers.go: -------------------------------------------------------------------------------- 1 | package fwmark 2 | 3 | import ( 4 | kglb_pb "dropbox/proto/kglb" 5 | ) 6 | 7 | func IsFwmarkService(service *kglb_pb.IpvsService) bool { 8 | switch service.Attributes.(type) { 9 | case *kglb_pb.IpvsService_FwmarkAttributes: 10 | return true 11 | default: 12 | return false 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /kglb/utils/discovery/stats.go: -------------------------------------------------------------------------------- 1 | package discovery 2 | 3 | import ( 4 | "dropbox/vortex2/v2stats" 5 | ) 6 | 7 | // Indicates how many upstreams currently resolved. 8 | // Tags: 9 | // - setup: setup name 10 | // - service: service name 11 | var staticResolverGauge = v2stats.MustDefineGauge("kglb/control_plane/discovery/static_upstream_count", "setup", "service") 12 | -------------------------------------------------------------------------------- /kglb/data_plane/utils_test.go: -------------------------------------------------------------------------------- 1 | package data_plane 2 | 3 | import ( 4 | . "gopkg.in/check.v1" 5 | ) 6 | 7 | type UtilsSuite struct { 8 | } 9 | 10 | var _ = Suite(&UtilsSuite{}) 11 | 12 | func (m *ManagerSuite) TestDiffMetric(c *C) { 13 | c.Assert(int(diffMetric(0, 0)), Equals, 0) 14 | c.Assert(int(diffMetric(10, 10)), Equals, 0) 15 | c.Assert(int(diffMetric(25, 10)), Equals, 15) 16 | c.Assert(int(diffMetric(10, 25)), Equals, 10) 17 | } 18 | -------------------------------------------------------------------------------- /kglb/common/utils.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | pb "dropbox/proto/kglb" 5 | ) 6 | 7 | // Returns alive ratio of set of upstreams. 8 | func AliveUpstreamsRatio(ups []*pb.UpstreamState) float32 { 9 | all := len(ups) 10 | 11 | alive := 0 12 | for _, u := range ups { 13 | if u.Weight != 0 { 14 | alive += 1 15 | } 16 | } 17 | 18 | if alive == 0 { 19 | return 0.0 20 | } 21 | 22 | return float32(alive) / float32(all) 23 | } 24 | -------------------------------------------------------------------------------- /kglb/control_plane/data_plane_interface.go: -------------------------------------------------------------------------------- 1 | package control_plane 2 | 3 | import ( 4 | "time" 5 | 6 | kglb_pb "dropbox/proto/kglb" 7 | ) 8 | 9 | const ( 10 | // how often do we send state to data plane 11 | DefaultSendInterval = 30 * time.Second 12 | 13 | // how long we can wait for data plane 14 | DefaultSendTimeout = 60 * time.Second 15 | ) 16 | 17 | type DataPlaneClient interface { 18 | Set(state *kglb_pb.DataPlaneState) error 19 | } 20 | -------------------------------------------------------------------------------- /kglb/utils/discovery/discovery.go: -------------------------------------------------------------------------------- 1 | package discovery 2 | 3 | type DiscoveryResolver interface { 4 | // Discovery Resolver id. 5 | GetId() string 6 | 7 | // Get current state of the resolver. 8 | GetState() DiscoveryState 9 | 10 | // Get changes of the resolver through channel. 11 | Updates() <-chan DiscoveryState 12 | 13 | // Close/Stop resolver. 14 | Close() 15 | 16 | // Check if the item discovers exactly the same things. 17 | Equal(item DiscoveryResolver) bool 18 | } 19 | -------------------------------------------------------------------------------- /kglb/utils/fwmark/fwmark_gen.go: -------------------------------------------------------------------------------- 1 | package fwmark 2 | 3 | import ( 4 | "fmt" 5 | 6 | "godropbox/murmur3" 7 | ) 8 | 9 | const ( 10 | fwMarkSeed = 18410482 11 | ) 12 | 13 | type FwmarkParams struct { 14 | Hostname string 15 | IP string 16 | Port uint32 17 | } 18 | 19 | func (p FwmarkParams) String() string { 20 | return fmt.Sprintf("%s:%s:%d", p.Hostname, p.IP, p.Port) 21 | } 22 | 23 | func GetFwmark(params FwmarkParams) uint32 { 24 | return murmur3.Hash32([]byte(params.String()), fwMarkSeed) 25 | } 26 | -------------------------------------------------------------------------------- /kglb/utils/health_manager/stats.go: -------------------------------------------------------------------------------- 1 | package health_manager 2 | 3 | import ( 4 | "dropbox/vortex2/v2stats" 5 | ) 6 | 7 | // KGLB healthcheck counter. 8 | // Tags: 9 | // - setup: setup name 10 | // - service: service name 11 | // - host: actual host name being health checked 12 | // - result: pass/fail 13 | var healthCheckCounter = v2stats.MustDefineCounter("kglb/control_plane/healthcheck", "setup", "service", "host", "result") 14 | 15 | var aliveRatioGauge = v2stats.MustDefineGauge("kglb/control_plane/alive_ratio", "setup", "service") 16 | -------------------------------------------------------------------------------- /kglb/data_plane/bgp_module.go: -------------------------------------------------------------------------------- 1 | package data_plane 2 | 3 | import kglb_pb "dropbox/proto/kglb" 4 | 5 | // Bgp module. 6 | type BgpModule interface { 7 | // init speaker with given ASN 8 | Init(asn uint32) error 9 | // advertise path. 10 | Advertise(config *kglb_pb.BgpRouteAttributes) error 11 | // withdraw path. 12 | Withdraw(config *kglb_pb.BgpRouteAttributes) error 13 | // get list of current bgp paths 14 | ListPaths() ([]*kglb_pb.BgpRouteAttributes, error) 15 | // get BGP session state 16 | IsSessionEstablished() (bool, error) 17 | } 18 | -------------------------------------------------------------------------------- /kglb/control_plane/balancer_state.go: -------------------------------------------------------------------------------- 1 | package control_plane 2 | 3 | import ( 4 | pb "dropbox/proto/kglb" 5 | ) 6 | 7 | // State generated by Balancer to simlify access to required information since 8 | // it provides set of pb.BalancerState's. 9 | type BalancerState struct { 10 | // set of main BalancerState and fwmark states if it's enabled. 11 | States []*pb.BalancerState 12 | // rate of healthy upstreams. 13 | AliveRatio float32 14 | // the flag is false when Alive ratio of Balancer was positive, otherwise 15 | // false. 16 | InitialState bool 17 | } 18 | -------------------------------------------------------------------------------- /kglb/data_plane/netlink_address_module.go: -------------------------------------------------------------------------------- 1 | package data_plane 2 | 3 | import ( 4 | "net" 5 | ) 6 | 7 | // Address list manipulations through netlink. 8 | type AddressTableModule interface { 9 | // Add address to the interface. 10 | Add(addr net.IP, iface string) error 11 | // Remove address from the interface. 12 | Delete(addr net.IP, iface string) error 13 | // Check if the link had specific address. 14 | IsExists(addr net.IP, iface string) (bool, error) 15 | // List of all configured addresses for specific interface. 16 | List(iface string) ([]net.IP, error) 17 | } 18 | -------------------------------------------------------------------------------- /kglb/common/utils_test.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | . "gopkg.in/check.v1" 5 | 6 | pb "dropbox/proto/kglb" 7 | ) 8 | 9 | type UtilsSuite struct{} 10 | 11 | var _ = Suite(&UtilsSuite{}) 12 | 13 | func (s *UtilsSuite) TestAliveUpstreamsRatio(c *C) { 14 | c.Assert(AliveUpstreamsRatio([]*pb.UpstreamState{ 15 | {Weight: 100}, {Weight: 100}, 16 | }), Equals, float32(1)) 17 | c.Assert(AliveUpstreamsRatio([]*pb.UpstreamState{ 18 | {Weight: 100}, {Weight: 0}, 19 | }), Equals, float32(0.5)) 20 | c.Assert(AliveUpstreamsRatio([]*pb.UpstreamState{ 21 | {Weight: 0}, {Weight: 0}, 22 | }), Equals, float32(0)) 23 | } 24 | -------------------------------------------------------------------------------- /dlog/dlog.go: -------------------------------------------------------------------------------- 1 | package dlog 2 | 3 | import ( 4 | "github.com/golang/glog" 5 | ) 6 | 7 | type Level = glog.Level 8 | 9 | var ( 10 | V = glog.V 11 | Stats = glog.Stats 12 | Flush = glog.Flush 13 | 14 | Info = glog.Info 15 | Infoln = glog.Infoln 16 | Infof = glog.Infof 17 | Warning = glog.Warning 18 | Warningln = glog.Warningln 19 | Warningf = glog.Warningf 20 | Error = glog.Error 21 | Errorln = glog.Errorln 22 | Errorf = glog.Errorf 23 | Fatal = glog.Fatal 24 | Fatalln = glog.Fatalln 25 | Fatalf = glog.Fatalf 26 | ) 27 | -------------------------------------------------------------------------------- /kglb/data_plane/resolver_module.go: -------------------------------------------------------------------------------- 1 | package data_plane 2 | 3 | import ( 4 | "net" 5 | 6 | kglb_pb "dropbox/proto/kglb" 7 | ) 8 | 9 | type ResolverModule interface { 10 | // lookup (hostname resolution). 11 | Lookup(name string) (*HostnameCacheEntry, error) 12 | // reverse lookup (ip -> hostname resolution). 13 | ReverseLookup(ip net.IP) (string, error) 14 | // Get service name based on information available in IpvsService message. 15 | ServiceLookup(srv *kglb_pb.IpvsService) string 16 | // cluster name by hostname. 17 | Cluster(hostname string) string 18 | } 19 | 20 | type HostnameCacheEntry struct { 21 | IPv4 net.IP 22 | IPv6 net.IP 23 | } 24 | -------------------------------------------------------------------------------- /kglb/utils/fwmark/fwmark_gen_test.go: -------------------------------------------------------------------------------- 1 | package fwmark 2 | 3 | import ( 4 | . "gopkg.in/check.v1" 5 | 6 | . "godropbox/gocheck2" 7 | ) 8 | 9 | type FwmarkManagerSuite struct{} 10 | 11 | var _ = Suite(&FwmarkManagerSuite{}) 12 | 13 | func (s *FwmarkManagerSuite) TestGet(c *C) { 14 | p1 := FwmarkParams{ 15 | Hostname: "test-hostname-1", 16 | IP: "127.0.0.1", 17 | Port: 80, 18 | } 19 | 20 | r1 := GetFwmark(p1) 21 | r2 := GetFwmark(p1) 22 | c.Assert(r1, Equals, r2) 23 | 24 | p2 := FwmarkParams{ 25 | Hostname: "test-hostname-2", 26 | IP: "127.0.0.1", 27 | Port: 80, 28 | } 29 | 30 | r3 := GetFwmark(p2) 31 | c.Assert(r3 == r1, IsFalse) 32 | } 33 | -------------------------------------------------------------------------------- /kglb/utils/health_checker/dummy_checker_test.go: -------------------------------------------------------------------------------- 1 | package health_checker 2 | 3 | import ( 4 | . "gopkg.in/check.v1" 5 | 6 | hc_pb "dropbox/proto/kglb/healthchecker" 7 | . "godropbox/gocheck2" 8 | ) 9 | 10 | type DummyCheckerSuite struct { 11 | } 12 | 13 | var _ = Suite(&DummyCheckerSuite{}) 14 | 15 | func (m *DummyCheckerSuite) TestDummyChecker(c *C) { 16 | checker, err := NewDummyChecker(&hc_pb.DummyCheckerAttributes{}) 17 | c.Assert(err, IsNil) 18 | c.Assert(checker.Check("localhost", 0), IsNil) 19 | 20 | c.Assert( 21 | checker.GetConfiguration(), 22 | DeepEqualsPretty, 23 | &hc_pb.HealthCheckerAttributes{ 24 | Attributes: &hc_pb.HealthCheckerAttributes_Dummy{ 25 | Dummy: &hc_pb.DummyCheckerAttributes{}, 26 | }}) 27 | } 28 | -------------------------------------------------------------------------------- /kglb/data_plane/all_test.go: -------------------------------------------------------------------------------- 1 | package data_plane 2 | 3 | import ( 4 | "testing" 5 | 6 | . "gopkg.in/check.v1" 7 | ) 8 | 9 | func Test(t *testing.T) { 10 | TestingT(t) 11 | } 12 | 13 | func GetMockModules(modules *ManagerModules) (*ManagerModules, error) { 14 | 15 | if modules == nil { 16 | modules = &ManagerModules{} 17 | } 18 | if modules.AddressTable == nil { 19 | modules.AddressTable = NewMockAddressTableWithState() 20 | } 21 | if modules.Ipvs == nil { 22 | modules.Ipvs = NewMockIpvsModuleWithState() 23 | } 24 | if modules.Resolver == nil { 25 | resolver, err := NewCacheResolver() 26 | if err != nil { 27 | return nil, err 28 | } 29 | modules.Resolver = resolver 30 | } 31 | if modules.Bgp == nil { 32 | modules.Bgp = NewMockBgpModuleWithState() 33 | } 34 | 35 | return modules, nil 36 | } 37 | -------------------------------------------------------------------------------- /exclog/exclog_stdout.go: -------------------------------------------------------------------------------- 1 | package exclog 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | ) 7 | 8 | // Severity level for reported exceptions 9 | type Severity int 10 | 11 | const ( 12 | Uncaught Severity = 20 13 | Critical Severity = 40 14 | Operational Severity = 50 15 | Noncritical Severity = 60 16 | UserError Severity = 80 17 | ) 18 | 19 | func Report(err error, severity Severity, id string) { 20 | fmt.Errorf("e: %v, severity: %v, id: %v", err, severity, id) 21 | } 22 | 23 | func PanicAndReport(err error) { 24 | panic(fmt.Sprintf("exclog: err: %s", err)) 25 | } 26 | func PanicAndReportf(format string, args ...interface{}) { 27 | panic(fmt.Sprintf(format, args...)) 28 | } 29 | func ReportAndCrash(err error) { 30 | panic(fmt.Sprintf("exclog: ReportAndCrash(): %v", err)) 31 | } 32 | func Flush(wait time.Duration) { 33 | } 34 | -------------------------------------------------------------------------------- /kglb/utils/health_checker/dns_handler.go: -------------------------------------------------------------------------------- 1 | package health_checker 2 | 3 | import ( 4 | "github.com/miekg/dns" 5 | ) 6 | 7 | // Simple dns server impl based on dns lib with custom handler to use in tests. 8 | type DnsHandler struct { 9 | Handler func(w dns.ResponseWriter, r *dns.Msg) 10 | 11 | server *dns.Server 12 | } 13 | 14 | func (d *DnsHandler) ServeDNS(w dns.ResponseWriter, r *dns.Msg) { 15 | d.Handler(w, r) 16 | } 17 | 18 | func (d *DnsHandler) ListenAndServe(network, addr string) { 19 | d.server = &dns.Server{Addr: addr, Net: network, Handler: d} 20 | go func() { 21 | _ = d.server.ListenAndServe() 22 | }() 23 | 24 | return 25 | } 26 | 27 | // Shutdowns dns server and closes listener. Should be called after 28 | // ListenAndServe() only. 29 | func (d *DnsHandler) Close() { 30 | d.server.Shutdown() 31 | } 32 | 33 | var _ dns.Handler = &DnsHandler{} 34 | -------------------------------------------------------------------------------- /kglb/utils/health_checker/dummy_checker.go: -------------------------------------------------------------------------------- 1 | package health_checker 2 | 3 | import ( 4 | hc_pb "dropbox/proto/kglb/healthchecker" 5 | ) 6 | 7 | // Dummy health checker implementation which always returns true from Check(). 8 | type DummyChecker struct { 9 | params *hc_pb.DummyCheckerAttributes 10 | } 11 | 12 | func NewDummyChecker(params *hc_pb.DummyCheckerAttributes) (*DummyChecker, error) { 13 | return &DummyChecker{params: params}, nil 14 | } 15 | 16 | // Performs test and returns true when test was succeed. 17 | func (d *DummyChecker) Check(host string, port int) error { 18 | return nil 19 | } 20 | 21 | func (d *DummyChecker) GetConfiguration() *hc_pb.HealthCheckerAttributes { 22 | return &hc_pb.HealthCheckerAttributes{ 23 | Attributes: &hc_pb.HealthCheckerAttributes_Dummy{ 24 | Dummy: d.params, 25 | }, 26 | } 27 | } 28 | 29 | var _ HealthChecker = &DummyChecker{} 30 | -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: Lint 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | 11 | lint: 12 | name: Lint 13 | runs-on: ubuntu-latest 14 | steps: 15 | 16 | - name: Set up Go 1.13 17 | uses: actions/setup-go@v1 18 | with: 19 | go-version: 1.13 20 | id: go 21 | 22 | - name: Check out code into the Go module directory 23 | uses: actions/checkout@v2 24 | 25 | - name: Get dependencies 26 | run: | 27 | go get -v -t -d ./... 28 | if [ -f Gopkg.toml ]; then 29 | curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh 30 | dep ensure 31 | fi 32 | 33 | - name: GolangCI-Linter 34 | uses: Mushus/golangci-linter@v1.1.2 35 | with: 36 | # Path of the Golang project root directory 37 | basePath: . 38 | -------------------------------------------------------------------------------- /kglb/utils/dns_resolver/dns_resolver_mock.go: -------------------------------------------------------------------------------- 1 | package dns_resolver 2 | 3 | import ( 4 | pb "dropbox/proto/kglb" 5 | "godropbox/errors" 6 | ) 7 | 8 | type dnsResolverMock struct { 9 | data map[string]*pb.IP 10 | } 11 | 12 | func NewDnsResolverMock(data map[string]*pb.IP) *dnsResolverMock { 13 | return &dnsResolverMock{data: data} 14 | } 15 | 16 | func (r *dnsResolverMock) ResolveHost(hostname string, af pb.AddressFamily) (*pb.IP, error) { 17 | addr, ok := r.data[hostname] 18 | if !ok { 19 | return nil, errors.Newf("Host %s not found", hostname) 20 | } 21 | return addr, nil 22 | } 23 | 24 | var _ DnsResolver = &dnsResolverMock{} 25 | 26 | // Helper to mock DnsResolver interface with custom impl of ResolveHost func. 27 | type ResolverMock struct { 28 | ResolverFunc func(string, pb.AddressFamily) (*pb.IP, error) 29 | } 30 | 31 | func (r *ResolverMock) ResolveHost( 32 | hostname string, 33 | af pb.AddressFamily) (*pb.IP, error) { 34 | 35 | return r.ResolverFunc(hostname, af) 36 | } 37 | 38 | var _ DnsResolver = &ResolverMock{} 39 | -------------------------------------------------------------------------------- /kglb/utils/fwmark/fwmark_helpers_test.go: -------------------------------------------------------------------------------- 1 | package fwmark 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | 8 | kglb_pb "dropbox/proto/kglb" 9 | ) 10 | 11 | var ( 12 | service1 = &kglb_pb.IpvsService{ 13 | Attributes: &kglb_pb.IpvsService_TcpAttributes{ 14 | TcpAttributes: &kglb_pb.IpvsTcpAttributes{ 15 | Address: &kglb_pb.IP{Address: &kglb_pb.IP_Ipv4{Ipv4: "4.3.2.1"}}, 16 | Port: 443, 17 | }, 18 | }, 19 | Scheduler: kglb_pb.IpvsService_RR, 20 | } 21 | 22 | service2 = &kglb_pb.IpvsService{ 23 | Attributes: &kglb_pb.IpvsService_FwmarkAttributes{ 24 | FwmarkAttributes: &kglb_pb.IpvsFwmarkAttributes{ 25 | AddressFamily: kglb_pb.AddressFamily_AF_INET, 26 | Fwmark: 10, 27 | }, 28 | }, 29 | Scheduler: kglb_pb.IpvsService_RR, 30 | Flags: []kglb_pb.IpvsService_Flag{ 31 | kglb_pb.IpvsService_ONEPACKET, 32 | }, 33 | } 34 | ) 35 | 36 | func TestIsFwmarkService(t *testing.T) { 37 | require.False(t, IsFwmarkService(service1)) 38 | require.True(t, IsFwmarkService(service2)) 39 | } 40 | -------------------------------------------------------------------------------- /kglb/data_plane/ipvs_module.go: -------------------------------------------------------------------------------- 1 | package data_plane 2 | 3 | import kglb_pb "dropbox/proto/kglb" 4 | 5 | type IpvsModule interface { 6 | // Add ipvs service with specific VIP:port, proto ("tcp"|"udp") and 7 | // scheduler. 8 | AddService(service *kglb_pb.IpvsService) error 9 | // Delete ipvs service. 10 | DeleteService(service *kglb_pb.IpvsService) error 11 | // Get list of existent ipvs Services. 12 | ListServices() ([]*kglb_pb.IpvsService, []*kglb_pb.Stats, error) 13 | // Get list of destinations of specific ipvs service 14 | GetRealServers(service *kglb_pb.IpvsService) ([]*kglb_pb.UpstreamState, []*kglb_pb.Stats, error) 15 | 16 | // Add destinations to the specific ipvs service. 17 | AddRealServers(service *kglb_pb.IpvsService, dsts []*kglb_pb.UpstreamState) error 18 | // Delete destination from specific ipvs service. 19 | DeleteRealServers(service *kglb_pb.IpvsService, dsts []*kglb_pb.UpstreamState) error 20 | // Update destination for specific ipvs service. 21 | UpdateRealServers(service *kglb_pb.IpvsService, dsts []*kglb_pb.UpstreamState) error 22 | } 23 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module dropbox 2 | 3 | go 1.13 4 | 5 | require ( 6 | dropbox/proto/kglb v0.0.0-00010101000000-000000000000 7 | github.com/dropbox/godropbox v0.0.0-20200221053928-caf2e8d91700 // indirect 8 | github.com/gogo/protobuf v1.3.1 9 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b 10 | github.com/golang/protobuf v1.3.5 11 | github.com/hkwi/nlgo v0.0.0-20190926025335-08733afbfe04 // indirect 12 | github.com/miekg/dns v1.1.27 13 | github.com/mqliang/libipvs v0.0.0-20181031074626-20f197c976a3 14 | github.com/prometheus/client_golang v1.3.0 15 | github.com/stretchr/testify v1.3.0 16 | github.com/vishvananda/netlink v1.0.0 17 | github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df // indirect 18 | godropbox v0.0.0-00010101000000-000000000000 19 | golang.org/x/net v0.0.0-20200202094626-16171245cfb2 20 | golang.org/x/sys v0.0.0-20191220142924-d4481acd189f 21 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 22 | ) 23 | 24 | replace ( 25 | dropbox/proto/kglb => ./proto/dropbox/proto/kglb 26 | godropbox => github.com/dropbox/godropbox v0.0.0-20200228041828-52ad444d3502 27 | ) 28 | -------------------------------------------------------------------------------- /kglb/utils/config_loader/config_loader.go: -------------------------------------------------------------------------------- 1 | package config_loader 2 | 3 | type ConfigProvider interface { 4 | // The default configuration when the config file (or the locally cached 5 | // config file) is absent. The default configuration must be non-nil and 6 | // valid (i.e., Validate(cfg) returns nil). 7 | Default() interface{} 8 | 9 | // Given a config file content, return the parsed config object (or error). 10 | // The parsed config object must be non-nil. 11 | Parse(content []byte) (cfg interface{}, err error) 12 | 13 | // Validate ensures the config is semantically correct. If the config is 14 | // invalid, the config loader will simply ignore the config. 15 | Validate(cfg interface{}) error 16 | 17 | // The config loader will only sent updates when the config file's 18 | // content has modified. 19 | Equals(cfg1 interface{}, cfg2 interface{}) bool 20 | } 21 | 22 | // Basic interface for config loader. 23 | type ConfigLoader interface { 24 | // Config updates channel. 25 | Updates() <-chan interface{} 26 | // Stopping config loader and closing its update channel. 27 | Stop() 28 | } 29 | -------------------------------------------------------------------------------- /kglbd/bgp_module.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "sync" 5 | 6 | "dropbox/kglb/common" 7 | kglb_pb "dropbox/proto/kglb" 8 | ) 9 | 10 | // Mock Bgp module. 11 | type NoOpBgpModule struct { 12 | mu sync.Mutex 13 | state []*kglb_pb.BgpRouteAttributes 14 | } 15 | 16 | func (b *NoOpBgpModule) Init(asn uint32) error { 17 | return nil 18 | } 19 | 20 | func (b *NoOpBgpModule) Advertise(config *kglb_pb.BgpRouteAttributes) error { 21 | b.mu.Lock() 22 | b.mu.Unlock() 23 | 24 | b.state = append(b.state, config) 25 | return nil 26 | } 27 | 28 | func (b *NoOpBgpModule) Withdraw(config *kglb_pb.BgpRouteAttributes) error { 29 | b.mu.Lock() 30 | b.mu.Unlock() 31 | 32 | for i, cfg := range b.state { 33 | if common.BgpRoutingAttributesComparable.Equal(cfg, config) { 34 | b.state = append(b.state[:i], b.state[i+1:]...) 35 | break 36 | } 37 | } 38 | return nil 39 | } 40 | 41 | func (b *NoOpBgpModule) ListPaths() ([]*kglb_pb.BgpRouteAttributes, error) { 42 | b.mu.Lock() 43 | b.mu.Unlock() 44 | 45 | return b.state, nil 46 | } 47 | 48 | func (b *NoOpBgpModule) IsSessionEstablished() (bool, error) { 49 | return true, nil 50 | } 51 | -------------------------------------------------------------------------------- /kglbd/config_loader.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/gogo/protobuf/proto" 5 | "github.com/golang/protobuf/jsonpb" 6 | 7 | "dropbox/kglb/common" 8 | "dropbox/kglb/utils/config_loader" 9 | pb "dropbox/proto/kglb" 10 | ) 11 | 12 | type ConfigProvider struct{} 13 | 14 | func (c *ConfigProvider) Default() interface{} { 15 | return &pb.ControlPlaneConfig{} 16 | } 17 | 18 | func (c *ConfigProvider) Parse(content []byte) (interface{}, error) { 19 | newConfig := &pb.ControlPlaneConfig{} 20 | if err := jsonpb.UnmarshalString(string(content), newConfig); err != nil { 21 | return nil, err 22 | } 23 | return newConfig, nil 24 | } 25 | 26 | func (c *ConfigProvider) Validate(cfg interface{}) error { 27 | config := cfg.(*pb.ControlPlaneConfig) 28 | return common.ValidateControlPlaneConfig(config) 29 | } 30 | 31 | func (c *ConfigProvider) Equals(cfg1 interface{}, cfg2 interface{}) bool { 32 | return proto.Equal(cfg1.(*pb.ControlPlaneConfig), cfg2.(*pb.ControlPlaneConfig)) 33 | } 34 | 35 | func MakeConfigLoader(configPath string) (config_loader.ConfigLoader, error) { 36 | return config_loader.NewOneTimeFileLoader(&ConfigProvider{}, configPath) 37 | } 38 | -------------------------------------------------------------------------------- /kglb/utils/config_loader/one_time_file_loader.go: -------------------------------------------------------------------------------- 1 | package config_loader 2 | 3 | import ( 4 | "io/ioutil" 5 | 6 | "godropbox/errors" 7 | ) 8 | 9 | type OneTimeFileLoader struct { 10 | path string 11 | 12 | updateChan chan interface{} 13 | } 14 | 15 | var _ ConfigLoader = &OneTimeFileLoader{} 16 | 17 | func NewOneTimeFileLoader(provider ConfigProvider, path string) (*OneTimeFileLoader, error) { 18 | loader := &OneTimeFileLoader{ 19 | path: path, 20 | updateChan: make(chan interface{}, 1), 21 | } 22 | 23 | if content, err := ioutil.ReadFile(path); err != nil { 24 | return nil, errors.Wrapf(err, "fails to read '%s' file: ", path) 25 | } else { 26 | // validate config. 27 | cfg, err := provider.Parse(content) 28 | if err != nil { 29 | return nil, err 30 | } 31 | if err = provider.Validate(cfg); err != nil { 32 | return nil, err 33 | } 34 | loader.updateChan <- cfg 35 | } 36 | 37 | return loader, nil 38 | } 39 | 40 | // Config updates channel. 41 | func (l *OneTimeFileLoader) Updates() <-chan interface{} { 42 | return l.updateChan 43 | } 44 | 45 | // Stopping config loader and closing its update channel. 46 | func (l *OneTimeFileLoader) Stop() { 47 | close(l.updateChan) 48 | } 49 | -------------------------------------------------------------------------------- /vortex2/v2stats/util.go: -------------------------------------------------------------------------------- 1 | package v2stats 2 | 3 | import ( 4 | "sort" 5 | ) 6 | 7 | type byKey struct { 8 | keys []string 9 | values []string 10 | } 11 | 12 | func (a byKey) Len() int { 13 | return len(a.keys) 14 | } 15 | func (a byKey) Swap(i, j int) { 16 | a.keys[i], a.keys[j] = a.keys[j], a.keys[i] 17 | a.values[i], a.values[j] = a.values[j], a.values[i] 18 | } 19 | func (a byKey) Less(i, j int) bool { 20 | return a.keys[i] < a.keys[j] 21 | } 22 | 23 | // Metric tags are key=value pairs. 24 | type KV map[string]string 25 | 26 | func (kv KV) WithTag(tagName, tagValue string) KV { 27 | kvCopy := KV{} 28 | for k, v := range kv { 29 | kvCopy[k] = v 30 | } 31 | kvCopy[tagName] = tagValue 32 | return kvCopy 33 | } 34 | 35 | // Convert all colection of key and values into string with {keyN=valN,...} format. 36 | func (kv KV) String() string { 37 | keys := make([]string, 0, len(kv)) 38 | values := make([]string, 0, len(kv)) 39 | for key, value := range kv { 40 | keys = append(keys, key) 41 | values = append(values, value) 42 | } 43 | 44 | sort.Sort(byKey{keys: keys, values: values}) 45 | var res string 46 | for i, key := range keys { 47 | res = res + key + "=" + values[i] 48 | if i != len(keys)-1 { 49 | res = res + "," 50 | } 51 | } 52 | 53 | return res 54 | } 55 | -------------------------------------------------------------------------------- /kglb/data_plane/utils.go: -------------------------------------------------------------------------------- 1 | package data_plane 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "dropbox/kglb/common" 8 | kglb_pb "dropbox/proto/kglb" 9 | ) 10 | 11 | func diffMetric(new, old uint64) float64 { 12 | if new < old { 13 | return float64(new) 14 | } 15 | return float64(new - old) 16 | } 17 | 18 | func getBgpRoutePrefix(route *kglb_pb.BgpRouteAttributes) string { 19 | return fmt.Sprintf("%s/%d", 20 | common.KglbAddrToNetIp(route.GetPrefix()), 21 | route.GetPrefixlen()) 22 | } 23 | 24 | func getBgpRouteName(route *kglb_pb.BgpRouteAttributes) string { 25 | return strings.Replace( 26 | strings.Replace(getBgpRoutePrefix(route), ":", ".", -1), 27 | " ", "_", -1) 28 | } 29 | 30 | // returns either Hostname or IP address. 31 | func getUpstreamHostname(dst *kglb_pb.UpstreamState) string { 32 | if len(dst.Hostname) > 0 { 33 | return dst.Hostname 34 | } 35 | 36 | // TODO(belyalov): it used to be reverse hostname resolution here 37 | // however it seems to be redundant (just for stats!) 38 | // If we found that decent amount of upstreams have no hostname 39 | // proper fix would be to resolve hostname when creating (discovering?) 40 | // upstream. 41 | 42 | // if no hostname present - return IP 43 | return fmt.Sprintf("%v", common.KglbAddrToNetIp(dst.Address)) 44 | } 45 | -------------------------------------------------------------------------------- /examples/example_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "balancers": [ 3 | { 4 | "dynamicRouting": { 5 | "announceLimitRatio": 0.9 6 | }, 7 | "lbService": { 8 | "ipvsService": { 9 | "tcpAttributes": { 10 | "address": { 11 | "ipv4": "192.168.100.10" 12 | }, 13 | "port": 80 14 | } 15 | } 16 | }, 17 | "name": "example-balancer", 18 | "setupName": "example-setup", 19 | "upstreamChecker": { 20 | "checker": { 21 | "dummy": {} 22 | }, 23 | "fallCount": 1, 24 | "intervalMs": 10000, 25 | "riseCount": 1 26 | }, 27 | "upstreamDiscovery": { 28 | "port": 80, 29 | "staticAttributes": { 30 | "hosts": [ 31 | "192.168.100.100", 32 | "192.168.100.101" 33 | ] 34 | } 35 | }, 36 | "upstreamRouting": { 37 | "forwardMethod": "TUNNEL" 38 | } 39 | } 40 | ] 41 | } 42 | -------------------------------------------------------------------------------- /kglb/control_plane/discovery_factory_test.go: -------------------------------------------------------------------------------- 1 | package control_plane 2 | 3 | import ( 4 | . "gopkg.in/check.v1" 5 | 6 | "dropbox/kglb/utils/discovery" 7 | pb "dropbox/proto/kglb" 8 | . "godropbox/gocheck2" 9 | ) 10 | 11 | type DiscoveryFactorySuite struct{} 12 | 13 | var _ = Suite(&DiscoveryFactorySuite{}) 14 | 15 | func (s *DiscoveryFactorySuite) TestStatic(c *C) { 16 | factory := NewDiscoveryFactory() 17 | 18 | resolver, err := factory.Resolver( 19 | c.TestName(), 20 | "testSetup", 21 | &pb.UpstreamDiscovery{ 22 | Port: 80, 23 | Attributes: &pb.UpstreamDiscovery_StaticAttributes{ 24 | StaticAttributes: &pb.StaticDiscoveryAttributes{ 25 | Hosts: []string{"test-host-1"}, 26 | }, 27 | }, 28 | }) 29 | c.Assert(err, IsNil) 30 | _, ok := resolver.(*discovery.StaticResolver) 31 | c.Assert(ok, IsTrue) 32 | 33 | // Updating. 34 | err = factory.Update(resolver, &pb.UpstreamDiscovery{ 35 | Port: 80, 36 | Attributes: &pb.UpstreamDiscovery_StaticAttributes{ 37 | StaticAttributes: &pb.StaticDiscoveryAttributes{ 38 | Hosts: []string{"test-host-2"}, 39 | }, 40 | }, 41 | }) 42 | c.Assert(err, IsNil) 43 | c.Assert(resolver.GetState().Equal( 44 | discovery.DiscoveryState([]*discovery.HostPort{ 45 | discovery.NewHostPort("test-host-2", 80, true), 46 | })), IsTrue) 47 | } 48 | -------------------------------------------------------------------------------- /kglb/utils/health_checker/tcp_checker.go: -------------------------------------------------------------------------------- 1 | package health_checker 2 | 3 | import ( 4 | "context" 5 | "net" 6 | "strconv" 7 | 8 | hc_pb "dropbox/proto/kglb/healthchecker" 9 | ) 10 | 11 | type TcpChecker struct { 12 | params *hc_pb.TcpCheckerAttributes 13 | dialContext DialContextFunc 14 | } 15 | 16 | func NewTcpChecker(params *hc_pb.TcpCheckerAttributes, dialContext DialContextFunc) (*TcpChecker, error) { 17 | var dc DialContextFunc 18 | if dialContext == nil { 19 | dc = defaultDialContext 20 | } else { 21 | dc = dialContext 22 | } 23 | checker := &TcpChecker{ 24 | dialContext: dc, 25 | params: params, 26 | } 27 | return checker, nil 28 | } 29 | 30 | func (h *TcpChecker) Check(host string, port int) error { 31 | timeout := timeoutMsToDuration(h.params.GetCheckTimeoutMs()) 32 | ctx, cancel := context.WithTimeout(context.Background(), timeout) 33 | defer cancel() 34 | 35 | address := net.JoinHostPort(host, strconv.Itoa(port)) 36 | c, err := h.dialContext(ctx, "tcp", address) 37 | defer func() { 38 | if c != nil { 39 | c.Close() 40 | } 41 | }() 42 | return err 43 | } 44 | 45 | func (h *TcpChecker) GetConfiguration() *hc_pb.HealthCheckerAttributes { 46 | return &hc_pb.HealthCheckerAttributes{ 47 | Attributes: &hc_pb.HealthCheckerAttributes_Tcp{ 48 | Tcp: h.params, 49 | }, 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | 11 | test: 12 | name: Test 13 | runs-on: ubuntu-latest 14 | steps: 15 | 16 | - name: Set up Go 1.13 17 | uses: actions/setup-go@v1 18 | with: 19 | go-version: 1.13 20 | id: go 21 | 22 | - name: Check out code into the Go module directory 23 | uses: actions/checkout@v2 24 | 25 | - name: Get dependencies 26 | run: | 27 | go install github.com/golang/protobuf/proto 28 | go install github.com/golang/protobuf/protoc-gen-go 29 | go get -v -t -d ./... 30 | if [ -f Gopkg.toml ]; then 31 | curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh 32 | dep ensure 33 | fi 34 | - name: Setup protoc 35 | uses: arduino/setup-protoc@v1.1.0 36 | with: 37 | version: 3.11.4 38 | # Include github pre-releases in latest version calculation 39 | include-pre-releases: false 40 | 41 | - name: Compile protobuf 42 | run: | 43 | cd ./proto 44 | protoc --go_out=. ./dropbox/proto/kglb/healthchecker/healthchecker.proto 45 | protoc --go_out=. ./dropbox/proto/kglb/*.proto 46 | - name: Test 47 | run: sudo go test -v ./... 48 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | 11 | build: 12 | name: Build 13 | runs-on: ubuntu-latest 14 | steps: 15 | 16 | - name: Set up Go 1.13 17 | uses: actions/setup-go@v1 18 | with: 19 | go-version: 1.13 20 | id: go 21 | 22 | - name: Check out code into the Go module directory 23 | uses: actions/checkout@v2 24 | 25 | - name: Get dependencies 26 | run: | 27 | go install github.com/golang/protobuf/proto 28 | go install github.com/golang/protobuf/protoc-gen-go 29 | go get -v -t -d ./... 30 | if [ -f Gopkg.toml ]; then 31 | curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh 32 | dep ensure 33 | fi 34 | 35 | - name: Setup protoc 36 | uses: arduino/setup-protoc@v1.1.0 37 | with: 38 | version: 3.11.4 39 | # Include github pre-releases in latest version calculation 40 | include-pre-releases: false 41 | 42 | - name: Compile protobuf 43 | run: | 44 | cd ./proto 45 | protoc --go_out=. ./dropbox/proto/kglb/healthchecker/healthchecker.proto 46 | protoc --go_out=. ./dropbox/proto/kglb/*.proto 47 | 48 | - name: Build 49 | run: go build -v -o kglbd.bin ./kglbd 50 | -------------------------------------------------------------------------------- /kglb/utils/dns_resolver/system_resolver.go: -------------------------------------------------------------------------------- 1 | package dns_resolver 2 | 3 | import ( 4 | "context" 5 | "net" 6 | "time" 7 | 8 | kglb_pb "dropbox/proto/kglb" 9 | "godropbox/errors" 10 | ) 11 | 12 | type SystemResolver struct { 13 | resolver *net.Resolver 14 | maxResolveTime time.Duration 15 | } 16 | 17 | var _ DnsResolver = &SystemResolver{} 18 | 19 | func NewSystemResolver(maxResolveTime time.Duration) (*SystemResolver, error) { 20 | return &SystemResolver{ 21 | resolver: &net.Resolver{}, 22 | maxResolveTime: maxResolveTime, 23 | }, nil 24 | } 25 | 26 | func (s *SystemResolver) ResolveHost(hostname string, af kglb_pb.AddressFamily) (*kglb_pb.IP, error) { 27 | ctx, cancel := context.WithTimeout(context.Background(), s.maxResolveTime) 28 | defer cancel() 29 | 30 | if addrs, err := s.resolver.LookupIPAddr(ctx, hostname); err != nil { 31 | return nil, err 32 | } else { 33 | for _, addr := range addrs { 34 | if af == kglb_pb.AddressFamily_AF_INET && addr.IP.To4() != nil { 35 | return &kglb_pb.IP{ 36 | Address: &kglb_pb.IP_Ipv4{Ipv4: addr.String()}, 37 | }, nil 38 | } else if af == kglb_pb.AddressFamily_AF_INET6 && addr.IP.To4() == nil { 39 | return &kglb_pb.IP{ 40 | Address: &kglb_pb.IP_Ipv6{Ipv6: addr.String()}, 41 | }, nil 42 | } 43 | } 44 | } 45 | 46 | return nil, errors.Newf("fails to resolve: hostname: %s, af: %s", hostname, af.String()) 47 | } 48 | -------------------------------------------------------------------------------- /kglb/utils/health_checker/all_test.go: -------------------------------------------------------------------------------- 1 | package health_checker 2 | 3 | import ( 4 | "net" 5 | "net/http" 6 | "net/http/httptest" 7 | "strconv" 8 | "testing" 9 | 10 | . "gopkg.in/check.v1" 11 | ) 12 | 13 | func Test(t *testing.T) { 14 | TestingT(t) 15 | } 16 | 17 | type BackendHandler struct { 18 | handler func(writer http.ResponseWriter, request *http.Request) 19 | } 20 | 21 | func (b *BackendHandler) ServeHTTP( 22 | writer http.ResponseWriter, 23 | request *http.Request) { 24 | 25 | b.handler(writer, request) 26 | } 27 | 28 | // Create http test server and returns its address and port 29 | func NewBackend( 30 | c *C, 31 | handler func(writer http.ResponseWriter, 32 | request *http.Request)) (*httptest.Server, string, int) { 33 | 34 | server := httptest.NewServer(&BackendHandler{ 35 | handler: handler, 36 | }) 37 | 38 | backendHost, backendPortStr, err := net.SplitHostPort( 39 | server.Listener.Addr().String()) 40 | c.Assert(err, IsNil) 41 | backendPort, err := strconv.Atoi(backendPortStr) 42 | c.Assert(err, IsNil) 43 | c.Log("created backend: ", server.Listener.Addr().String()) 44 | 45 | return server, backendHost, backendPort 46 | } 47 | 48 | func NewBackendWithCustomAddr( 49 | c *C, 50 | address string, 51 | handler func(writer http.ResponseWriter, 52 | request *http.Request)) *http.Server { 53 | 54 | srv := &http.Server{ 55 | Addr: address, 56 | Handler: &BackendHandler{ 57 | handler: handler, 58 | }, 59 | } 60 | go func() { 61 | srv.ListenAndServe() 62 | }() 63 | 64 | return srv 65 | } 66 | -------------------------------------------------------------------------------- /kglb/control_plane/stats.go: -------------------------------------------------------------------------------- 1 | package control_plane 2 | 3 | import ( 4 | "dropbox/vortex2/v2stats" 5 | ) 6 | 7 | // Current balancer state (on given setup/service) 8 | // Tags: 9 | // - state: current state of balancer [initial, available, no_upstreams, failsafe, shutdown] 10 | // - setup: setup name 11 | // - service: service name 12 | var balancerStateGauge = v2stats.MustDefineGauge("kglb/control_plane/balancer_state", "setup", "service", "state") 13 | 14 | // Route announcement gauge 15 | // Tags: 16 | // - state: current state of route announcement: [on/off] 17 | // - setup: setup name 18 | // - service: service name 19 | var routeAnnouncementGauge = v2stats.MustDefineGauge("kglb/control_plane/route_announcement", "setup", "service", "state") 20 | 21 | // Simple gauge of alive upstreams count. 22 | // Tags: 23 | // - setup: setup name 24 | // - service: service name 25 | // - alive: [true/false] 26 | var upstreamsCountGauge = v2stats.MustDefineGauge("kglb/control_plane/upstreams_count", "setup", "service", "alive") 27 | 28 | // KGLB availability gauge (gauge value indicates availabilit: 0/1) 29 | // Tags: 30 | // - initialized: [true/false] - indicates control plane initialization status 31 | var availabilityGauge = v2stats.MustDefineGauge("kglb/control_plane/availability", "initialized") 32 | 33 | // KGLB state hash gauge (gauge value being used to compare between different kglbs in same cluster) 34 | // same value means they are consistent (see the same amount of up/down hosts) 35 | var stateHashGauge = v2stats.MustDefineGauge("kglb/control_plane/state_hash", "entity", "setup") 36 | -------------------------------------------------------------------------------- /kglb/data_plane/config_provider.go: -------------------------------------------------------------------------------- 1 | package data_plane 2 | 3 | import ( 4 | "github.com/gogo/protobuf/proto" 5 | "godropbox/errors" 6 | 7 | "dropbox/kglb/common" 8 | "dropbox/kglb/utils/config_loader" 9 | pb "dropbox/proto/kglb" 10 | ) 11 | 12 | type ConfigProvider struct { 13 | } 14 | 15 | // The default configuration when the config file (or the locally cached 16 | // config file) is absent. The default configuration must be non-nil and 17 | // valid (i.e., Validate(cfg) returns nil). 18 | func (k *ConfigProvider) Default() interface{} { 19 | return &pb.DataPlaneState{} 20 | } 21 | 22 | // Given a config file content, return the parsed config object (or error). 23 | func (k *ConfigProvider) Parse(content []byte) (interface{}, error) { 24 | cfg := &pb.DataPlaneState{} 25 | if err := proto.UnmarshalText(string(content), cfg); err != nil { 26 | return nil, errors.Wrapf(err, "Failed to parse config: ") 27 | } 28 | return cfg, nil 29 | } 30 | 31 | // Validate ensures the config is semantically correct. 32 | func (k *ConfigProvider) Validate(abstractCfg interface{}) error { 33 | state, ok := abstractCfg.(*pb.DataPlaneState) 34 | if !ok { 35 | return errors.New("unexpected type of config") 36 | } 37 | 38 | // Validate config. 39 | return common.ValidateDataPlaneState(state) 40 | } 41 | 42 | // The config loader will only sent updates when the config file's 43 | // content has modified. 44 | func (k *ConfigProvider) Equals(cfg1 interface{}, cfg2 interface{}) bool { 45 | return common.DataPlaneStateComparable.Equal(cfg1, cfg2) 46 | } 47 | 48 | var _ config_loader.ConfigProvider = &ConfigProvider{} 49 | -------------------------------------------------------------------------------- /kglb/utils/config_loader/one_time_file_loader_test.go: -------------------------------------------------------------------------------- 1 | package config_loader 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | 7 | . "gopkg.in/check.v1" 8 | 9 | . "godropbox/gocheck2" 10 | ) 11 | 12 | type StringProvider struct { 13 | } 14 | 15 | func (p *StringProvider) Default() interface{} { 16 | return "" 17 | } 18 | 19 | func (p *StringProvider) Parse(content []byte) (cfg interface{}, err error) { 20 | return string(content), nil 21 | } 22 | 23 | func (p *StringProvider) Validate(cfg interface{}) error { 24 | return nil 25 | } 26 | 27 | func (p *StringProvider) Equals(cfg1 interface{}, cfg2 interface{}) bool { 28 | return cfg1.(string) == cfg2.(string) 29 | } 30 | 31 | type OneTimeFileLoaderSuite struct { 32 | } 33 | 34 | var _ = Suite(&OneTimeFileLoaderSuite{}) 35 | 36 | func (l *OneTimeFileLoaderSuite) TestLoader(c *C) { 37 | content := []byte("test content") 38 | tmpfile, err := ioutil.TempFile("", "test.*.txt") 39 | c.Assert(err, IsNil) 40 | defer os.Remove(tmpfile.Name()) // clean up 41 | 42 | _, err = tmpfile.Write(content) 43 | c.Assert(err, IsNil) 44 | err = tmpfile.Close() 45 | c.Assert(err, IsNil) 46 | 47 | loader, err := NewOneTimeFileLoader(&StringProvider{}, tmpfile.Name()) 48 | c.Assert(err, IsNil) 49 | 50 | select { 51 | case fileContent, ok := <-loader.Updates(): 52 | c.Assert(ok, IsTrue) 53 | c.Assert(fileContent, Equals, string(content)) 54 | default: 55 | c.Fail() 56 | } 57 | } 58 | 59 | func (l *OneTimeFileLoaderSuite) TestMissedFile(c *C) { 60 | loader, err := NewOneTimeFileLoader(&StringProvider{}, c.TestName()) 61 | c.Assert(err, NotNil) 62 | c.Assert(loader, IsNil) 63 | } 64 | -------------------------------------------------------------------------------- /kglb/utils/fwmark/fwmark_manager_test.go: -------------------------------------------------------------------------------- 1 | package fwmark 2 | 3 | import ( 4 | . "gopkg.in/check.v1" 5 | ) 6 | 7 | var _ = Suite(&FwmarkManagerSuite{}) 8 | 9 | func (s *FwmarkManagerSuite) TestGetAndRelease(c *C) { 10 | manager := NewManager(2, 10) 11 | 12 | // new fwmark 13 | fwmark, err := manager.AllocateFwmark("10.0.0.1") 14 | c.Assert(err, IsNil) 15 | c.Assert(fwmark, Equals, uint32(10)) 16 | 17 | // trying to get non allocated fwmark 18 | _, err = manager.GetAllocatedFwmark("10.0.0.2") 19 | c.Assert(err, NotNil) 20 | 21 | // another fwmark 22 | fwmark, err = manager.AllocateFwmark("10.0.0.2") 23 | c.Assert(err, IsNil) 24 | c.Assert(fwmark, Equals, uint32(11)) 25 | 26 | // bump refcount for existing 27 | fwmark, err = manager.AllocateFwmark("10.0.0.1") 28 | c.Assert(err, IsNil) 29 | c.Assert(fwmark, Equals, uint32(10)) 30 | 31 | // get allocated fwmark w/o refcounter bumping 32 | fwmark, err = manager.GetAllocatedFwmark("10.0.0.1") 33 | c.Assert(err, IsNil) 34 | c.Assert(fwmark, Equals, uint32(10)) 35 | 36 | c.Assert(len(manager.fwmarksPool), Equals, 0) 37 | 38 | // no more fwmarks in free pool 39 | _, err = manager.AllocateFwmark("10.0.0.3") 40 | c.Assert(err, NotNil) 41 | 42 | // releasing one w/ refcount 2. 2 times no error, 43 | // and error on 3rd time 44 | c.Assert(len(manager.fwmarksPool), Equals, 0) 45 | 46 | err = manager.ReleaseFwmark("10.0.0.1") 47 | c.Assert(err, IsNil) 48 | 49 | err = manager.ReleaseFwmark("10.0.0.1") 50 | c.Assert(err, IsNil) 51 | 52 | c.Assert(len(manager.fwmarksPool), Equals, 1) 53 | 54 | err = manager.ReleaseFwmark("10.0.0.1") 55 | c.Assert(err, NotNil) 56 | } 57 | -------------------------------------------------------------------------------- /kglbd/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "net/http" 7 | "os" 8 | "os/signal" 9 | "syscall" 10 | "time" 11 | 12 | "github.com/golang/glog" 13 | "github.com/prometheus/client_golang/prometheus/promhttp" 14 | ) 15 | 16 | func main() { 17 | flagStatusPort := flag.String( 18 | "status_port", 19 | "127.0.0.1:5678", 20 | "status port.") 21 | 22 | flagConfigPath := flag.String( 23 | "config", 24 | "", 25 | "full path to the configuration.") 26 | flag.Parse() 27 | 28 | if len(*flagConfigPath) == 0 { 29 | glog.Fatal("-config is required path.") 30 | } 31 | 32 | ctx, cancel := context.WithCancel(context.Background()) 33 | defer cancel() 34 | 35 | mng, err := NewService(ctx, *flagConfigPath) 36 | if err != nil { 37 | glog.Fatal(err) 38 | } 39 | 40 | // handle signals defined in the array below to perform graceful 41 | // data plane shutdown. 42 | sigChan := make(chan os.Signal, 1) 43 | signal.Notify(sigChan, []os.Signal{ 44 | syscall.SIGINT, 45 | syscall.SIGQUIT, 46 | syscall.SIGTERM, 47 | syscall.SIGUSR1, 48 | }...) 49 | 50 | mux := http.NewServeMux() 51 | mux.Handle("/stats", promhttp.Handler()) 52 | 53 | srv := &http.Server{ 54 | Addr: *flagStatusPort, 55 | Handler: mux, 56 | ReadTimeout: time.Minute, 57 | WriteTimeout: time.Minute, 58 | MaxHeaderBytes: 1 << 20, 59 | } 60 | 61 | go func() { 62 | glog.Fatal(srv.ListenAndServe()) 63 | }() 64 | 65 | select { 66 | case sig := <-sigChan: 67 | glog.Infof("Received '%v', starting shutdown process...", sig) 68 | ctx.Done() 69 | } 70 | 71 | mng.Shutdown() 72 | srv.Close() 73 | } 74 | -------------------------------------------------------------------------------- /kglb/utils/discovery/host_ports.go: -------------------------------------------------------------------------------- 1 | package discovery 2 | 3 | import ( 4 | "net" 5 | "strconv" 6 | ) 7 | 8 | // Resolver primitive. 9 | type HostPort struct { 10 | // Hostname 11 | Host string 12 | // port number 13 | Port int 14 | // address or hostname to use for healthchecks (depends on DnsResolver config) 15 | Address string 16 | // flag which indicates if host is enabled for taking traffic or not 17 | Enabled bool 18 | } 19 | 20 | func NewHostPort(host string, port int, enabled bool) *HostPort { 21 | return &HostPort{ 22 | Host: host, 23 | Port: port, 24 | Address: host, 25 | Enabled: enabled, 26 | } 27 | } 28 | 29 | // Compare HostPort items. 30 | func (h *HostPort) Equal(item *HostPort) bool { 31 | if item != nil && h.Host == item.Host && h.Port == item.Port && h.Address == item.Address && h.Enabled == item.Enabled { 32 | return true 33 | } 34 | return false 35 | } 36 | 37 | func (h *HostPort) String() string { 38 | return net.JoinHostPort(h.Host, strconv.Itoa(h.Port)) 39 | } 40 | 41 | type DiscoveryState []*HostPort 42 | 43 | func (h DiscoveryState) Len() int { 44 | return len(h) 45 | } 46 | 47 | // Returns true when DiscoveryState contains hostPort, otherwise false. 48 | func (h DiscoveryState) Contains(hostPort *HostPort) bool { 49 | for _, entry := range h { 50 | if hostPort.Equal(entry) { 51 | return true 52 | } 53 | } 54 | 55 | return false 56 | } 57 | 58 | // Compare two DiscoveryState's 59 | func (h DiscoveryState) Equal(state DiscoveryState) bool { 60 | if h.Len() != state.Len() { 61 | return false 62 | } 63 | 64 | for _, curEntry := range h { 65 | found := false 66 | for _, newEntry := range state { 67 | if curEntry.Equal(newEntry) { 68 | found = true 69 | break 70 | } 71 | } 72 | if !found { 73 | return false 74 | } 75 | } 76 | 77 | return true 78 | } 79 | -------------------------------------------------------------------------------- /proto/dropbox/proto/kglb/config.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package kglb; 4 | 5 | import "dropbox/proto/kglb/common.proto"; 6 | import "dropbox/proto/kglb/healthchecker/healthchecker.proto"; 7 | 8 | 9 | message MdbDiscoveryAttributes { 10 | string query = 1; 11 | } 12 | 13 | message StaticDiscoveryAttributes { 14 | repeated string hosts = 1; 15 | } 16 | 17 | message UpstreamDiscovery { 18 | uint32 port = 1; 19 | 20 | AddressFamily resolve_family = 2; 21 | 22 | oneof attributes { 23 | MdbDiscoveryAttributes mdb_attributes = 10; 24 | StaticDiscoveryAttributes static_attributes = 11; 25 | } 26 | } 27 | 28 | message UpstreamRouting { 29 | ForwardMethods forward_method = 1; 30 | } 31 | 32 | message DynamicRouting { 33 | float announce_limit_ratio = 1; 34 | 35 | oneof attributes { 36 | BgpRouteAttributes bgp_attributes = 10; 37 | } 38 | } 39 | 40 | // next id: 11 41 | message BalancerConfig { 42 | // balancer name, one setup_name may consist of multiple name's 43 | string name = 1; 44 | // global setup name: nginx-edge, block-wwww, etc. 45 | string setup_name = 8; 46 | 47 | LoadBalancerService lb_service = 2; 48 | 49 | UpstreamRouting upstream_routing = 3; 50 | UpstreamDiscovery upstream_discovery = 4; 51 | healthchecker.UpstreamChecker upstream_checker = 5; 52 | // set SO_MARK on the healthcheck socket 53 | bool enable_fwmarks = 10; 54 | 55 | DynamicRouting dynamic_routing = 6; 56 | 57 | // Connects to discovered reals and adds vips to their loopback interface. 58 | // (required with tunneling forward method to properly handle encapsulated 59 | // packets). 60 | bool configure_reals = 7; 61 | 62 | // Default weight of healthy discovered realserver. 63 | // Default value is 1000. 64 | uint32 weight_up = 9; 65 | } 66 | 67 | message ControlPlaneConfig { 68 | repeated BalancerConfig balancers = 1; 69 | } 70 | -------------------------------------------------------------------------------- /kglb/utils/health_checker/health_checker.go: -------------------------------------------------------------------------------- 1 | package health_checker 2 | 3 | import ( 4 | "context" 5 | "net" 6 | "time" 7 | 8 | hc_pb "dropbox/proto/kglb/healthchecker" 9 | "godropbox/errors" 10 | ) 11 | 12 | const ( 13 | // default Timeout to perform individual health check. 14 | defaultTimeout = 5 * time.Second 15 | ) 16 | 17 | // Common interface for all kind of health checkers. 18 | type HealthChecker interface { 19 | // Performs test and returns true when test was succeed. It may return error 20 | // when test cannot be performed and as result healthy state cannot be 21 | // properly determined. 22 | Check(host string, port int) error 23 | // Returns current HC configuration 24 | GetConfiguration() *hc_pb.HealthCheckerAttributes 25 | } 26 | 27 | func NewHealthChecker(checker *hc_pb.UpstreamChecker, dialContext DialContextFunc) (HealthChecker, error) { 28 | switch attr := checker.GetChecker().GetAttributes().(type) { 29 | case *hc_pb.HealthCheckerAttributes_Dummy: 30 | return NewDummyChecker(attr.Dummy) 31 | case *hc_pb.HealthCheckerAttributes_Http: 32 | return NewHttpChecker(attr.Http, dialContext) 33 | case *hc_pb.HealthCheckerAttributes_Syslog: 34 | return NewSyslogChecker(attr.Syslog, dialContext) 35 | case *hc_pb.HealthCheckerAttributes_Dns: 36 | return NewDnsChecker(attr.Dns, dialContext) 37 | case *hc_pb.HealthCheckerAttributes_Tcp: 38 | return NewTcpChecker(attr.Tcp, dialContext) 39 | default: 40 | return nil, errors.Newf("Unknown Health Checker type: %s", attr) 41 | } 42 | } 43 | 44 | type DialContextFunc func(ctx context.Context, network, addr string) (net.Conn, error) 45 | 46 | var defaultDialContext DialContextFunc = func(ctx context.Context, network, address string) (net.Conn, error) { 47 | dialer := net.Dialer{} 48 | return dialer.DialContext(ctx, network, address) 49 | } 50 | 51 | func timeoutMsToDuration(timeoutMs uint32) time.Duration { 52 | if timeoutMs == 0 { 53 | return defaultTimeout 54 | } 55 | return time.Duration(timeoutMs) * time.Millisecond 56 | } 57 | -------------------------------------------------------------------------------- /kglb/data_plane/stats.go: -------------------------------------------------------------------------------- 1 | package data_plane 2 | 3 | import ( 4 | "dropbox/vortex2/v2stats" 5 | ) 6 | 7 | // KGLB Dataplane manager state (defined as public since it is being used by dbx_data_plane) 8 | // Tags: 9 | // - state: current state [initialized, init_failed, available, set_state_failed, shutdown_failed] 10 | var ManagerStateGauge = v2stats.MustDefineGauge("kglb/data_plane/manager_state", "state") 11 | 12 | // Time since the last successful state change in seconds (since startup) 13 | var ManagerStateAgeSec = v2stats.MustDefineGauge("kglb/data_plane/manager_state_age_sec") 14 | 15 | // KGLB data plane adds IP address to lo interface 16 | // Here is the gauge to track this state 17 | // Tags: 18 | // - address: Interface / IP address pair which is being added, e.g. "lo/1.2.3.4" 19 | // - state: [alive, add_failed, delete_failed] 20 | var linkAddressGauge = v2stats.MustDefineGauge("kglb/data_plane/link_address", "address", "state") 21 | 22 | // Per Service stats // 23 | // 24 | // bytes received / sent 25 | // Tags: 26 | // - name: service name 27 | // - direction: [in, out] 28 | var serviceBytesCounter = v2stats.MustDefineCounter("kglb/data_plane/service_bytes", "name", "direction") 29 | 30 | // Packets received / sent 31 | // Tags: 32 | // - name: service name 33 | // - direction: [in, out] 34 | var servicePacketsCounter = v2stats.MustDefineCounter("kglb/data_plane/service_packets", "name", "direction") 35 | 36 | // Per Upstream Stats // 37 | // 38 | // Bytes received / sent 39 | // Tags: 40 | // - name: upstream name 41 | // - direction: [in, out] 42 | var upstreamBytesCounter = v2stats.MustDefineCounter("kglb/data_plane/upstream_bytes", "name", "direction") 43 | 44 | // Packets received / sent per service 45 | // Tags: 46 | // - name: upstream name 47 | // - direction: [in, out] 48 | var upstreamPacketsCounter = v2stats.MustDefineCounter("kglb/data_plane/upstream_packets", "name", "direction") 49 | 50 | // BGP 51 | // 52 | // BGP session state. 53 | // Tags: 54 | // - state [established, not_established] 55 | var bgpSessionStateGauge = v2stats.MustDefineGauge("kglb/data_plane/bgp_session", "state") 56 | 57 | // BGP routes state. 58 | // Tags: 59 | // - route - advertised IP CIDR, e.g. 162.125.248.1/32 60 | var bgpRouteGauge = v2stats.MustDefineGauge("kglb/data_plane/bgp_route", "route") 61 | -------------------------------------------------------------------------------- /kglb/utils/health_manager/health_status.go: -------------------------------------------------------------------------------- 1 | package health_manager 2 | 3 | import ( 4 | "sync" 5 | ) 6 | 7 | type healthStatusEntry struct { 8 | // Current status. 9 | isHealthy bool 10 | 11 | // positive = # of consecutive successful health checks 12 | // negative = # of consecutive failed health checks 13 | healthCount int 14 | 15 | // Mutex to protect fields of the struct. 16 | mutex sync.Mutex 17 | } 18 | 19 | // Returns new and initialized entry of healthStatusEntry. 20 | func NewHealthStatusEntry(initialHealthyState bool) *healthStatusEntry { 21 | return &healthStatusEntry{ 22 | isHealthy: initialHealthyState, 23 | } 24 | } 25 | 26 | // Getter to access internal isHealthy field. Returns true when current status 27 | // of the entry is healthy, otherwise false. 28 | func (h *healthStatusEntry) IsHealthy() bool { 29 | h.mutex.Lock() 30 | defer h.mutex.Unlock() 31 | 32 | return h.isHealthy 33 | } 34 | 35 | // Updates status of the entry based on latest health check results and returns 36 | // true when status has been changed during UpdateEntry call. 37 | func (h *healthStatusEntry) UpdateHealthCheckStatus( 38 | healthCheckPassed bool, 39 | riseCount int, 40 | fallCount int) bool { 41 | 42 | h.mutex.Lock() 43 | defer h.mutex.Unlock() 44 | 45 | if healthCheckPassed { 46 | if h.healthCount < 0 { // previous health check(s) failed 47 | h.healthCount = 1 48 | } else { 49 | h.healthCount++ 50 | } 51 | 52 | if !h.isHealthy && 53 | h.healthCount >= riseCount { 54 | 55 | h.isHealthy = true 56 | return true 57 | } 58 | 59 | return false 60 | } 61 | 62 | if h.healthCount > 0 { // previous health check(s) passed 63 | h.healthCount = -1 64 | } else { 65 | h.healthCount-- 66 | } 67 | 68 | // NOTE: -entry.healthCount because we use negative value 69 | // to track failed health checks. 70 | if h.isHealthy && 71 | -h.healthCount >= fallCount { 72 | 73 | h.isHealthy = false 74 | return true 75 | } 76 | 77 | return false 78 | } 79 | 80 | // Comparese two healthStatusEntry entries and returns true when both are 81 | // identical (including healthy status and internal counter). 82 | func (h *healthStatusEntry) Equal(entry *healthStatusEntry) bool { 83 | h.mutex.Lock() 84 | defer h.mutex.Unlock() 85 | 86 | return h.healthCount == entry.healthCount && h.isHealthy == entry.isHealthy 87 | } 88 | -------------------------------------------------------------------------------- /kglb/utils/concurrency/concurrency.go: -------------------------------------------------------------------------------- 1 | package concurrency 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | ) 7 | 8 | // CompleteTasks is a blocking call to perform execution of taskFunccallback 9 | // numTasks times by using specific number of workers specified in numWorkers 10 | // argument (0 means unlimited, number of spawn workers will be equal to number 11 | // of numTasks). 12 | func CompleteTasks( 13 | ctx context.Context, 14 | numWorkers int, 15 | numTasks int, 16 | taskFunc func(workerId, taskId int)) error { 17 | 18 | // nothing to do if there is no any entries to check. 19 | if numTasks == 0 { 20 | return nil 21 | } 22 | 23 | // use as much workers as number of entries when numWorkers is not 24 | // specified. 25 | if numTasks < numWorkers || numWorkers == 0 { 26 | numWorkers = numTasks 27 | } 28 | 29 | numToCheck := (numTasks + numWorkers - 1) / numWorkers 30 | 31 | // using wait group to track of completed tasks to make whole call of 32 | // CompleteTasks() blocking until completion of all tasks. 33 | var waitGroup sync.WaitGroup 34 | waitGroup.Add(numWorkers) 35 | 36 | for i := 0; i < numWorkers; i++ { 37 | go func(workerId int, wg *sync.WaitGroup) { 38 | defer wg.Done() 39 | 40 | start := workerId * numToCheck 41 | end := (workerId + 1) * numToCheck 42 | if end > numTasks { 43 | end = numTasks 44 | } 45 | 46 | for pos := start; pos < end; pos++ { 47 | // Stop loop if context canceled 48 | select { 49 | case <-ctx.Done(): 50 | return 51 | default: 52 | taskFunc(workerId, pos) 53 | } 54 | } 55 | }(i, &waitGroup) 56 | } 57 | 58 | doneChan := make(chan interface{}) 59 | go func(ch chan interface{}, wg *sync.WaitGroup) { 60 | // Wait for all workers to complete tasks. 61 | wg.Wait() 62 | close(ch) 63 | }(doneChan, &waitGroup) 64 | 65 | select { 66 | case <-doneChan: 67 | // From documentation: 68 | // A select blocks until one of its cases can run, then it executes that case. 69 | // It chooses one at random if multiple are ready. 70 | // Saying that it is possible that all 2 channels are readable. 71 | // In this case it may also mean that all workers have finished due to canceled context 72 | select { 73 | case <-ctx.Done(): 74 | return ctx.Err() 75 | default: 76 | return nil 77 | } 78 | case <-ctx.Done(): 79 | return ctx.Err() 80 | } 81 | 82 | } 83 | -------------------------------------------------------------------------------- /kglb/utils/comparable/arrays_test.go: -------------------------------------------------------------------------------- 1 | package comparable 2 | 3 | import ( 4 | . "gopkg.in/check.v1" 5 | 6 | . "godropbox/gocheck2" 7 | ) 8 | 9 | type ArraysDiffSuite struct { 10 | } 11 | 12 | var _ = Suite(&ArraysDiffSuite{}) 13 | 14 | // Validate functionality on array of strings. 15 | func (m *ArraysDiffSuite) TestBasicStrings(c *C) { 16 | 17 | // Comparable interface implementation. 18 | comparable := &ComparableImpl{ 19 | KeyFunc: func(item interface{}) string { 20 | return item.(string) 21 | }, 22 | EqualFunc: func(item1, item2 interface{}) bool { 23 | return item1.(string) == item2.(string) 24 | }, 25 | } 26 | 27 | result := CompareArrays( 28 | []interface{}{ 29 | "a", 30 | "b", 31 | "c", 32 | }, 33 | []interface{}{ 34 | "b", 35 | "d", 36 | }, 37 | comparable) 38 | 39 | c.Assert(result.Added, DeepEqualsPretty, []interface{}{"d"}) 40 | c.Assert(len(result.Deleted), Equals, 2) // deleted "a", "c" 41 | c.Assert(result.Unchanged, DeepEqualsPretty, []interface{}{"b"}) 42 | c.Assert(result.Changed, IsNil) 43 | } 44 | 45 | // Validate functionality on array of strings. 46 | func (m *ArraysDiffSuite) TestChanges(c *C) { 47 | type testItem struct { 48 | el1 string 49 | el2 int 50 | } 51 | 52 | // Comparable interface implementation. 53 | comparable := &ComparableImpl{ 54 | KeyFunc: func(item interface{}) string { 55 | return item.(*testItem).el1 56 | }, 57 | EqualFunc: func(item1, item2 interface{}) bool { 58 | return item1.(*testItem).el2 == item2.(*testItem).el2 59 | }, 60 | } 61 | 62 | result := CompareArrays( 63 | []interface{}{ 64 | &testItem{el1: "1", el2: 10}, 65 | &testItem{el1: "2", el2: 20}, 66 | &testItem{el1: "3", el2: 30}, 67 | }, 68 | []interface{}{ 69 | &testItem{el1: "2", el2: 40}, // modify second 70 | &testItem{el1: "3", el2: 30}, 71 | &testItem{el1: "4", el2: 10}, 72 | }, 73 | 74 | comparable) 75 | 76 | c.Assert(result.Added, DeepEqualsPretty, []interface{}{ 77 | &testItem{el1: "4", el2: 10}, 78 | }) 79 | c.Assert(result.Deleted, DeepEqualsPretty, []interface{}{ 80 | &testItem{el1: "1", el2: 10}, 81 | }) 82 | c.Assert(result.Unchanged, DeepEqualsPretty, []interface{}{ 83 | &testItem{el1: "3", el2: 30}, 84 | }) 85 | c.Assert(result.Changed, DeepEqualsPretty, []ChangedPair{ 86 | ChangedPair{ 87 | OldItem: &testItem{el1: "2", el2: 20}, 88 | NewItem: &testItem{el1: "2", el2: 40}, 89 | }, 90 | }) 91 | } 92 | -------------------------------------------------------------------------------- /kglb/data_plane/resolver.go: -------------------------------------------------------------------------------- 1 | package data_plane 2 | 3 | import ( 4 | "context" 5 | "net" 6 | "time" 7 | 8 | "godropbox/errors" 9 | 10 | kglb_pb "dropbox/proto/kglb" 11 | ) 12 | 13 | var ( 14 | // default resolution timeout. 15 | defaultResolutionTimeout = 500 * time.Millisecond 16 | ) 17 | 18 | const ( 19 | // value of nknown service_name tag in case of failed resolution. 20 | unknownService = "default" 21 | // value of hostname tag in case of failed resolution. 22 | unknownHostname = "unknown" 23 | // value of real_cluster tag in case of failed resolution. 24 | unknownRealCluster = "none" 25 | ) 26 | 27 | type Resolver struct { 28 | resolver *net.Resolver 29 | } 30 | 31 | func NewResolver() (*Resolver, error) { 32 | return &Resolver{ 33 | resolver: &net.Resolver{}, 34 | }, nil 35 | } 36 | 37 | // lookup (hostname resolution). 38 | func (r *Resolver) Lookup(name string) (*HostnameCacheEntry, error) { 39 | ctx, cancel := context.WithTimeout( 40 | context.TODO(), 41 | defaultResolutionTimeout) 42 | defer cancel() 43 | var result = &HostnameCacheEntry{} 44 | 45 | addrs, err := r.resolver.LookupIPAddr(ctx, name) 46 | if err != nil { 47 | return nil, err 48 | } 49 | 50 | if len(addrs) == 0 { 51 | return nil, errors.Newf( 52 | "fails to resolve %s name, reason: %v", name, err) 53 | } 54 | 55 | for _, addr := range addrs { 56 | if addr.IP.To4() != nil { 57 | result.IPv4 = addr.IP 58 | } else { 59 | result.IPv6 = addr.IP 60 | } 61 | } 62 | return result, nil 63 | } 64 | 65 | // cluster name by hostname 66 | func (r *Resolver) Cluster(hostname string) string { 67 | if len(hostname) > 3 { 68 | return hostname[:3] 69 | } 70 | return unknownRealCluster 71 | } 72 | 73 | // reverse lookup (ip -> hostname resolution). 74 | func (r *Resolver) ReverseLookup(ip net.IP) (string, error) { 75 | ctx, cancel := context.WithTimeout( 76 | context.TODO(), 77 | defaultResolutionTimeout) 78 | defer cancel() 79 | 80 | names, err := r.resolver.LookupAddr(ctx, ip.String()) 81 | if err != nil { 82 | return unknownHostname, err 83 | } 84 | 85 | if len(names) == 0 { 86 | return unknownHostname, errors.Newf( 87 | "fails to resolve %s addr, reason: %v", ip.String(), err) 88 | } 89 | 90 | return names[0], nil 91 | } 92 | 93 | // Get service name by information available in IpvsService message. 94 | func (r *Resolver) ServiceLookup(srv *kglb_pb.IpvsService) string { 95 | return unknownService 96 | } 97 | -------------------------------------------------------------------------------- /vortex2/v2stats/gauge_group.go: -------------------------------------------------------------------------------- 1 | package v2stats 2 | 3 | import ( 4 | "math" 5 | "sync" 6 | ) 7 | 8 | type gaugeAndValue struct { 9 | gauge Gauge 10 | value float64 11 | } 12 | 13 | type GaugeGroup struct { 14 | mu sync.Mutex 15 | gaugeDef GaugeDefinition 16 | gaugesAndValues map[string]gaugeAndValue 17 | } 18 | 19 | func NewGaugeGroup(gaugeDef GaugeDefinition) *GaugeGroup { 20 | return &GaugeGroup{ 21 | gaugeDef: gaugeDef, 22 | gaugesAndValues: make(map[string]gaugeAndValue), 23 | } 24 | } 25 | 26 | // PrepareToSet records that we want to set the kv to value. It does not 27 | // actually set the gauge value until SetAndReset is called. 28 | // 29 | // Note: we don't immediately set the value to ensure that SetAndReset is 30 | // called, since that's required for the metric clearing to work. 31 | func (g *GaugeGroup) PrepareToSet(value float64, kv KV) error { 32 | g.mu.Lock() 33 | defer g.mu.Unlock() 34 | 35 | kvStr := kv.String() 36 | 37 | var gauge Gauge 38 | if gv, ok := g.gaugesAndValues[kvStr]; ok { 39 | gauge = gv.gauge 40 | } else { 41 | var err error 42 | gauge, err = g.gaugeDef.V(kv) 43 | if err != nil { 44 | return err 45 | } 46 | } 47 | 48 | g.gaugesAndValues[kvStr] = gaugeAndValue{ 49 | gauge: gauge, 50 | value: value, 51 | } 52 | return nil 53 | } 54 | 55 | func (g *GaugeGroup) MustPrepareToSet(value float64, kv KV) { 56 | g.mu.Lock() 57 | defer g.mu.Unlock() 58 | 59 | err := g.PrepareToSet(value, kv) 60 | if err != nil { 61 | panic(err) 62 | } 63 | } 64 | 65 | // SetAndReset sets all the gauge values and clears values that weren't prepared 66 | // since the last call to SetAndReset. 67 | func (g *GaugeGroup) SetAndReset() { 68 | g.mu.Lock() 69 | defer g.mu.Unlock() 70 | 71 | // Set all the values. 72 | for _, gv := range g.gaugesAndValues { 73 | gv.gauge.Set(gv.value) 74 | } 75 | 76 | // Prepare gauges to be cleared upon the next call to SetAndReset, and 77 | // delete gauges that were cleared on this call. 78 | for kvStr, gv := range g.gaugesAndValues { 79 | if math.IsNaN(gv.value) { 80 | // Value has already been cleared, no need to store it anymore. 81 | delete(g.gaugesAndValues, kvStr) 82 | } else { 83 | // Value will be cleared next time unless it's set to a non-NaN value. 84 | gv.gauge.Set(math.NaN()) 85 | g.gaugesAndValues[kvStr] = gaugeAndValue{ 86 | gauge: gv.gauge, 87 | value: math.NaN(), 88 | } 89 | } 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /kglb/utils/fwmark/fwmark_manager.go: -------------------------------------------------------------------------------- 1 | package fwmark 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | ) 7 | 8 | type fwmarkMdata struct { 9 | fwmark uint32 10 | refcount uint32 11 | } 12 | 13 | // Manager handles all fwmark allocation related routines. 14 | type Manager struct { 15 | maxFwmark uint32 16 | base uint32 17 | 18 | fwmarkLock sync.RWMutex 19 | fwmarksPool []uint32 20 | allocated map[string]*fwmarkMdata 21 | } 22 | 23 | // NewManager returns new Manager object w/ maximum specified 24 | // unique ids for fwmarks started from base value 25 | func NewManager(maxFwmark uint32, base uint32) *Manager { 26 | manager := &Manager{ 27 | fwmarksPool: make([]uint32, 0, maxFwmark), 28 | allocated: make(map[string]*fwmarkMdata), 29 | maxFwmark: maxFwmark, 30 | base: base, 31 | } 32 | for i := base; i < base+maxFwmark; i++ { 33 | manager.fwmarksPool = append(manager.fwmarksPool, i) 34 | } 35 | return manager 36 | } 37 | 38 | // AllocateFwmark returns next unused fwmark 39 | func (m *Manager) AllocateFwmark(address string) (uint32, error) { 40 | m.fwmarkLock.Lock() 41 | defer m.fwmarkLock.Unlock() 42 | 43 | if v, exists := m.allocated[address]; exists { 44 | v.refcount++ 45 | return v.fwmark, nil 46 | } 47 | 48 | if len(m.fwmarksPool) == 0 { 49 | return 0, fmt.Errorf("no more unused fwmarks") 50 | } 51 | fwmark := m.fwmarksPool[0] 52 | m.fwmarksPool = m.fwmarksPool[1:] 53 | m.allocated[address] = &fwmarkMdata{ 54 | fwmark: fwmark, 55 | refcount: 1, 56 | } 57 | return fwmark, nil 58 | } 59 | 60 | // GetAllocatedFwmark returns fwmark for address if it was already allocated 61 | // (and hence wont bump refcount) or error if it was not allocated prior 62 | func (m *Manager) GetAllocatedFwmark(address string) (uint32, error) { 63 | m.fwmarkLock.RLock() 64 | defer m.fwmarkLock.RUnlock() 65 | 66 | if v, exists := m.allocated[address]; exists { 67 | return v.fwmark, nil 68 | } 69 | return 0, fmt.Errorf("no fwmark allocated for address: %v", address) 70 | } 71 | 72 | // ReleaseFwmark returns unused fwmark into the pool 73 | func (m *Manager) ReleaseFwmark(address string) error { 74 | m.fwmarkLock.Lock() 75 | defer m.fwmarkLock.Unlock() 76 | 77 | if v, exists := m.allocated[address]; exists { 78 | v.refcount-- 79 | if v.refcount == 0 { 80 | m.fwmarksPool = append(m.fwmarksPool, v.fwmark) 81 | delete(m.allocated, address) 82 | } 83 | return nil 84 | } 85 | return fmt.Errorf("no allocated fwmark for address %v", address) 86 | } 87 | -------------------------------------------------------------------------------- /kglb/common/data_types_test.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | . "gopkg.in/check.v1" 5 | 6 | kglb_pb "dropbox/proto/kglb" 7 | ) 8 | 9 | type DataTypesSuite struct{} 10 | 11 | var _ = Suite(&DataTypesSuite{}) 12 | 13 | func (s *DataTypesSuite) TestKglbAddrToAddressFamily(c *C) { 14 | c.Assert(KglbAddrToAddressFamily(&kglb_pb.IP{ 15 | Address: &kglb_pb.IP_Ipv4{Ipv4: "172.0.0.1"}}), Equals, "v4") 16 | c.Assert(KglbAddrToAddressFamily( 17 | &kglb_pb.IP{Address: &kglb_pb.IP_Ipv6{Ipv6: "::1"}}), Equals, "v6") 18 | } 19 | 20 | func (s *DataTypesSuite) TestGetKeyFromLbService(c *C) { 21 | key, err := 22 | GetKeyFromLbService(&kglb_pb.LoadBalancerService{Service: &kglb_pb.LoadBalancerService_IpvsService{ 23 | IpvsService: &kglb_pb.IpvsService{ 24 | Attributes: &kglb_pb.IpvsService_TcpAttributes{ 25 | TcpAttributes: &kglb_pb.IpvsTcpAttributes{ 26 | Address: &kglb_pb.IP{Address: &kglb_pb.IP_Ipv4{Ipv4: "172.0.0.1"}}, 27 | Port: 443, 28 | }}, 29 | Scheduler: kglb_pb.IpvsService_RR, 30 | }}}) 31 | c.Assert(err, IsNil) 32 | c.Assert(key, Equals, "172.0.0.1:443-tcp") 33 | 34 | key, err = 35 | GetKeyFromLbService(&kglb_pb.LoadBalancerService{Service: &kglb_pb.LoadBalancerService_IpvsService{ 36 | IpvsService: &kglb_pb.IpvsService{ 37 | Attributes: &kglb_pb.IpvsService_TcpAttributes{ 38 | TcpAttributes: &kglb_pb.IpvsTcpAttributes{ 39 | Address: &kglb_pb.IP{Address: &kglb_pb.IP_Ipv6{Ipv6: "::1"}}, 40 | Port: 443, 41 | }}, 42 | Scheduler: kglb_pb.IpvsService_RR, 43 | }}}) 44 | c.Assert(err, IsNil) 45 | c.Assert(key, Equals, "::1:443-tcp") 46 | 47 | key, err = 48 | GetKeyFromLbService(&kglb_pb.LoadBalancerService{Service: &kglb_pb.LoadBalancerService_IpvsService{ 49 | IpvsService: &kglb_pb.IpvsService{ 50 | Attributes: &kglb_pb.IpvsService_UdpAttributes{ 51 | UdpAttributes: &kglb_pb.IpvsUdpAttributes{ 52 | Address: &kglb_pb.IP{ 53 | Address: &kglb_pb.IP_Ipv4{ 54 | Ipv4: "127.0.0.1", 55 | }, 56 | }, 57 | Port: 80, 58 | }}, 59 | Scheduler: kglb_pb.IpvsService_RR, 60 | }}}) 61 | c.Assert(err, IsNil) 62 | c.Assert(key, Equals, "127.0.0.1:80-udp") 63 | 64 | key, err = 65 | GetKeyFromLbService(&kglb_pb.LoadBalancerService{Service: &kglb_pb.LoadBalancerService_IpvsService{ 66 | IpvsService: &kglb_pb.IpvsService{ 67 | Attributes: &kglb_pb.IpvsService_FwmarkAttributes{ 68 | FwmarkAttributes: &kglb_pb.IpvsFwmarkAttributes{ 69 | AddressFamily: kglb_pb.AddressFamily_AF_INET, 70 | Fwmark: 1010, 71 | }}, 72 | Scheduler: kglb_pb.IpvsService_RR, 73 | }}}) 74 | c.Assert(err, IsNil) 75 | c.Assert(key, Equals, "fwmark:1010:v4") 76 | } 77 | -------------------------------------------------------------------------------- /kglb/utils/discovery/host_ports_test.go: -------------------------------------------------------------------------------- 1 | package discovery 2 | 3 | import ( 4 | . "gopkg.in/check.v1" 5 | 6 | . "godropbox/gocheck2" 7 | ) 8 | 9 | type HostPortSuite struct{} 10 | 11 | var _ = Suite(&HostPortSuite{}) 12 | 13 | func (s *HostPortSuite) TestHostPortEqual(c *C) { 14 | c.Assert( 15 | NewHostPort("host1", 80, true).Equal(NewHostPort("host1", 80, true)), IsTrue) 16 | c.Assert( 17 | NewHostPort("host1", 80, true).Equal(NewHostPort("host2", 80, true)), IsFalse) 18 | c.Assert( 19 | NewHostPort("host1", 80, true).Equal(NewHostPort("host1", 82, true)), IsFalse) 20 | } 21 | 22 | func (s *HostPortSuite) TestDiscoveryStateEqual(c *C) { 23 | c.Assert(DiscoveryState([]*HostPort{ 24 | NewHostPort("host1", 80, true), 25 | NewHostPort("host2", 80, true), 26 | }).Equal(DiscoveryState([]*HostPort{ 27 | NewHostPort("host1", 80, true), 28 | NewHostPort("host2", 80, true), 29 | })), IsTrue) 30 | 31 | // check reorder. 32 | c.Assert(DiscoveryState([]*HostPort{ 33 | NewHostPort("host1", 80, true), 34 | NewHostPort("host2", 80, true), 35 | }).Equal(DiscoveryState([]*HostPort{ 36 | NewHostPort("host2", 80, true), 37 | NewHostPort("host1", 80, true), 38 | })), IsTrue) 39 | 40 | // different port with the same name. 41 | c.Assert(DiscoveryState([]*HostPort{ 42 | NewHostPort("host1", 80, true), 43 | NewHostPort("host2", 80, true), 44 | }).Equal(DiscoveryState([]*HostPort{ 45 | NewHostPort("host1", 80, true), 46 | NewHostPort("host2", 81, true), 47 | })), IsFalse) 48 | 49 | // differente size. 50 | c.Assert(DiscoveryState([]*HostPort{ 51 | NewHostPort("host1", 80, true), 52 | NewHostPort("host2", 80, true), 53 | }).Equal(DiscoveryState([]*HostPort{ 54 | NewHostPort("host1", 80, true), 55 | NewHostPort("host2", 80, true), 56 | NewHostPort("host3", 80, true), 57 | })), IsFalse) 58 | c.Assert(DiscoveryState([]*HostPort{ 59 | NewHostPort("host1", 80, true), 60 | NewHostPort("host2", 80, true), 61 | NewHostPort("host3", 80, true), 62 | }).Equal(DiscoveryState([]*HostPort{ 63 | NewHostPort("host1", 80, true), 64 | NewHostPort("host2", 80, true), 65 | })), IsFalse) 66 | } 67 | 68 | func (s *HostPortSuite) TestContains(c *C) { 69 | // true conditions. 70 | c.Assert(DiscoveryState([]*HostPort{ 71 | NewHostPort("host1", 80, true), 72 | NewHostPort("host2", 80, true), 73 | }).Contains(NewHostPort("host2", 80, true)), IsTrue) 74 | 75 | // false conditions. 76 | c.Assert(DiscoveryState([]*HostPort{ 77 | NewHostPort("host1", 80, true), 78 | NewHostPort("host2", 80, true), 79 | }).Contains(NewHostPort("host2", 81, true)), IsFalse) 80 | c.Assert(DiscoveryState([]*HostPort{ 81 | NewHostPort("host1", 80, true), 82 | NewHostPort("host2", 80, true), 83 | }).Contains(NewHostPort("host3", 80, true)), IsFalse) 84 | 85 | } 86 | -------------------------------------------------------------------------------- /kglb/data_plane/manager_balancer_test.go: -------------------------------------------------------------------------------- 1 | package data_plane 2 | 3 | import ( 4 | . "gopkg.in/check.v1" 5 | 6 | kglb_pb "dropbox/proto/kglb" 7 | . "godropbox/gocheck2" 8 | ) 9 | 10 | type BalancerManagerSuite struct { 11 | // Mock IPVS module. 12 | mockIpvs *MockIpvsModule 13 | // BalancerManager. 14 | manager *BalancerManager 15 | } 16 | 17 | var _ = Suite(&BalancerManagerSuite{}) 18 | 19 | func (m *BalancerManagerSuite) SetUpTest(c *C) { 20 | m.mockIpvs = &MockIpvsModule{} 21 | 22 | resolver, err := NewResolver() 23 | c.Assert(err, IsNil) 24 | 25 | params := BalancerManagerParams{ 26 | Ipvs: m.mockIpvs, 27 | Resolver: resolver, 28 | } 29 | 30 | m.manager, err = NewBalancerManager(params) 31 | c.Assert(err, IsNil) 32 | } 33 | 34 | // Test AddBalancer. 35 | func (m *BalancerManagerSuite) TestAddBalancer(c *C) { 36 | // 1. empty list of services. 37 | m.mockIpvs.ListServicesFunc = func() ([]*kglb_pb.IpvsService, []*kglb_pb.Stats, error) { 38 | return []*kglb_pb.IpvsService{}, []*kglb_pb.Stats{}, nil 39 | } 40 | m.mockIpvs.AddServiceFunc = func(service *kglb_pb.IpvsService) error { 41 | c.Assert( 42 | service, 43 | DeepEqualsPretty, 44 | &kglb_pb.IpvsService{ 45 | Attributes: &kglb_pb.IpvsService_TcpAttributes{ 46 | TcpAttributes: &kglb_pb.IpvsTcpAttributes{ 47 | Address: &kglb_pb.IP{Address: &kglb_pb.IP_Ipv4{Ipv4: "172.0.0.1"}}, 48 | Port: 443, 49 | }}, 50 | Scheduler: kglb_pb.IpvsService_RR, 51 | }) 52 | return nil 53 | } 54 | 55 | m.mockIpvs.AddRealServerFunc = func(service *kglb_pb.IpvsService, 56 | dst *kglb_pb.UpstreamState) error { 57 | 58 | // expecting 10.0.0.4 to be added only. 59 | c.Assert(dst, DeepEqualsPretty, &kglb_pb.UpstreamState{ 60 | Address: &kglb_pb.IP{Address: &kglb_pb.IP_Ipv4{Ipv4: "10.0.0.1"}}, 61 | Port: 443, 62 | Hostname: "hostname1", 63 | Weight: 50, 64 | ForwardMethod: kglb_pb.ForwardMethods_TUNNEL, 65 | }) 66 | 67 | return nil 68 | } 69 | 70 | err := m.manager.AddBalancer(&kglb_pb.BalancerState{ 71 | Name: "TestName1", 72 | LbService: &kglb_pb.LoadBalancerService{Service: &kglb_pb.LoadBalancerService_IpvsService{ 73 | IpvsService: &kglb_pb.IpvsService{ 74 | Attributes: &kglb_pb.IpvsService_TcpAttributes{ 75 | TcpAttributes: &kglb_pb.IpvsTcpAttributes{ 76 | Address: &kglb_pb.IP{Address: &kglb_pb.IP_Ipv4{Ipv4: "172.0.0.1"}}, 77 | Port: 443, 78 | }}, 79 | Scheduler: kglb_pb.IpvsService_RR, 80 | }}}, 81 | Upstreams: []*kglb_pb.UpstreamState{ 82 | { 83 | Address: &kglb_pb.IP{Address: &kglb_pb.IP_Ipv4{Ipv4: "10.0.0.1"}}, 84 | Port: 443, 85 | Hostname: "hostname1", 86 | Weight: 50, 87 | ForwardMethod: kglb_pb.ForwardMethods_TUNNEL, 88 | }, 89 | }, 90 | }) 91 | c.Assert(err, IsNil) 92 | } 93 | -------------------------------------------------------------------------------- /kglb/control_plane/all_test.go: -------------------------------------------------------------------------------- 1 | package control_plane 2 | 3 | import ( 4 | "net" 5 | "net/http" 6 | "net/http/httptest" 7 | "strconv" 8 | "testing" 9 | 10 | . "gopkg.in/check.v1" 11 | 12 | kglb_pb "dropbox/proto/kglb" 13 | ) 14 | 15 | // Bootstrap gocheck for this package. 16 | func Test(t *testing.T) { 17 | TestingT(t) 18 | } 19 | 20 | type BackendHandler struct { 21 | handler func(writer http.ResponseWriter, request *http.Request) 22 | } 23 | 24 | func (b *BackendHandler) ServeHTTP( 25 | writer http.ResponseWriter, 26 | request *http.Request) { 27 | 28 | b.handler(writer, request) 29 | } 30 | 31 | // Create http test server and returns its address and port 32 | func NewBackend( 33 | c *C, 34 | handler func(writer http.ResponseWriter, 35 | request *http.Request)) (*httptest.Server, string, int) { 36 | 37 | server := httptest.NewServer(&BackendHandler{ 38 | handler: handler, 39 | }) 40 | 41 | backendHost, backendPortStr, err := net.SplitHostPort( 42 | server.Listener.Addr().String()) 43 | c.Assert(err, IsNil) 44 | backendPort, err := strconv.Atoi(backendPortStr) 45 | c.Assert(err, IsNil) 46 | c.Log("created backend: ", server.Listener.Addr().String()) 47 | 48 | return server, backendHost, backendPort 49 | } 50 | 51 | func NewBackendWithCustomAddr( 52 | c *C, 53 | address string, 54 | handler func(writer http.ResponseWriter, 55 | request *http.Request)) *http.Server { 56 | 57 | srv := &http.Server{ 58 | Addr: address, 59 | Handler: &BackendHandler{ 60 | handler: handler, 61 | }, 62 | } 63 | go func() { 64 | srv.ListenAndServe() 65 | }() 66 | 67 | return srv 68 | } 69 | 70 | // Create tcp listener with custom handler called after accepting conn. 71 | func NewTcpBackend(c *C, handler func(conn net.Conn)) (net.Listener, string, int) { 72 | 73 | lAddr, err := net.ResolveTCPAddr("tcp", "127.0.0.1:0") 74 | c.Assert(err, IsNil) 75 | 76 | listener, err := net.ListenTCP("tcp", lAddr) 77 | c.Assert(err, IsNil) 78 | 79 | // getting port 80 | addr := listener.Addr() 81 | host, portStr, err := net.SplitHostPort(addr.String()) 82 | c.Assert(err, IsNil) 83 | port, err := strconv.Atoi(portStr) 84 | c.Assert(err, IsNil) 85 | 86 | go func() { 87 | conn, err := listener.Accept() 88 | if err != nil { 89 | return 90 | } 91 | handler(conn) 92 | }() 93 | 94 | return listener, host, port 95 | } 96 | 97 | // Mock config loader. 98 | type mockConfigLoader struct { 99 | configChan chan interface{} 100 | } 101 | 102 | func newMockConfigLoader() *mockConfigLoader { 103 | return &mockConfigLoader{ 104 | configChan: make(chan interface{}, 1), 105 | } 106 | } 107 | 108 | func (c *mockConfigLoader) UpdateConfig(config *kglb_pb.ControlPlaneConfig) { 109 | c.configChan <- config 110 | } 111 | 112 | func (c *mockConfigLoader) Updates() <-chan interface{} { 113 | return c.configChan 114 | } 115 | 116 | func (c *mockConfigLoader) Stop() { 117 | close(c.configChan) 118 | } 119 | -------------------------------------------------------------------------------- /kglb/utils/health_checker/dns_checker.go: -------------------------------------------------------------------------------- 1 | package health_checker 2 | 3 | import ( 4 | "context" 5 | "net" 6 | "strconv" 7 | "strings" 8 | "time" 9 | 10 | "github.com/miekg/dns" 11 | 12 | hc_pb "dropbox/proto/kglb/healthchecker" 13 | "godropbox/errors" 14 | ) 15 | 16 | var ( 17 | // list of support dns query types. 18 | queryTypes = map[hc_pb.DnsCheckerAttributes_DNSQueryType]uint16{ 19 | hc_pb.DnsCheckerAttributes_A: dns.TypeA, 20 | // TODO(oleg): keeping old behaviour, why AAAA->A? 21 | hc_pb.DnsCheckerAttributes_AAAA: dns.TypeA, 22 | hc_pb.DnsCheckerAttributes_NS: dns.TypeNS, 23 | hc_pb.DnsCheckerAttributes_MX: dns.TypeMX, 24 | } 25 | ) 26 | 27 | var _ HealthChecker = &DnsChecker{} 28 | 29 | // dns health checker. 30 | type DnsChecker struct { 31 | params *hc_pb.DnsCheckerAttributes 32 | 33 | // hostname and pid used as part of probe message sent to syslog daemon. 34 | dialContext DialContextFunc 35 | } 36 | 37 | func NewDnsChecker(params *hc_pb.DnsCheckerAttributes, dialContext DialContextFunc) (*DnsChecker, error) { 38 | checker := &DnsChecker{ 39 | params: params, 40 | dialContext: defaultDialContext, 41 | } 42 | 43 | if dialContext != nil { 44 | checker.dialContext = dialContext 45 | } 46 | 47 | if params.GetQueryString() == "" { 48 | return nil, errors.Newf("QueryString is required: %+v", params) 49 | } 50 | 51 | if _, ok := queryTypes[params.GetQueryType()]; !ok { 52 | return nil, errors.Newf("Unknown QueryType: %+v", params) 53 | } 54 | 55 | return checker, nil 56 | } 57 | 58 | func (h *DnsChecker) GetConfiguration() *hc_pb.HealthCheckerAttributes { 59 | return &hc_pb.HealthCheckerAttributes{ 60 | Attributes: &hc_pb.HealthCheckerAttributes_Dns{ 61 | Dns: h.params, 62 | }, 63 | } 64 | } 65 | 66 | func (h *DnsChecker) Check(host string, port int) error { 67 | timeout := timeoutMsToDuration(h.params.GetCheckTimeoutMs()) 68 | ctx, cancel := context.WithTimeout(context.Background(), timeout) 69 | defer cancel() 70 | 71 | address := net.JoinHostPort(host, strconv.Itoa(port)) 72 | conn, err := h.dialContext(ctx, strings.ToLower(h.params.GetProtocol().String()), address) 73 | 74 | if err != nil { 75 | return err 76 | } 77 | defer func() { 78 | _ = conn.Close() 79 | }() 80 | _ = conn.SetDeadline(time.Now().Add(timeout)) 81 | 82 | dnsConn := &dns.Conn{Conn: conn} 83 | msgReq := &dns.Msg{} 84 | msgReq.SetQuestion(h.params.GetQueryString(), queryTypes[h.params.GetQueryType()]) 85 | 86 | // requesting.. 87 | if err = dnsConn.WriteMsg(msgReq); err != nil { 88 | return err 89 | } 90 | // handling response. 91 | msgRecv, err := dnsConn.ReadMsg() 92 | if err != nil { 93 | return err 94 | } else if msgRecv.Id != msgReq.Id { 95 | return errors.Newf( 96 | "message id in response doesn't match requested: %d != %d", 97 | msgRecv.Id, 98 | msgReq.Id) 99 | } 100 | 101 | // finally checking code. 102 | if msgRecv.Rcode != int(h.params.GetRcode()) { 103 | return errors.Newf("incorrect rcode: %d != %d", msgRecv.Rcode, h.params.GetRcode()) 104 | } 105 | 106 | return nil 107 | } 108 | -------------------------------------------------------------------------------- /kglb/control_plane/discovery_factory.go: -------------------------------------------------------------------------------- 1 | package control_plane 2 | 3 | import ( 4 | "fmt" 5 | 6 | "dropbox/kglb/utils/discovery" 7 | pb "dropbox/proto/kglb" 8 | "godropbox/errors" 9 | ) 10 | 11 | var ( 12 | ErrResolverIncompatibleType = fmt.Errorf("incompatible resolver type") 13 | ) 14 | 15 | // Interface to create and update DiscoveryResolver instance based on 16 | // DiscoveryResolver proto. 17 | type DiscoveryFactory interface { 18 | // Returns configured instance of DiscoveryResolver based on 19 | // UpstreamDiscovery proto. 20 | // TODO(oleg): propagate context to resolver 21 | Resolver( 22 | name string, 23 | setupName string, 24 | discoveryConf *pb.UpstreamDiscovery) (discovery.DiscoveryResolver, error) 25 | 26 | // Updates Resolver config or returns error when it fails for any reason. 27 | // errResolverIncompatibleType will be returned when type of resolver 28 | // instance doesn't match new config. 29 | Update( 30 | resolver discovery.DiscoveryResolver, 31 | conf *pb.UpstreamDiscovery) error 32 | } 33 | 34 | type BaseDiscoveryFactory struct{} 35 | 36 | func NewDiscoveryFactory() *BaseDiscoveryFactory { 37 | 38 | return &BaseDiscoveryFactory{} 39 | } 40 | 41 | // Returns Discovery Resolver instance based on configuration. 42 | func (f *BaseDiscoveryFactory) Resolver( 43 | name string, 44 | setupName string, 45 | conf *pb.UpstreamDiscovery) (discovery.DiscoveryResolver, error) { 46 | 47 | switch attr := conf.Attributes.(type) { 48 | case *pb.UpstreamDiscovery_StaticAttributes: 49 | port := int(conf.Port) 50 | hostPorts := make( 51 | []*discovery.HostPort, 52 | len(attr.StaticAttributes.Hosts)) 53 | for i, host := range attr.StaticAttributes.Hosts { 54 | hostPorts[i] = discovery.NewHostPort(host, port, true) 55 | } 56 | params := discovery.StaticResolverParams{ 57 | Id: fmt.Sprintf("%s/static", name), // Resolver Id. 58 | Hosts: hostPorts, // Initial state. 59 | SetupName: setupName, 60 | ServiceName: name, 61 | } 62 | 63 | return discovery.NewStaticResolver(params) 64 | default: 65 | return nil, errors.Newf( 66 | "DiscoverResolver is not implemented for %s", 67 | attr) 68 | } 69 | } 70 | 71 | // Updates Resolver config or returns error when it fails. 72 | func (f *BaseDiscoveryFactory) Update( 73 | resolver discovery.DiscoveryResolver, 74 | conf *pb.UpstreamDiscovery) error { 75 | 76 | switch attr := conf.Attributes.(type) { 77 | case *pb.UpstreamDiscovery_StaticAttributes: 78 | staticResolver, ok := resolver.(*discovery.StaticResolver) 79 | if !ok { 80 | return ErrResolverIncompatibleType 81 | } 82 | 83 | port := int(conf.Port) 84 | hostPorts := make( 85 | []*discovery.HostPort, 86 | len(attr.StaticAttributes.Hosts)) 87 | for i, host := range attr.StaticAttributes.Hosts { 88 | hostPorts[i] = discovery.NewHostPort(host, port, true) 89 | } 90 | 91 | staticResolver.Update(hostPorts) 92 | return nil 93 | default: 94 | return errors.Newf( 95 | "DiscoverResolver is not implemented for %s", 96 | attr) 97 | } 98 | } 99 | 100 | var _ DiscoveryFactory = &BaseDiscoveryFactory{} 101 | -------------------------------------------------------------------------------- /proto/dropbox/proto/kglb/common.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package kglb; 4 | 5 | enum ForwardMethods { 6 | TUNNEL = 0; 7 | MASQ = 1; 8 | } 9 | 10 | enum AddressFamily { 11 | AF_INET = 0; 12 | AF_INET6 = 1; 13 | } 14 | 15 | message IP { 16 | oneof address { 17 | string ipv4 = 1; 18 | string ipv6 = 2; 19 | } 20 | } 21 | 22 | enum IPProtocol { 23 | TCP = 0; 24 | UDP = 1; 25 | } 26 | 27 | // Service/Upstream stats. 28 | message Stats { 29 | uint64 connections_count = 1; // number of connections. 30 | uint64 packets_in_count = 2; // number of incoming packets. 31 | uint64 packets_out_count = 3; // number of outgoing packets. 32 | uint64 bytes_in_count = 4; // incoming bytes. 33 | uint64 bytes_out_count = 5; // outgoing bytes. 34 | uint64 connections_rate = 6; // connection rate. 35 | uint64 packets_in_rate = 7; // packets in rate. 36 | uint64 packets_out_rate = 8; // packets out rate. 37 | uint64 bytes_in_rate = 9; // inbound bytes rate. 38 | uint64 bytes_out_rate = 10; // outbound bytes rate. 39 | 40 | uint64 active_conns_count = 11; // number of active connections. 41 | uint64 inact_conns_count = 12; // number of active connections. 42 | uint64 persist_conns_count = 13; // number of persistent connections. 43 | } 44 | 45 | message UpstreamState { 46 | IP address = 1; 47 | uint32 port = 2; 48 | string hostname = 3; 49 | uint32 weight = 4; 50 | ForwardMethods forward_method = 5; 51 | } 52 | 53 | message LoadBalancerService { 54 | oneof service { 55 | IpvsService ipvs_service = 1; 56 | } 57 | } 58 | 59 | message BalancerState { 60 | string name = 1; 61 | LoadBalancerService lb_service = 2; 62 | repeated UpstreamState upstreams = 3; 63 | } 64 | 65 | message DynamicRoute { 66 | oneof attributes { 67 | BgpRouteAttributes bgp_attributes = 10; 68 | } 69 | } 70 | 71 | message IpvsFwmarkAttributes { 72 | AddressFamily address_family = 1; 73 | uint32 fwmark = 2; 74 | } 75 | 76 | message IpvsTcpAttributes { 77 | IP address = 1; 78 | uint32 port = 2; 79 | } 80 | 81 | message IpvsUdpAttributes { 82 | IP address = 1; 83 | uint32 port = 2; 84 | } 85 | 86 | message IpvsService { 87 | enum Flag { 88 | EMPTY = 0; 89 | PERSISTENT = 1; 90 | HASHED = 2; 91 | ONEPACKET = 3; 92 | SCHED1 = 4; 93 | SCHED2 = 5; 94 | SCHED3 = 6; 95 | SH_FALLBACK = 7; 96 | SH_PORT = 8; 97 | SH_NOPORT = 9; 98 | } 99 | 100 | enum Scheduler { 101 | RR = 0; 102 | WRR = 1; 103 | IP_VS_SCH = 2; 104 | } 105 | 106 | oneof attributes { 107 | IpvsTcpAttributes tcp_attributes = 1; 108 | IpvsUdpAttributes udp_attributes = 2; 109 | IpvsFwmarkAttributes fwmark_attributes = 3; 110 | } 111 | 112 | Scheduler scheduler = 4; 113 | repeated Flag flags = 5; 114 | } 115 | 116 | message BgpRouteAttributes { 117 | uint32 local_asn = 1; 118 | uint32 peer_asn = 2; 119 | string community = 3; 120 | 121 | IP prefix = 5; 122 | uint32 prefixlen = 6; 123 | 124 | // Delay after withdrawing bgp route. Zero means no delay. 125 | uint32 hold_time_ms = 7; 126 | } 127 | 128 | message LinkAddress { 129 | string link_name = 1; 130 | IP address = 2; 131 | } 132 | 133 | -------------------------------------------------------------------------------- /kglb/data_plane/ipvs_mqliang_types_test.go: -------------------------------------------------------------------------------- 1 | package data_plane 2 | 3 | import ( 4 | "github.com/mqliang/libipvs" 5 | . "gopkg.in/check.v1" 6 | 7 | kglb_pb "dropbox/proto/kglb" 8 | . "godropbox/gocheck2" 9 | ) 10 | 11 | type ConvertTypesTestSuite struct{} 12 | 13 | var _ = Suite(&ConvertTypesTestSuite{}) 14 | 15 | func (ts *ConvertTypesTestSuite) TestKglbFlags(c *C) { 16 | // flags not set 17 | flags, err := toLibIpvsFlags(nil) 18 | c.Assert(err, NoErr) 19 | c.Assert(flags, DeepEqualsPretty, libipvs.Flags{Mask: ^uint32(0)}) 20 | 21 | // single flag 22 | flags, err = toLibIpvsFlags([]kglb_pb.IpvsService_Flag{kglb_pb.IpvsService_SH_PORT}) 23 | c.Assert(err, NoErr) 24 | c.Assert(flags, DeepEqualsPretty, 25 | libipvs.Flags{ 26 | Flags: libipvs.IP_VS_SVC_F_SCHED_SH_PORT, 27 | Mask: ^uint32(0), 28 | }) 29 | 30 | // multiple flags 31 | flags, err = toLibIpvsFlags( 32 | []kglb_pb.IpvsService_Flag{ 33 | kglb_pb.IpvsService_SH_PORT, 34 | kglb_pb.IpvsService_SH_FALLBACK, 35 | }) 36 | c.Assert(err, NoErr) 37 | c.Assert(flags, DeepEqualsPretty, 38 | libipvs.Flags{ 39 | Flags: libipvs.IP_VS_SVC_F_SCHED_SH_PORT | libipvs.IP_VS_SVC_F_SCHED_SH_FALLBACK, 40 | Mask: ^uint32(0), 41 | }) 42 | 43 | // unsupported flag (IpvsService_EMPTY) 44 | _, err = toLibIpvsFlags([]kglb_pb.IpvsService_Flag{kglb_pb.IpvsService_EMPTY}) 45 | c.Assert(err, MultilineErrorMatches, "Unsupported kglb flag: EMPTY") 46 | _, err = toLibIpvsFlags([]kglb_pb.IpvsService_Flag{kglb_pb.IpvsService_SH_PORT, kglb_pb.IpvsService_EMPTY}) 47 | c.Assert(err, MultilineErrorMatches, "Unsupported kglb flag: EMPTY") 48 | } 49 | 50 | func (ts *ConvertTypesTestSuite) TestlibIpvsFlags(c *C) { 51 | // mask zero value 52 | _, err := toKglbFlags(libipvs.Flags{}) 53 | c.Assert(err, MultilineErrorMatches, "Unsupported mask: 0") 54 | 55 | // incorrect mask 56 | _, err = toKglbFlags(libipvs.Flags{Mask: 10}) 57 | c.Assert(err, MultilineErrorMatches, "Unsupported mask: 10") 58 | 59 | // no flags 60 | flags, err := toKglbFlags(libipvs.Flags{Mask: ^uint32(0)}) 61 | c.Assert(err, NoErr) 62 | c.Assert(flags, DeepEqualsPretty, []kglb_pb.IpvsService_Flag{}) 63 | 64 | // single flag 65 | flags, err = toKglbFlags( 66 | libipvs.Flags{ 67 | Flags: libipvs.IP_VS_SVC_F_SCHED_SH_PORT, 68 | Mask: ^uint32(0), 69 | }) 70 | c.Assert(err, NoErr) 71 | c.Assert(flags, DeepEqualsPretty, []kglb_pb.IpvsService_Flag{kglb_pb.IpvsService_SH_PORT}) 72 | 73 | // multiple flags 74 | flags, err = toKglbFlags( 75 | libipvs.Flags{ 76 | Flags: libipvs.IP_VS_SVC_F_SCHED_SH_PORT | libipvs.IP_VS_SVC_F_SCHED_SH_FALLBACK, 77 | Mask: ^uint32(0), 78 | }) 79 | c.Assert(err, NoErr) 80 | c.Assert(flags, DeepEqualsPretty, []kglb_pb.IpvsService_Flag{ 81 | kglb_pb.IpvsService_SH_FALLBACK, 82 | kglb_pb.IpvsService_SH_PORT, 83 | }) 84 | 85 | // unknown flag 86 | _, err = toKglbFlags( 87 | libipvs.Flags{ 88 | Flags: 64, 89 | Mask: ^uint32(0), 90 | }) 91 | c.Assert(err, MultilineErrorMatches, "Unsupported flags left: 64") 92 | 93 | // flag combination 94 | _, err = toKglbFlags( 95 | libipvs.Flags{ 96 | Flags: libipvs.IP_VS_SVC_F_SCHED_SH_PORT | 128, 97 | Mask: ^uint32(0), 98 | }) 99 | c.Assert(err, MultilineErrorMatches, "Unsupported flags left: 128") 100 | } 101 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # KgLb ![Build](https://github.com/piscesdk/kglb/workflows/Build/badge.svg) ![Test](https://github.com/piscesdk/kglb/workflows/Test/badge.svg) ![Lint](https://github.com/piscesdk/kglb/workflows/Lint/badge.svg) 2 | 3 | KgLb is L4 a load balancer built on top of linux ip virtual server (ip_vs). 4 | 5 | ![KgLb image](doc/kglb.png) 6 | 7 | It provides rich functionality such as discovery, health checks for real servers, automatically update their weights based on status, and apply all required changes to make ipvs alive including registering virtual services, updating reals and adding required vip addresses. 8 | 9 | ## Overview 10 | KgLb consists of two key components called Control Plane and Data Plane. Both components are written as libraries and based on a few modules which follow predefined interfaces. Next picture shows high-level overview of KgLb internals. 11 | ![design](doc/kglb_open_arch.jpg) 12 | 13 | * Control Plane is a library which discovers, health checks real servers, and generates data plane state: 14 | * Services is a library which creates set of Balancers according received configuration, generates data plane state and applies it via DataPlaneClient interface. 15 | * Balancer discovers, health checks and generate single or multiple BalancerState which represents single ipvs service. Balancer may generate extra fwmark states when health checking via fwmark is enabled. 16 | * StateGenerator generates complete Data Plane state based on ControlPlane config and generated Balancers. 17 | * DiscoveryFactory is an interface to create appropriate discovery instance based on configuration. Open version supports statis discovery method only (pre-defined set of hosts provided in config). 18 | * HealthCheckerFactory is an interface to create required health checking instance instane. Currently supported checks are: http including http proxy, tcp, dns, syslog. 19 | * DataPlaneClient provides communication interface with DataPlane. Current imlementation of DataPlaneClient in kglbd consists of simple API call of data plane, but it might provides grpc or rest bridge when control plane and data plane are separate services. 20 | * Data Plane is a library which represents middle layer between control pland and multiple system components, and makes system changes based on received data plane state. Today Data Plane can do following: 21 | * add/delete ip address. 22 | * interact with ip_vs module (ipvs service creation/deletion + updating real servers). 23 | 24 | ## Requirements 25 | - Go 1.13 26 | - Linux Kernel 4.4+ 27 | - protoc 3.6.1+ and protoc-gen-go 28 | 29 | ## Supported features 30 | - Discovery: static only. 31 | - Health Checkers: http, dns, syslog, tcp. 32 | - Tunneled health checking through fwmarks. 33 | - Stats exported in prometheus format and available on http://127.0.0.1:5678/stats by default. 34 | - Graceful shutdown. 35 | 36 | ## Installation 37 | ```bash 38 | # Compile protobufs 39 | pushd ./proto 40 | protoc --go_out=. ./dropbox/proto/kglb/healthchecker/healthchecker.proto 41 | protoc --go_out=. ./dropbox/proto/kglb/*.proto 42 | popd 43 | 44 | # Compiling 45 | go build -o kglbd.bin ./kglbd 46 | ``` 47 | 48 | ## Quick start 49 | ```bash 50 | sudo ./kglbd.bin -config=./examples/example_config.json -logtostderr 51 | ``` 52 | 53 | ## Next Steps 54 | - Releasing bgp module. 55 | - Integration with ![Katran](https://github.com/facebookincubator/katran). 56 | -------------------------------------------------------------------------------- /kglb/utils/comparable/arrays.go: -------------------------------------------------------------------------------- 1 | package comparable 2 | 3 | // Construct key of the abstract element. 4 | type Keyable interface { 5 | // Construct key for the the item. 6 | Key(item interface{}) string 7 | } 8 | 9 | // Set of required function to perform comparison on top of array of abstract 10 | // elements. 11 | type Comparable interface { 12 | Keyable 13 | // Compare two items and returns true when they are equal, otherwise false. 14 | Equal(item1, item2 interface{}) bool 15 | } 16 | 17 | // implementation of Comparable interface with required func as callbacks to 18 | // simplify constructing Comparable impls on a fly. 19 | type ComparableImpl struct { 20 | KeyFunc func(item interface{}) string 21 | EqualFunc func(item1, item2 interface{}) bool 22 | } 23 | 24 | func (c *ComparableImpl) Key(item interface{}) string { 25 | return c.KeyFunc(item) 26 | } 27 | 28 | func (c *ComparableImpl) Equal(item1, item2 interface{}) bool { 29 | return c.EqualFunc(item1, item2) 30 | } 31 | 32 | // Pair of states of the item in the old and new states. 33 | type ChangedPair struct { 34 | OldItem interface{} 35 | NewItem interface{} 36 | } 37 | 38 | // Result of CompareArrays with explicit field naming instead of using multiple 39 | // return arguments which may be accidentally misplaces/misordered. 40 | type ComparableResult struct { 41 | Added []interface{} 42 | Deleted []interface{} 43 | Changed []ChangedPair 44 | Unchanged []interface{} 45 | } 46 | 47 | // Returns true when there is any change, otherwise false. 48 | func (c *ComparableResult) IsChanged() bool { 49 | return len(c.Added) != 0 || len(c.Deleted) != 0 || len(c.Changed) != 0 50 | } 51 | 52 | // return new state of changed elements. 53 | func (c *ComparableResult) NewChangedStates() []interface{} { 54 | result := make([]interface{}, len(c.Changed)) 55 | for i, val := range c.Changed { 56 | result[i] = val.NewItem 57 | } 58 | return result 59 | } 60 | 61 | // Compare to arrays of elements and returns multiple arrays with added, deleted, 62 | // changed and unmodified elements. 63 | func CompareArrays( 64 | oldSet, 65 | newSet []interface{}, 66 | comparable Comparable) *ComparableResult { 67 | 68 | result := &ComparableResult{} 69 | oldMap := keyGen(oldSet, comparable) 70 | newMap := keyGen(newSet, comparable) 71 | 72 | // 1. identify deleted, updated and unchanged elements. 73 | for key, oldItem := range oldMap { 74 | if newItem, ok := newMap[key]; !ok { 75 | result.Deleted = append(result.Deleted, oldItem) 76 | } else { 77 | if !comparable.Equal(oldItem, newItem) { 78 | result.Changed = append(result.Changed, ChangedPair{ 79 | OldItem: oldItem, 80 | NewItem: newItem, 81 | }) 82 | } else { 83 | result.Unchanged = append(result.Unchanged, oldItem) 84 | } 85 | } 86 | } 87 | 88 | // 2. identify new items. 89 | for key, newItem := range newMap { 90 | if _, ok := oldMap[key]; !ok { 91 | result.Added = append(result.Added, newItem) 92 | } 93 | } 94 | 95 | return result 96 | } 97 | 98 | // generate key for each item from array and return key -> item map. 99 | func keyGen(items []interface{}, keyable Keyable) map[string]interface{} { 100 | keyMap := make(map[string]interface{}, len(items)) 101 | for _, item := range items { 102 | key := keyable.Key(item) 103 | keyMap[key] = item 104 | } 105 | 106 | return keyMap 107 | } 108 | -------------------------------------------------------------------------------- /kglb/utils/health_manager/health_manager_state.go: -------------------------------------------------------------------------------- 1 | package health_manager 2 | 3 | import ( 4 | "fmt" 5 | 6 | "dropbox/kglb/utils/discovery" 7 | ) 8 | 9 | type HealthManagerEntry struct { 10 | HostPort *discovery.HostPort 11 | Status *healthStatusEntry 12 | Enabled bool 13 | } 14 | 15 | // Compares two HealthManagerEntry entries and returns true when both are equal, 16 | // otherwise false. 17 | func (h *HealthManagerEntry) Equal(entry *HealthManagerEntry) bool { 18 | return h.HostPort.Equal(entry.HostPort) && h.Status.Equal(entry.Status) 19 | } 20 | 21 | // Returns new HealthManagerEntry instance. 22 | func NewHealthManagerEntry( 23 | initialHealthyState bool, 24 | host string, 25 | port int) HealthManagerEntry { 26 | 27 | return HealthManagerEntry{ 28 | HostPort: discovery.NewHostPort(host, port, true), 29 | Status: NewHealthStatusEntry(initialHealthyState), 30 | } 31 | } 32 | 33 | type HealthManagerState []HealthManagerEntry 34 | 35 | // Returns true when HealthManagerState contains provided hostPort entry, 36 | // otherwise false. 37 | func (h HealthManagerState) Contains(hostPort *discovery.HostPort) bool { 38 | entry := h.GetEntry(hostPort) 39 | return entry != nil 40 | } 41 | 42 | // Returns reference to the entry for provided HostPort when state has it, 43 | // otherwise nil. 44 | func (h HealthManagerState) GetEntry( 45 | hostPort *discovery.HostPort) *HealthManagerEntry { 46 | 47 | for _, entry := range h { 48 | if hostPort.Equal(entry.HostPort) { 49 | return &entry 50 | } 51 | } 52 | 53 | return nil 54 | } 55 | 56 | // Returns true when at least one entry is healthy. 57 | func (h HealthManagerState) IsHealthy() bool { 58 | for _, entry := range h { 59 | if entry.Status.IsHealthy() { 60 | return true 61 | } 62 | } 63 | 64 | return false 65 | } 66 | 67 | // Convert all entries into the string. 68 | func (h HealthManagerState) String() string { 69 | out := "[" 70 | prefix := "" 71 | for i, entry := range h { 72 | if i != 0 { 73 | prefix = ", " 74 | } 75 | out += fmt.Sprintf( 76 | "%s%s/%v", 77 | prefix, 78 | entry.HostPort.String(), 79 | entry.Status.IsHealthy()) 80 | } 81 | out += "]" 82 | return out 83 | } 84 | 85 | // Returns new copy of HealthManagerState (it's not thread-safe call). 86 | func (h HealthManagerState) Clone() HealthManagerState { 87 | newState := make(HealthManagerState, len(h)) 88 | 89 | for i, entry := range h { 90 | newState[i] = HealthManagerEntry{ 91 | HostPort: discovery.NewHostPort( 92 | entry.HostPort.Host, 93 | entry.HostPort.Port, 94 | entry.HostPort.Enabled), 95 | Status: &healthStatusEntry{ 96 | isHealthy: entry.Status.isHealthy, 97 | healthCount: entry.Status.healthCount, 98 | }, 99 | } 100 | } 101 | 102 | return newState 103 | } 104 | 105 | // Returns true when all HostPort entries equal to provided DiscoveryState, 106 | // otherwise false. 107 | func (h HealthManagerState) Equal(state HealthManagerState) bool { 108 | if len(h) != len(state) { 109 | return false 110 | } 111 | 112 | // TODO(dkopytkov): improve comparison since it may check the same items 113 | // twice. 114 | for _, entry := range h { 115 | if !state.Contains(entry.HostPort) { 116 | return false 117 | } 118 | } 119 | 120 | for _, entry := range state { 121 | if !h.Contains(entry.HostPort) { 122 | return false 123 | } 124 | } 125 | 126 | return true 127 | } 128 | -------------------------------------------------------------------------------- /kglb/utils/health_manager/health_status_test.go: -------------------------------------------------------------------------------- 1 | package health_manager 2 | 3 | import ( 4 | . "gopkg.in/check.v1" 5 | 6 | . "godropbox/gocheck2" 7 | ) 8 | 9 | type StatusCheckerSuite struct { 10 | } 11 | 12 | var _ = Suite(&StatusCheckerSuite{}) 13 | 14 | func (m *StatusCheckerSuite) TestRaisingFalling(c *C) { 15 | riseCount := 3 16 | fallCount := 1 17 | 18 | entry := NewHealthStatusEntry(false) 19 | // check initial healthy status. 20 | c.Assert(entry.IsHealthy(), IsFalse) 21 | 22 | // need 3 successfull health checks before marking host as healthy. 23 | c.Assert(entry.UpdateHealthCheckStatus(true, riseCount, fallCount), IsFalse) 24 | c.Assert(entry.UpdateHealthCheckStatus(true, riseCount, fallCount), IsFalse) 25 | c.Assert(entry.UpdateHealthCheckStatus(true, riseCount, fallCount), IsTrue) 26 | c.Assert(entry.IsHealthy(), IsTrue) 27 | // but single failure to mark it as unhealthy, because of params configured 28 | // above. 29 | c.Assert(entry.UpdateHealthCheckStatus(false, riseCount, fallCount), IsTrue) 30 | c.Assert(entry.IsHealthy(), IsFalse) 31 | // repeating test. 32 | c.Assert(entry.UpdateHealthCheckStatus(true, riseCount, fallCount), IsFalse) 33 | c.Assert(entry.UpdateHealthCheckStatus(true, riseCount, fallCount), IsFalse) 34 | c.Assert(entry.UpdateHealthCheckStatus(true, riseCount, fallCount), IsTrue) 35 | c.Assert(entry.IsHealthy(), IsTrue) 36 | c.Assert(entry.UpdateHealthCheckStatus(false, riseCount, fallCount), IsTrue) 37 | c.Assert(entry.IsHealthy(), IsFalse) 38 | 39 | // the same set of tests, but with initially healthy entry. 40 | entry = NewHealthStatusEntry(true) 41 | // check initial healthy status. 42 | c.Assert(entry.IsHealthy(), IsTrue) 43 | 44 | // need 3 successful health checks before marking host as healthy. 45 | c.Assert(entry.UpdateHealthCheckStatus(true, riseCount, fallCount), IsFalse) 46 | c.Assert(entry.UpdateHealthCheckStatus(true, riseCount, fallCount), IsFalse) 47 | c.Assert(entry.UpdateHealthCheckStatus(true, riseCount, fallCount), IsFalse) 48 | c.Assert(entry.IsHealthy(), IsTrue) 49 | // but single failure to mark it as unhealthy, because of params configured 50 | // above. 51 | c.Assert(entry.UpdateHealthCheckStatus(false, riseCount, fallCount), IsTrue) 52 | c.Assert(entry.IsHealthy(), IsFalse) 53 | // repeating test. 54 | c.Assert(entry.UpdateHealthCheckStatus(true, riseCount, fallCount), IsFalse) 55 | c.Assert(entry.UpdateHealthCheckStatus(true, riseCount, fallCount), IsFalse) 56 | c.Assert(entry.UpdateHealthCheckStatus(true, riseCount, fallCount), IsTrue) 57 | c.Assert(entry.IsHealthy(), IsTrue) 58 | c.Assert(entry.UpdateHealthCheckStatus(false, riseCount, fallCount), IsTrue) 59 | c.Assert(entry.IsHealthy(), IsFalse) 60 | } 61 | 62 | // Concurrency test of read/update healthStatusEntry. 63 | func (m *StatusCheckerSuite) TestConcurrency(c *C) { 64 | entry := NewHealthStatusEntry(false) 65 | // check initial healthy status. 66 | c.Assert(entry.IsHealthy(), IsFalse) 67 | 68 | stopChan := make(chan interface{}, 20) 69 | // separate goroutine to read fields. 70 | go func() { 71 | for { 72 | select { 73 | case _, ok := <-stopChan: 74 | if ok { 75 | _ = entry.IsHealthy() 76 | } else { 77 | return 78 | } 79 | } 80 | } 81 | }() 82 | 83 | // main goroutine to update fields. 84 | for i := 0; i < 20; i++ { 85 | stopChan <- nil 86 | originalStatus := entry.IsHealthy() 87 | entry.UpdateHealthCheckStatus(!originalStatus, 1, 1) 88 | stopChan <- nil 89 | } 90 | 91 | close(stopChan) 92 | } 93 | -------------------------------------------------------------------------------- /kglb/utils/health_checker/syslog_checker.go: -------------------------------------------------------------------------------- 1 | package health_checker 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net" 7 | "os" 8 | "strconv" 9 | "strings" 10 | "time" 11 | 12 | hc_pb "dropbox/proto/kglb/healthchecker" 13 | "godropbox/errors" 14 | ) 15 | 16 | const ( 17 | defaultSyslogTag = "SyslogHealthChecker" 18 | defaultProbeMessage = "probe message." 19 | ) 20 | 21 | var _ HealthChecker = &SyslogChecker{} 22 | 23 | // Syslog checker. 24 | type SyslogChecker struct { 25 | params *hc_pb.SyslogCheckerAttributes 26 | 27 | // hostname and pid used as part of probe message sent to syslog daemon. 28 | hostname string 29 | pid int 30 | dialContext DialContextFunc 31 | } 32 | 33 | func NewSyslogChecker(params *hc_pb.SyslogCheckerAttributes, dialContext DialContextFunc) (*SyslogChecker, error) { 34 | // overriding defaultDialContext is not allowed for syslog checker 35 | if dialContext != nil { 36 | return nil, errors.New("custom dial context is not supported in SyslogChecker") 37 | } 38 | 39 | checker := &SyslogChecker{ 40 | params: params, 41 | pid: os.Getpid(), 42 | dialContext: defaultDialContext, 43 | } 44 | 45 | var err error 46 | checker.hostname, err = os.Hostname() 47 | if err != nil { 48 | return nil, err 49 | } 50 | 51 | if params.GetPort() > 0 { 52 | // make conversion once instead of repeating it every Dial call. 53 | dstPortStr := strconv.Itoa(int(params.GetPort())) 54 | // dialer. 55 | dialer := net.Dialer{} 56 | 57 | checker.dialContext = func( 58 | ctx context.Context, 59 | network, 60 | address string) (net.Conn, error) { 61 | 62 | host, _, err := net.SplitHostPort(address) 63 | if err != nil { 64 | return nil, errors.Wrapf(err, "fails to parse address: %s, err: ", address) 65 | } 66 | address = net.JoinHostPort(host, dstPortStr) 67 | return dialer.DialContext(ctx, network, address) 68 | } 69 | } 70 | 71 | return checker, nil 72 | } 73 | 74 | func (h *SyslogChecker) GetConfiguration() *hc_pb.HealthCheckerAttributes { 75 | return &hc_pb.HealthCheckerAttributes{ 76 | Attributes: &hc_pb.HealthCheckerAttributes_Syslog{ 77 | Syslog: h.params, 78 | }, 79 | } 80 | } 81 | 82 | // Generate message in syslog format. 83 | func (h *SyslogChecker) syslogFormat(level int, tag, msg string) []byte { 84 | // ensure it ends in a \n 85 | nl := "" 86 | if !strings.HasSuffix(msg, "\n") { 87 | nl = "\n" 88 | } 89 | 90 | timestamp := time.Now().Format(time.RFC3339) 91 | formattedMsg := fmt.Sprintf( 92 | "<%d>%s %s %s[%d]: %s%s", 93 | level, timestamp, h.hostname, tag, h.pid, msg, nl) 94 | return []byte(formattedMsg) 95 | } 96 | 97 | // Performs test and returns true when test was succeed. 98 | func (h *SyslogChecker) Check(host string, port int) error { 99 | timeout := timeoutMsToDuration(h.params.GetCheckTimeoutMs()) 100 | ctx, cancel := context.WithTimeout(context.Background(), timeout) 101 | defer cancel() 102 | 103 | address := net.JoinHostPort(host, strconv.Itoa(port)) 104 | conn, err := h.dialContext(ctx, strings.ToLower(h.params.GetProtocol().String()), address) 105 | if err != nil { 106 | return err 107 | } 108 | defer conn.Close() 109 | 110 | // making probe message. 111 | probeMsg := h.syslogFormat(int(h.params.GetLogLevel()), defaultSyslogTag, defaultProbeMessage) 112 | // sending message, unfortunately without any reply. 113 | _, err = conn.Write([]byte(probeMsg)) 114 | if err != nil { 115 | return err 116 | } 117 | 118 | return nil 119 | } 120 | -------------------------------------------------------------------------------- /kglb/data_plane/manager_dynamic_routing_test.go: -------------------------------------------------------------------------------- 1 | package data_plane 2 | 3 | import ( 4 | "time" 5 | 6 | . "gopkg.in/check.v1" 7 | 8 | kglb_pb "dropbox/proto/kglb" 9 | . "godropbox/gocheck2" 10 | ) 11 | 12 | type DynamicRoutingManagerSuite struct { 13 | // Mock Bgp module. 14 | mockBgp *MockBgpModule 15 | // DynamicRoutingManager. 16 | manager *DynamicRoutingManager 17 | } 18 | 19 | var _ = Suite(&DynamicRoutingManagerSuite{}) 20 | 21 | func (m *DynamicRoutingManagerSuite) SetUpTest(c *C) { 22 | m.mockBgp = NewMockBgpModuleWithState().(*MockBgpModule) 23 | 24 | params := DynamicRoutingManagerParams{ 25 | Bgp: m.mockBgp, 26 | } 27 | 28 | var err error 29 | m.manager, err = NewDynamicRoutingManager(params) 30 | c.Assert(err, IsNil) 31 | } 32 | 33 | // Advertise routes. 34 | func (m *DynamicRoutingManagerSuite) TestHoldTimeouts(c *C) { 35 | err := m.manager.AdvertiseRoutes( 36 | []*kglb_pb.DynamicRoute{ 37 | { 38 | Attributes: &kglb_pb.DynamicRoute_BgpAttributes{ 39 | BgpAttributes: &kglb_pb.BgpRouteAttributes{ 40 | LocalAsn: 10, 41 | PeerAsn: 20, 42 | Community: "my_community", 43 | Prefix: &kglb_pb.IP{ 44 | Address: &kglb_pb.IP_Ipv4{ 45 | Ipv4: "10.0.0.2", 46 | }, 47 | }, 48 | Prefixlen: 32, 49 | HoldTimeMs: 51, 50 | }, 51 | }, 52 | }, 53 | }) 54 | c.Assert(err, IsNil) 55 | c.Assert(len(m.manager.holdTimeouts), Equals, 1) 56 | key := &kglb_pb.IP{ 57 | Address: &kglb_pb.IP_Ipv4{Ipv4: "10.0.0.2"}, 58 | } 59 | c.Assert(m.manager.holdTimeouts[key.String()], Equals, 51*time.Millisecond) 60 | 61 | holdTime, err := m.manager.withdrawRouteLocked( 62 | &kglb_pb.DynamicRoute{ 63 | Attributes: &kglb_pb.DynamicRoute_BgpAttributes{ 64 | BgpAttributes: &kglb_pb.BgpRouteAttributes{ 65 | LocalAsn: 10, 66 | PeerAsn: 20, 67 | Community: "my_community", 68 | Prefix: &kglb_pb.IP{ 69 | Address: &kglb_pb.IP_Ipv4{ 70 | Ipv4: "10.0.0.2", 71 | }, 72 | }, 73 | Prefixlen: 32, 74 | HoldTimeMs: 51, 75 | }, 76 | }, 77 | }) 78 | c.Assert(err, IsNil) 79 | c.Assert(holdTime, Equals, 51*time.Millisecond) 80 | c.Assert(len(m.manager.holdTimeouts), Equals, 0) 81 | 82 | err = m.manager.AdvertiseRoutes( 83 | []*kglb_pb.DynamicRoute{ 84 | { 85 | Attributes: &kglb_pb.DynamicRoute_BgpAttributes{ 86 | BgpAttributes: &kglb_pb.BgpRouteAttributes{ 87 | LocalAsn: 10, 88 | PeerAsn: 20, 89 | Community: "my_community", 90 | Prefix: &kglb_pb.IP{ 91 | Address: &kglb_pb.IP_Ipv4{ 92 | Ipv4: "10.0.0.2", 93 | }, 94 | }, 95 | Prefixlen: 32, 96 | HoldTimeMs: 51, 97 | }, 98 | }, 99 | }, 100 | }) 101 | c.Assert(err, IsNil) 102 | 103 | // withdrawing route. 104 | startTime := time.Now() 105 | err = m.manager.WithdrawRoutes( 106 | []*kglb_pb.DynamicRoute{ 107 | { 108 | Attributes: &kglb_pb.DynamicRoute_BgpAttributes{ 109 | BgpAttributes: &kglb_pb.BgpRouteAttributes{ 110 | LocalAsn: 10, 111 | PeerAsn: 20, 112 | Community: "my_community", 113 | Prefix: &kglb_pb.IP{ 114 | Address: &kglb_pb.IP_Ipv4{ 115 | Ipv4: "10.0.0.2", 116 | }, 117 | }, 118 | Prefixlen: 32, 119 | HoldTimeMs: 51, 120 | }, 121 | }, 122 | }, 123 | }) 124 | elapsed := time.Since(startTime) 125 | c.Assert(err, IsNil) 126 | c.Assert(len(m.manager.holdTimeouts), Equals, 0) 127 | c.Assert(elapsed/time.Millisecond, GreaterThan, 50) 128 | 129 | } 130 | -------------------------------------------------------------------------------- /kglb/utils/discovery/static_resolver.go: -------------------------------------------------------------------------------- 1 | package discovery 2 | 3 | import ( 4 | "sync" 5 | 6 | "dropbox/dlog" 7 | "dropbox/vortex2/v2stats" 8 | ) 9 | 10 | // Static resolver specific params. 11 | type StaticResolverParams struct { 12 | // Resolver Id. 13 | Id string 14 | // Initial state. 15 | Hosts DiscoveryState 16 | 17 | // Used to report v2 stat 18 | SetupName string 19 | ServiceName string 20 | } 21 | 22 | // Static Resolver implementation. 23 | type StaticResolver struct { 24 | // resolver id. 25 | id string 26 | // current state of the resolver. 27 | state DiscoveryState 28 | 29 | // update channel. 30 | updateChan chan DiscoveryState 31 | closeOnce sync.Once 32 | mutex *sync.Mutex 33 | 34 | // v2 stats 35 | statResolverGauge v2stats.Gauge 36 | } 37 | 38 | func NewStaticResolver(params StaticResolverParams) (*StaticResolver, error) { 39 | var hostPorts []*HostPort 40 | for _, entry := range params.Hosts { 41 | hostPorts = append( 42 | hostPorts, 43 | &HostPort{Host: entry.Host, Port: entry.Port, Address: entry.Host, Enabled: entry.Enabled}) 44 | } 45 | 46 | resolver := &StaticResolver{ 47 | id: params.Id, 48 | state: DiscoveryState(hostPorts), 49 | mutex: &sync.Mutex{}, 50 | updateChan: make(chan DiscoveryState, 1), 51 | 52 | statResolverGauge: staticResolverGauge.Must(v2stats.KV{ 53 | "setup": params.SetupName, 54 | "service": params.ServiceName, 55 | }), 56 | } 57 | 58 | // put initial state into update channel since there will not be any auto 59 | // update based on discovery changes. 60 | resolver.updateChan <- resolver.state 61 | 62 | return resolver, nil 63 | } 64 | 65 | // Returns resolver id. 66 | func (r *StaticResolver) GetId() string { 67 | return r.id 68 | } 69 | 70 | // Implements DiscoveryResolver interface 71 | func (r *StaticResolver) GetState() DiscoveryState { 72 | r.mutex.Lock() 73 | defer r.mutex.Unlock() 74 | 75 | return r.state 76 | } 77 | 78 | // Updates state. 79 | func (r *StaticResolver) Update(newState DiscoveryState) { 80 | r.mutex.Lock() 81 | defer r.mutex.Unlock() 82 | 83 | dlog.Infof("Update state of '%s' resolver: %v", r.id, newState) 84 | 85 | // 1. update internal state. 86 | r.state = newState 87 | 88 | // Update gauge 89 | r.statResolverGauge.Set(float64(len(newState))) 90 | 91 | // remove state from chan if any 92 | select { 93 | case <-r.updateChan: 94 | default: 95 | } 96 | 97 | // 2. notify about the change. 98 | r.updateChan <- r.state 99 | } 100 | 101 | // Returns update channel. 102 | func (r *StaticResolver) Updates() <-chan DiscoveryState { 103 | return r.updateChan 104 | } 105 | 106 | // Implements DiscoveryResolver interface. 107 | func (r *StaticResolver) Close() { 108 | dlog.Infof("Closing '%s' resolver", r.id) 109 | 110 | r.closeOnce.Do(func() { 111 | close(r.updateChan) 112 | }) 113 | } 114 | 115 | // Check if the item discovers exactly the same things. 116 | func (r *StaticResolver) Equal(item DiscoveryResolver) bool { 117 | if _, ok := item.(*StaticResolver); ok { 118 | thisState := r.GetState() 119 | itemState := item.GetState() 120 | if len(thisState) != len(itemState) { 121 | return false 122 | } 123 | 124 | if r.GetId() != item.GetId() { 125 | return false 126 | } 127 | 128 | for i, _ := range thisState { 129 | if !thisState[i].Equal(itemState[i]) { 130 | return false 131 | } 132 | } 133 | // states are exactly the same including order of the items. 134 | return true 135 | } 136 | 137 | return false 138 | } 139 | 140 | var _ DiscoveryResolver = &StaticResolver{} 141 | -------------------------------------------------------------------------------- /vortex2/v2stats/prometheus.go: -------------------------------------------------------------------------------- 1 | package v2stats 2 | 3 | import ( 4 | "math" 5 | "strings" 6 | 7 | "github.com/prometheus/client_golang/prometheus" 8 | ) 9 | 10 | type Counter struct { 11 | prometheus.Counter 12 | } 13 | 14 | type Gauge struct { 15 | prometheus.Gauge 16 | 17 | gauges *prometheus.GaugeVec 18 | labels prometheus.Labels 19 | } 20 | 21 | func (g Gauge) Set(val float64) { 22 | if math.IsNaN(val) { 23 | g.Clear() 24 | } else { 25 | g.Gauge.Set(val) 26 | } 27 | } 28 | 29 | func (g Gauge) Clear() { 30 | g.gauges.Delete(g.labels) 31 | } 32 | 33 | // Generic interface for gauge metric definition. 34 | type GaugeDefinition struct { 35 | tagNames []string 36 | gauges *prometheus.GaugeVec 37 | } 38 | 39 | func (g *GaugeDefinition) V(kvs ...KV) (Gauge, error) { 40 | labels := make(prometheus.Labels) 41 | for _, tag := range g.tagNames { 42 | for _, kv := range kvs { 43 | if val, ok := kv[tag]; ok { 44 | labels[tag] = val 45 | } 46 | } 47 | } 48 | 49 | m, err := g.gauges.GetMetricWith(labels) 50 | if err != nil { 51 | return Gauge{}, err 52 | } 53 | return Gauge{ 54 | Gauge: m, 55 | gauges: g.gauges, 56 | labels: labels, 57 | }, nil 58 | } 59 | 60 | func (g *GaugeDefinition) Must(kvs ...KV) Gauge { 61 | m, err := g.V(kvs...) 62 | if err != nil { 63 | panic(err) 64 | } 65 | return m 66 | } 67 | 68 | // Generic interface for counter metric definition. 69 | type CounterDefinition struct { 70 | tagNames []string 71 | counters *prometheus.CounterVec 72 | } 73 | 74 | func (c *CounterDefinition) V(kvs KV) (Counter, error) { 75 | labels := make(prometheus.Labels) 76 | for _, tag := range c.tagNames { 77 | if val, ok := kvs[tag]; ok { 78 | labels[tag] = val 79 | } 80 | } 81 | 82 | if m, err := c.counters.GetMetricWith(labels); err != nil { 83 | return Counter{}, err 84 | } else { 85 | return Counter{m}, nil 86 | } 87 | } 88 | 89 | func (c *CounterDefinition) Must(kvs KV) Counter { 90 | m, err := c.V(kvs) 91 | if err != nil { 92 | panic(err) 93 | } 94 | return m 95 | } 96 | 97 | func DefineGauge(name string, tagNames ...string) (GaugeDefinition, error) { 98 | name = strings.Replace(name, "/", ":", -1) 99 | opts := prometheus.GaugeOpts{ 100 | Name: name, 101 | Help: "TODO", 102 | } 103 | 104 | gauges := prometheus.NewGaugeVec(opts, tagNames) 105 | prometheus.MustRegister(gauges) 106 | 107 | return GaugeDefinition{ 108 | tagNames: tagNames, 109 | gauges: gauges, 110 | }, nil 111 | } 112 | 113 | func MustDefineGauge(name string, tagNames ...string) GaugeDefinition { 114 | m, err := DefineGauge(name, tagNames...) 115 | if err != nil { 116 | panic(err) 117 | } 118 | 119 | return m 120 | } 121 | 122 | func DefineGaugeWithHierarchy( 123 | name string, 124 | tagNamesHierarchy ...[]string) (GaugeDefinition, error) { 125 | 126 | var tagNames []string 127 | for _, tags := range tagNamesHierarchy { 128 | tagNames = append(tagNames, tags...) 129 | } 130 | return DefineGauge(name, tagNames...) 131 | } 132 | 133 | func DefineCounter(name string, tagNames ...string) (CounterDefinition, error) { 134 | name = strings.Replace(name, "/", ":", -1) 135 | opts := prometheus.CounterOpts{ 136 | Name: name, 137 | Help: "TODO", 138 | } 139 | 140 | counters := prometheus.NewCounterVec(opts, tagNames) 141 | prometheus.MustRegister(counters) 142 | 143 | return CounterDefinition{ 144 | tagNames: tagNames, 145 | counters: counters, 146 | }, nil 147 | } 148 | 149 | func MustDefineCounter(name string, tagNames ...string) CounterDefinition { 150 | m, err := DefineCounter(name, tagNames...) 151 | if err != nil { 152 | panic(err) 153 | } 154 | 155 | return m 156 | } 157 | -------------------------------------------------------------------------------- /kglbd/service.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/golang/glog" 8 | 9 | "dropbox/kglb/control_plane" 10 | "dropbox/kglb/data_plane" 11 | "dropbox/kglb/utils/dns_resolver" 12 | "dropbox/kglb/utils/fwmark" 13 | kglb_pb "dropbox/proto/kglb" 14 | ) 15 | 16 | const ( 17 | defaultMaxFwmark = 5000 18 | defaultFwmarkBase = 10000 19 | ) 20 | 21 | var ( 22 | // dns resolution timeout. 23 | maxDnsResolveTime = 1 * time.Minute 24 | 25 | // how often do we send state to data plane. 26 | DefaultSendInterval = 30 * time.Second 27 | 28 | // emitting interval of ipvs and realserver stats. 29 | statsEmitInterval = 10 * time.Second 30 | ) 31 | 32 | type Service struct { 33 | controlPlaneMng *control_plane.ControlPlaneServicer 34 | dataPlaneMng *data_plane.Manager 35 | } 36 | 37 | func NewService(ctx context.Context, configPath string) (*Service, error) { 38 | s := &Service{} 39 | if err := s.initModules(ctx, configPath); err != nil { 40 | return nil, err 41 | } 42 | 43 | // stats emitter loop 44 | go s.statsLoop(ctx) 45 | 46 | return s, nil 47 | } 48 | 49 | // Apply data plane state. 50 | func (s *Service) Set(state *kglb_pb.DataPlaneState) error { 51 | err := s.dataPlaneMng.SetState(state) 52 | if err != nil { 53 | glog.Errorf("fails to apply data plane state: %v", err) 54 | } 55 | 56 | return err 57 | } 58 | 59 | // Initialize all required modules and control/data planes. 60 | func (s *Service) initModules(ctx context.Context, configPath string) error { 61 | var err error 62 | 63 | // initializing data plane related modules. 64 | dpModules := data_plane.ManagerModules{ 65 | Bgp: &NoOpBgpModule{}, 66 | } 67 | 68 | cacheResolver, err := data_plane.NewCacheResolver() 69 | if err != nil { 70 | return err 71 | } 72 | 73 | dpModules.Resolver = cacheResolver 74 | 75 | if dpModules.Ipvs, err = data_plane.NewIpvsMqLiang(cacheResolver); err != nil { 76 | return err 77 | } 78 | 79 | if dpModules.AddressTable, err = data_plane.NewNetlinkAddress(); err != nil { 80 | return err 81 | } 82 | 83 | if s.dataPlaneMng, err = data_plane.NewManager(dpModules); err != nil { 84 | return err 85 | } 86 | 87 | // initializing control plane related modules. 88 | cpModules := control_plane.ServicerModules{ 89 | DataPlaneClient: s, 90 | } 91 | 92 | if cpModules.ConfigLoader, err = MakeConfigLoader(configPath); err != nil { 93 | return err 94 | } 95 | 96 | cpModules.FwmarkManager = fwmark.NewManager(defaultMaxFwmark, defaultFwmarkBase) 97 | 98 | if cpModules.DnsResolver, err = dns_resolver.NewSystemResolver(maxDnsResolveTime); err != nil { 99 | return err 100 | } 101 | 102 | cpModules.DiscoveryFactory = control_plane.NewDiscoveryFactory() 103 | 104 | cpModules.CheckerFactory = control_plane.NewHealthCheckerFactory( 105 | control_plane.BaseHealthCheckerFactoryParams{ 106 | FwmarkManager: cpModules.FwmarkManager, 107 | }) 108 | 109 | if s.controlPlaneMng, err = control_plane.NewControlPlaneServicer(ctx, cpModules, DefaultSendInterval); err != nil { 110 | return err 111 | } 112 | 113 | return nil 114 | } 115 | 116 | func (s *Service) Shutdown() error { 117 | err := s.dataPlaneMng.Shutdown() 118 | if err != nil { 119 | glog.Errorf("Fails to shutdown data plane manager: %v", err) 120 | } 121 | return err 122 | } 123 | 124 | // emit ipvs service and realserver stats. 125 | func (s *Service) statsLoop(ctx context.Context) { 126 | ticker := time.NewTicker(statsEmitInterval) 127 | defer ticker.Stop() 128 | 129 | for { 130 | select { 131 | case <-ctx.Done(): 132 | glog.Info("Closing statsLoop") 133 | return 134 | case <-ticker.C: 135 | if err := s.dataPlaneMng.EmitStats(); err != nil { 136 | glog.Errorf("Fails to emit stats: %v", err) 137 | } 138 | } 139 | } 140 | } 141 | -------------------------------------------------------------------------------- /kglb/utils/discovery/static_resolver_test.go: -------------------------------------------------------------------------------- 1 | package discovery 2 | 3 | import ( 4 | "time" 5 | 6 | . "gopkg.in/check.v1" 7 | 8 | . "godropbox/gocheck2" 9 | ) 10 | 11 | type StaticResolverSuite struct{} 12 | 13 | var _ = Suite(&StaticResolverSuite{}) 14 | 15 | func (s *StaticResolverSuite) TestEqual(c *C) { 16 | resolver1, err := NewStaticResolver( 17 | StaticResolverParams{ 18 | Id: "resolver1", 19 | Hosts: DiscoveryState([]*HostPort{ 20 | NewHostPort("host1", 80, true), 21 | NewHostPort("host2", 80, true), 22 | }), 23 | }) 24 | c.Assert(err, IsNil) 25 | 26 | // checking id. 27 | testResolver, err := NewStaticResolver( 28 | StaticResolverParams{ 29 | Id: "resolver2", 30 | Hosts: DiscoveryState([]*HostPort{ 31 | NewHostPort("host1", 80, true), 32 | NewHostPort("host2", 80, true), 33 | }), 34 | }) 35 | c.Assert(err, IsNil) 36 | c.Assert(resolver1.Equal(testResolver), IsFalse) 37 | testResolver, err = NewStaticResolver( 38 | StaticResolverParams{ 39 | Id: "resolver1", 40 | Hosts: DiscoveryState([]*HostPort{ 41 | NewHostPort("host1", 80, true), 42 | NewHostPort("host2", 80, true), 43 | }), 44 | }) 45 | c.Assert(err, IsNil) 46 | c.Assert(resolver1.Equal(testResolver), IsTrue) 47 | // checking states. 48 | testResolver, err = NewStaticResolver( 49 | StaticResolverParams{ 50 | Id: "resolver1", 51 | Hosts: DiscoveryState([]*HostPort{ 52 | NewHostPort("host1", 80, true), 53 | }), 54 | }) 55 | c.Assert(err, IsNil) 56 | c.Assert(resolver1.Equal(testResolver), IsFalse) 57 | testResolver, err = NewStaticResolver( 58 | StaticResolverParams{ 59 | Id: "resolver1", 60 | Hosts: DiscoveryState([]*HostPort{ 61 | NewHostPort("host2", 80, true), 62 | NewHostPort("host1", 80, true), 63 | }), 64 | }) 65 | c.Assert(err, IsNil) 66 | c.Assert(resolver1.Equal(testResolver), IsFalse) 67 | testResolver, err = NewStaticResolver( 68 | StaticResolverParams{ 69 | Id: "resolver1", 70 | Hosts: DiscoveryState([]*HostPort{ 71 | NewHostPort("host1", 80, true), 72 | NewHostPort("host2", 80, true), 73 | }), 74 | }) 75 | c.Assert(err, IsNil) 76 | c.Assert(resolver1.Equal(testResolver), IsTrue) 77 | } 78 | 79 | func (s *StaticResolverSuite) TestUpdate(c *C) { 80 | resolver1, err := NewStaticResolver( 81 | StaticResolverParams{ 82 | Id: "resolver1", 83 | Hosts: DiscoveryState([]*HostPort{ 84 | NewHostPort("host1", 80, true), 85 | NewHostPort("host2", 80, true), 86 | }), 87 | }) 88 | c.Assert(err, IsNil) 89 | 90 | // checking initial state. 91 | select { 92 | case state, ok := <-resolver1.Updates(): 93 | c.Assert(ok, IsTrue) 94 | c.Assert(state, DeepEquals, DiscoveryState([]*HostPort{ 95 | NewHostPort("host1", 80, true), 96 | NewHostPort("host2", 80, true), 97 | })) 98 | case <-time.After(time.Second): 99 | c.Log("timeout to wait update.") 100 | c.Fail() 101 | } 102 | 103 | updateChan := resolver1.Updates() 104 | resolver1.Update(DiscoveryState([]*HostPort{ 105 | NewHostPort("host1", 80, true), 106 | NewHostPort("host2", 80, true), 107 | NewHostPort("host3", 80, true), 108 | })) 109 | // waiting update though the channel. 110 | select { 111 | case newState, ok := <-updateChan: 112 | c.Assert(ok, IsTrue) 113 | c.Assert(newState, DeepEquals, DiscoveryState([]*HostPort{ 114 | NewHostPort("host1", 80, true), 115 | NewHostPort("host2", 80, true), 116 | NewHostPort("host3", 80, true), 117 | })) 118 | case <-time.After(time.Second): 119 | c.Log("timeout to wait update.") 120 | c.Fail() 121 | } 122 | 123 | // Closing. 124 | resolver1.Close() 125 | // waiting update though the channel. 126 | select { 127 | case _, ok := <-updateChan: 128 | c.Assert(ok, IsFalse) 129 | default: 130 | c.Log("closing fails.") 131 | c.Fail() 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /proto/dropbox/proto/kglb/healthchecker/healthchecker.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package healthchecker; 4 | 5 | message DnsCheckerAttributes { 6 | enum DNSQueryType { 7 | A = 0; // IPv4 address record. 8 | AAAA = 1; // IPv6 address record. 9 | NS = 2; // Name server record. 10 | MX = 3; // Mail exchange record. 11 | TXT = 4; // Text record. 12 | SOA = 5; // Start of authority record. 13 | } 14 | 15 | // dns query string (".", "www.dropbox.com" and etc) 16 | string query_string = 1; 17 | // type of dns query message. 18 | DNSQueryType query_type = 2; 19 | // dns client (tcp|udp) transport. 20 | IPProtocol protocol = 3; 21 | // expected rcode in response to path health check. 22 | uint32 rcode = 4; 23 | 24 | // max wait timeout in milliseconds to complete the test. 25 | // default value is 5000 ms. 26 | uint32 check_timeout_ms = 10; 27 | } 28 | 29 | message DummyCheckerAttributes { 30 | } 31 | 32 | message HttpCheckerAttributes { 33 | // http or https 34 | // perform https request to backend if set to 'https' 35 | string scheme = 1; 36 | // TODO(team): rename to urlPath or something same 37 | // url path to be requested on backend: "/", "/health", etc. 38 | string uri = 2; 39 | // headers to be set in http request 40 | map headers = 3; 41 | // list of expected http response codes 42 | repeated uint32 codes = 4; 43 | // if `proxy_check_url` is defined, healthcheck is performed as http[s] (based on `scheme` field) 44 | // proxy request to backend querying `proxy_check_url`, `uri` is ignored. 45 | string proxy_check_url = 5; 46 | // follow redirects and latest response code, disabled by default 47 | bool follow_redirects = 6; 48 | 49 | // max wait timeout in milliseconds to complete the test. 50 | // default value is 5000 ms. 51 | uint32 check_timeout_ms = 10; 52 | } 53 | 54 | // Configuration of syslog health checker. 55 | message SyslogCheckerAttributes { 56 | // Syslog log level specified in the message sent to syslog daemon. 57 | uint32 log_level = 1; 58 | 59 | // (Optional) Health checking port. When it's specified (has positive value) 60 | // health checker will use the number as dst port instead of using port 61 | // defined in lb_service config. 62 | uint32 port = 2; 63 | 64 | // tcp|udp health checking transport. It might be helpful to perform health 65 | // checking over tcp even lb_service is udp since udp health checking is not 66 | // much effective without any response from syslog daemon. 67 | IPProtocol protocol = 3; 68 | 69 | // max wait timeout in milliseconds to complete the test. 70 | // default value is 5000 ms. 71 | uint32 check_timeout_ms = 10; 72 | } 73 | 74 | // Configuration of TCP health checker 75 | message TcpCheckerAttributes { 76 | // max wait timeout in milliseconds to complete the test 77 | // default value is 5000ms 78 | uint32 check_timeout_ms = 1; 79 | } 80 | 81 | message UpstreamChecker { 82 | // individual entry check interval 83 | uint32 interval_ms = 2; 84 | 85 | // number of consecutive valid health checks before considering the upstream as UP 86 | uint32 rise_count = 3; 87 | // number of consecutive invalid health checks before considering the upstream as DOWN 88 | uint32 fall_count = 4; 89 | 90 | HealthCheckerAttributes checker = 5; 91 | 92 | // How many concurrency health checks can be send to this set of upstreams. 93 | // Default value is 100. 94 | uint32 concurrency_limit = 6; 95 | } 96 | 97 | message HealthCheckerAttributes { 98 | oneof attributes { 99 | DummyCheckerAttributes dummy = 1; 100 | DnsCheckerAttributes dns = 2; 101 | HttpCheckerAttributes http = 3; 102 | SyslogCheckerAttributes syslog = 4; 103 | TcpCheckerAttributes tcp = 5; 104 | } 105 | } 106 | 107 | enum IPProtocol { 108 | TCP = 0; 109 | UDP = 1; 110 | } 111 | -------------------------------------------------------------------------------- /kglb/data_plane/netlink_address_impl.go: -------------------------------------------------------------------------------- 1 | package data_plane 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "sort" 7 | 8 | "github.com/vishvananda/netlink" 9 | 10 | "godropbox/errors" 11 | ) 12 | 13 | type netlinkFunc struct { 14 | // AddrAdd will add an IP address to a link device. 15 | AddrAdd func(link netlink.Link, addr *netlink.Addr) error 16 | // AddrDel will delete an IP address from a link device. 17 | AddrDel func(link netlink.Link, addr *netlink.Addr) error 18 | // / AddrList gets a list of IP addresses in the system. 19 | AddrList func(link netlink.Link, family int) ([]netlink.Addr, error) 20 | // ParseAddr parses the string representation of an address in the 21 | // form $ip/$netmask $label. 22 | ParseAddr func(s string) (*netlink.Addr, error) 23 | // Convert string representation of the link into internal structure. 24 | LinkByName func(name string) (netlink.Link, error) 25 | } 26 | 27 | // wrapper for github.com/vishvananda/netlink lib. 28 | type NetlinkAddress struct { 29 | // netlink related funcs. 30 | nlApi *netlinkFunc 31 | } 32 | 33 | // using in tests to simplify mocking of low-level api. 34 | func newNetLinkAddress(nlApi *netlinkFunc) (AddressTableModule, error) { 35 | return &NetlinkAddress{ 36 | nlApi: nlApi, 37 | }, nil 38 | } 39 | 40 | func NewNetlinkAddress() (AddressTableModule, error) { 41 | // netlink related funcs. 42 | nlApi := &netlinkFunc{ 43 | AddrAdd: netlink.AddrAdd, 44 | AddrDel: netlink.AddrDel, 45 | AddrList: netlink.AddrList, 46 | ParseAddr: netlink.ParseAddr, 47 | LinkByName: netlink.LinkByName, 48 | } 49 | 50 | return newNetLinkAddress(nlApi) 51 | } 52 | 53 | // Add the address to the interface. 54 | func (m *NetlinkAddress) Add(addr net.IP, iface string) error { 55 | // 1. check if address already exists. 56 | if exist, err := m.IsExists(addr, iface); err != nil { 57 | return err 58 | } else if exist { 59 | return errors.Newf("address already exist: %v", addr.String()) 60 | } 61 | 62 | // 2. Convert to required lib format. 63 | ntAddr, err := m.nlApi.ParseAddr(fmt.Sprintf("%s/32", addr.String())) 64 | if err != nil { 65 | return err 66 | } 67 | 68 | ntLink, err := m.nlApi.LinkByName(iface) 69 | if err != nil { 70 | return err 71 | } 72 | 73 | // 3. Adding the address. 74 | if err = m.nlApi.AddrAdd(ntLink, ntAddr); err != nil { 75 | return err 76 | } 77 | 78 | return nil 79 | } 80 | 81 | // Delete the address from the interface. 82 | func (m *NetlinkAddress) Delete(addr net.IP, iface string) error { 83 | // 1. check if address exists, otherwise false. 84 | if exist, err := m.IsExists(addr, iface); err != nil { 85 | return err 86 | } else if !exist { 87 | return errors.Newf("address doesn't exist: %v", addr.String()) 88 | } 89 | 90 | // 2. Convert to required lib format. 91 | ntAddr, err := m.nlApi.ParseAddr(fmt.Sprintf("%s/32", addr.String())) 92 | if err != nil { 93 | return err 94 | } 95 | 96 | ntLink, err := m.nlApi.LinkByName(iface) 97 | if err != nil { 98 | return err 99 | } 100 | 101 | // 3. Finally delete the address. 102 | if err = m.nlApi.AddrDel(ntLink, ntAddr); err != nil { 103 | return err 104 | } 105 | 106 | return nil 107 | } 108 | 109 | // Checks if the address belongs to the link, otherwise returns false. 110 | func (m *NetlinkAddress) IsExists(addr net.IP, iface string) (bool, error) { 111 | addrs, err := m.List(iface) 112 | if err != nil { 113 | return false, err 114 | } 115 | 116 | for _, linkAddr := range addrs { 117 | if addr.Equal(linkAddr) { 118 | return true, nil 119 | } 120 | } 121 | 122 | return false, nil 123 | } 124 | 125 | // List of all configured addresses for specific interface. 126 | func (m *NetlinkAddress) List(iface string) ([]net.IP, error) { 127 | ntLink, err := m.nlApi.LinkByName(iface) 128 | if err != nil { 129 | return nil, err 130 | } 131 | 132 | var addrList []net.IP 133 | 134 | for _, addrFamily := range []int{netlink.FAMILY_V4, netlink.FAMILY_V6} { 135 | addrs, err := m.nlApi.AddrList(ntLink, addrFamily) 136 | if err != nil { 137 | return nil, errors.Wrap(err, "netlink.AddrList() fails: ") 138 | } 139 | 140 | for _, addr := range addrs { 141 | addrList = append(addrList, addr.IP) 142 | } 143 | 144 | } 145 | sort.Slice(addrList, func(i, j int) bool { return addrList[i].String() < addrList[j].String() }) 146 | return addrList, nil 147 | } 148 | -------------------------------------------------------------------------------- /kglb/utils/health_checker/syslog_checker_test.go: -------------------------------------------------------------------------------- 1 | package health_checker 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "strconv" 7 | "time" 8 | 9 | . "gopkg.in/check.v1" 10 | 11 | hc_pb "dropbox/proto/kglb/healthchecker" 12 | . "godropbox/gocheck2" 13 | ) 14 | 15 | type SyslogCheckerSuite struct { 16 | syslogV4 *net.TCPListener 17 | syslogV6 *net.TCPListener 18 | } 19 | 20 | var _ = Suite(&SyslogCheckerSuite{}) 21 | 22 | func createMockSyslogd(network string, host string) (*net.TCPListener, error) { 23 | // find listener address and resolve host when it is needed. 24 | lAddr, err := net.ResolveTCPAddr(network, fmt.Sprintf("%s:%d", host, 0)) 25 | if err != nil { 26 | return nil, err 27 | } 28 | tcpListener, err := net.ListenTCP(network, lAddr) 29 | if err != nil { 30 | return nil, err 31 | } 32 | 33 | go func() { 34 | for { 35 | conn, err := tcpListener.Accept() 36 | if err != nil { 37 | continue 38 | } 39 | buff := make([]byte, 512) 40 | n, err := conn.Read(buff) 41 | if err == nil { 42 | continue 43 | } 44 | fmt.Printf("Syslog got message: %s\n", string(buff[:n])) 45 | } 46 | }() 47 | 48 | return tcpListener, err 49 | } 50 | 51 | func hostPort(listener *net.TCPListener) (string, int, error) { 52 | addr := listener.Addr() 53 | host, ports, err := net.SplitHostPort(addr.String()) 54 | if err != nil { 55 | return "", 0, err 56 | } 57 | port, err := strconv.Atoi(ports) 58 | if err != nil { 59 | return "", 0, err 60 | } 61 | return host, port, nil 62 | } 63 | 64 | func (s *SyslogCheckerSuite) SetUpTest(c *C) { 65 | var err error 66 | s.syslogV4, err = createMockSyslogd("tcp", "127.0.0.1") 67 | c.Assert(err, NoErr) 68 | s.syslogV6, err = createMockSyslogd("tcp6", "[::1]") 69 | c.Assert(err, NoErr) 70 | } 71 | 72 | func (s *SyslogCheckerSuite) TearDownTest(c *C) { 73 | s.syslogV4.Close() 74 | s.syslogV6.Close() 75 | } 76 | 77 | func (s *SyslogCheckerSuite) TestTcp(c *C) { 78 | params := &hc_pb.SyslogCheckerAttributes{ 79 | LogLevel: 10, 80 | Protocol: hc_pb.IPProtocol_TCP, 81 | CheckTimeoutMs: uint32(time.Second / time.Millisecond), 82 | } 83 | 84 | syslogHost, syslogPort, err := hostPort(s.syslogV4) 85 | c.Assert(err, NoErr) 86 | 87 | syslogChecker, err := NewSyslogChecker(params, nil) 88 | c.Assert(err, NoErr) 89 | c.Assert(syslogChecker.Check(syslogHost, syslogPort), NoErr) 90 | 91 | // TODO(oleg): implement test for checking timeout 92 | //params.CheckTimeoutMs = -1 93 | ////check Timeout behaviour 94 | //syslogChecker, err = NewSyslogChecker(params, nil) 95 | //c.Assert(err, NoErr) 96 | //c.Assert(syslogChecker.Check(syslogHost, syslogPort), ErrorMatches, ".*i/o timeout") 97 | } 98 | 99 | func (s *SyslogCheckerSuite) TestTcp6Success(c *C) { 100 | params := &hc_pb.SyslogCheckerAttributes{ 101 | LogLevel: 10, 102 | Protocol: hc_pb.IPProtocol_TCP, 103 | CheckTimeoutMs: uint32(time.Second / time.Millisecond), 104 | } 105 | syslogHost, syslogPort, err := hostPort(s.syslogV6) 106 | c.Assert(err, NoErr) 107 | 108 | syslogChecker, err := NewSyslogChecker(params, nil) 109 | c.Assert(err, NoErr) 110 | c.Assert(syslogChecker.Check(syslogHost, syslogPort), NoErr) 111 | } 112 | 113 | // TestUdpSomewhere validates that checker really sends udp packets 114 | func (s *SyslogCheckerSuite) TestUdpSomewhere(c *C) { 115 | params := &hc_pb.SyslogCheckerAttributes{ 116 | LogLevel: 10, 117 | Protocol: hc_pb.IPProtocol_UDP, 118 | CheckTimeoutMs: uint32(time.Second / time.Millisecond), 119 | } 120 | results := make(chan string, 0) 121 | 122 | port := 5120 123 | lAddr, err := net.ResolveUDPAddr("udp", fmt.Sprintf("127.0.0.1:%d", port)) 124 | c.Assert(err, NoErr) 125 | listener, err := net.ListenUDP("udp", lAddr) 126 | c.Assert(err, NoErr) 127 | defer listener.Close() 128 | 129 | go func() { 130 | buff := make([]byte, 512) 131 | n, err := listener.Read(buff) 132 | c.Assert(err, NoErr) 133 | results <- string(buff[:n]) 134 | }() 135 | 136 | syslogChecker, err := NewSyslogChecker(params, nil) 137 | c.Assert(err, NoErr) 138 | c.Assert(syslogChecker.Check("127.0.0.1", port), NoErr) 139 | select { 140 | case sentMsg := <-results: 141 | c.Assert(sentMsg, Matches, "<10>.*SyslogHealthChecker.*probe message.\n") 142 | case <-time.After(3 * time.Second): 143 | c.Log("Timeout to wait probe message") 144 | c.Fail() 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /kglb/utils/dns_resolver/dns_resolver.go: -------------------------------------------------------------------------------- 1 | package dns_resolver 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "strings" 7 | "sync" 8 | "time" 9 | 10 | "github.com/miekg/dns" 11 | 12 | "dropbox/kglb/common" 13 | pb "dropbox/proto/kglb" 14 | "godropbox/errors" 15 | ) 16 | 17 | // NOTE: in ideal case we want to add local cache with ttl for each record, but since we use 18 | // local unbound (with cache already) we can implement it later. 19 | 20 | const ( 21 | dnsTimeout = 1 * time.Second 22 | ) 23 | 24 | type DnsResolver interface { 25 | ResolveHost(hostname string, af pb.AddressFamily) (*pb.IP, error) 26 | } 27 | 28 | type query struct { 29 | hostname string 30 | af pb.AddressFamily 31 | } 32 | 33 | type dnsResolverImpl struct { 34 | serverAddr string 35 | searchDomain string 36 | client *dns.Client 37 | dnsCache map[query]pb.IP 38 | cacheMiss uint64 39 | cacheMutex sync.RWMutex 40 | } 41 | 42 | func NewDnsResolver( 43 | addrstr string, 44 | searchDomain string) (*dnsResolverImpl, error) { 45 | 46 | searchDomain = strings.TrimPrefix(searchDomain, ".") 47 | 48 | return &dnsResolverImpl{ 49 | serverAddr: addrstr, 50 | searchDomain: searchDomain, 51 | client: &dns.Client{ 52 | Net: "udp", 53 | Timeout: dnsTimeout, 54 | DialTimeout: dnsTimeout, 55 | ReadTimeout: dnsTimeout, 56 | WriteTimeout: dnsTimeout}, 57 | dnsCache: make(map[query]pb.IP), 58 | }, nil 59 | } 60 | 61 | func (r *dnsResolverImpl) makeFqdn(hostname string) string { 62 | if !dns.IsFqdn(hostname) { 63 | hostname = fmt.Sprintf("%s.%s", hostname, r.searchDomain) 64 | } 65 | return dns.Fqdn(hostname) 66 | } 67 | 68 | func (r *dnsResolverImpl) performDnsQuery(hostname string, qType uint16) (*dns.Msg, error) { 69 | m := &dns.Msg{} 70 | m.SetQuestion(hostname, qType) 71 | 72 | res, _, err := r.client.Exchange(m, r.serverAddr) 73 | if err != nil { 74 | return nil, errors.Wrapf(err, "failed to perform dns query for %s:", hostname) 75 | } 76 | 77 | if res.Rcode != dns.RcodeSuccess { 78 | return nil, errors.Newf("Non-success status code from DNS: %v", res.Rcode) 79 | } 80 | 81 | if len(res.Answer) != 1 { 82 | return nil, 83 | errors.Newf("Invalid DNS responses count for %s, expected 1, obtained %d", 84 | hostname, len(res.Answer)) 85 | } 86 | return res, nil 87 | } 88 | 89 | // Converts string represantation of the address into pb.IP structure, otherwise 90 | // returns error when it's not possible. 91 | func (r *dnsResolverImpl) strToIp(addr string) (*pb.IP, error) { 92 | ip := net.ParseIP(addr) 93 | if ip == nil { 94 | return nil, errors.Newf("cannot parse addr: %s", addr) 95 | } 96 | 97 | return common.NetIpToKglbAddr(ip), nil 98 | } 99 | 100 | func (r *dnsResolverImpl) ResolveHost(hostname string, af pb.AddressFamily) (*pb.IP, error) { 101 | // checking if hostname variable is already ip address. 102 | pbAddr, err := r.strToIp(hostname) 103 | if err == nil { 104 | switch pbAddr.GetAddress().(type) { 105 | case *pb.IP_Ipv4: 106 | if af == pb.AddressFamily_AF_INET { 107 | return pbAddr, nil 108 | } 109 | return nil, errors.Newf("fails to resolve (v4 -> v6): %v", hostname) 110 | case *pb.IP_Ipv6: 111 | if af == pb.AddressFamily_AF_INET6 { 112 | return pbAddr, nil 113 | } 114 | return nil, errors.Newf("fails to resolve (v6 -> v4): %v", hostname) 115 | default: 116 | return nil, errors.Newf("fails to resolve (unknown addr type): %v", hostname) 117 | } 118 | } 119 | 120 | // check if we have resolved this pair of hostname + af before. return if found 121 | // already resolved entry 122 | cacheKey := query{ 123 | hostname: hostname, 124 | af: af, 125 | } 126 | 127 | r.cacheMutex.RLock() 128 | if ip, exists := r.dnsCache[query{hostname: hostname, af: af}]; exists { 129 | r.cacheMutex.RUnlock() 130 | return &ip, nil 131 | } 132 | r.cacheMutex.RUnlock() 133 | 134 | qType := dns.TypeA 135 | hostname = r.makeFqdn(hostname) 136 | 137 | if af == pb.AddressFamily_AF_INET6 { 138 | qType = dns.TypeAAAA 139 | } 140 | 141 | res, err := r.performDnsQuery(hostname, qType) 142 | 143 | if err != nil { 144 | return nil, err 145 | } 146 | 147 | var addr net.IP 148 | var resolvedIp pb.IP 149 | 150 | switch af { 151 | case pb.AddressFamily_AF_INET: 152 | addr := res.Answer[0].(*dns.A).A 153 | resolvedIp.Address = &pb.IP_Ipv4{Ipv4: addr.String()} 154 | case pb.AddressFamily_AF_INET6: 155 | addr = res.Answer[0].(*dns.AAAA).AAAA 156 | resolvedIp.Address = &pb.IP_Ipv6{Ipv6: addr.String()} 157 | default: 158 | return nil, errors.Newf("Unknown AddressFamily: %v", af) 159 | } 160 | 161 | r.cacheMutex.Lock() 162 | defer r.cacheMutex.Unlock() 163 | 164 | r.cacheMiss++ 165 | r.dnsCache[cacheKey] = resolvedIp 166 | return &resolvedIp, nil 167 | } 168 | -------------------------------------------------------------------------------- /kglb/utils/health_manager/health_manager_state_test.go: -------------------------------------------------------------------------------- 1 | package health_manager 2 | 3 | import ( 4 | . "gopkg.in/check.v1" 5 | 6 | "dropbox/kglb/utils/discovery" 7 | . "godropbox/gocheck2" 8 | ) 9 | 10 | type HealthManagerStateSuite struct { 11 | } 12 | 13 | var _ = Suite(&HealthManagerStateSuite{}) 14 | 15 | func (m *HealthManagerStateSuite) TestContains(c *C) { 16 | c.Assert( 17 | HealthManagerState([]HealthManagerEntry{ 18 | NewHealthManagerEntry(false, "host1", 80), 19 | NewHealthManagerEntry(false, "host1", 81), 20 | NewHealthManagerEntry(false, "host2", 80), 21 | }).Contains(discovery.NewHostPort("host1", 80, true)), IsTrue) 22 | 23 | c.Assert( 24 | HealthManagerState([]HealthManagerEntry{ 25 | NewHealthManagerEntry(false, "host1", 80), 26 | NewHealthManagerEntry(false, "host1", 81), 27 | NewHealthManagerEntry(false, "host2", 80), 28 | }).Contains(discovery.NewHostPort("host2", 80, true)), IsTrue) 29 | 30 | c.Assert( 31 | HealthManagerState([]HealthManagerEntry{ 32 | NewHealthManagerEntry(false, "host1", 80), 33 | NewHealthManagerEntry(false, "host1", 81), 34 | NewHealthManagerEntry(false, "host2", 80), 35 | }).Contains(discovery.NewHostPort("host1", 82, true)), IsFalse) 36 | c.Assert( 37 | HealthManagerState([]HealthManagerEntry{ 38 | NewHealthManagerEntry(false, "host1", 80), 39 | NewHealthManagerEntry(false, "host1", 81), 40 | NewHealthManagerEntry(false, "host2", 80), 41 | }).Contains(discovery.NewHostPort("host3", 80, true)), IsFalse) 42 | } 43 | 44 | func (m *HealthManagerStateSuite) TestGetEntry(c *C) { 45 | testEntry := NewHealthManagerEntry(false, "host1", 80) 46 | c.Assert( 47 | HealthManagerState([]HealthManagerEntry{ 48 | testEntry, 49 | NewHealthManagerEntry(false, "host1", 81), 50 | NewHealthManagerEntry(false, "host2", 80), 51 | }).GetEntry(discovery.NewHostPort("host1", 80, true)).Equal(&testEntry), 52 | IsTrue) 53 | 54 | c.Assert( 55 | HealthManagerState([]HealthManagerEntry{ 56 | testEntry, 57 | NewHealthManagerEntry(false, "host1", 81), 58 | NewHealthManagerEntry(false, "host2", 80), 59 | }).GetEntry(discovery.NewHostPort("host3", 80, true)), IsNil) 60 | c.Assert( 61 | HealthManagerState([]HealthManagerEntry{ 62 | testEntry, 63 | NewHealthManagerEntry(false, "host1", 81), 64 | NewHealthManagerEntry(false, "host2", 80), 65 | }).GetEntry(discovery.NewHostPort("host2", 81, true)), IsNil) 66 | } 67 | 68 | func (m *HealthManagerStateSuite) TestString(c *C) { 69 | c.Assert( 70 | HealthManagerState([]HealthManagerEntry{ 71 | NewHealthManagerEntry(false, "host1", 80), 72 | NewHealthManagerEntry(true, "host1", 81), 73 | NewHealthManagerEntry(false, "host2", 80), 74 | }).String(), Equals, "[host1:80/false, host1:81/true, host2:80/false]") 75 | } 76 | 77 | func (m *HealthManagerStateSuite) TestIsHealthy(c *C) { 78 | c.Assert( 79 | HealthManagerState([]HealthManagerEntry{ 80 | NewHealthManagerEntry(false, "host1", 80), 81 | NewHealthManagerEntry(true, "host1", 81), 82 | NewHealthManagerEntry(false, "host2", 80), 83 | }).IsHealthy(), IsTrue) 84 | 85 | c.Assert( 86 | HealthManagerState([]HealthManagerEntry{ 87 | NewHealthManagerEntry(false, "host1", 80), 88 | NewHealthManagerEntry(false, "host1", 81), 89 | NewHealthManagerEntry(false, "host2", 80), 90 | }).IsHealthy(), IsFalse) 91 | } 92 | 93 | func (m *HealthManagerStateSuite) TestEqual(c *C) { 94 | // just different order. 95 | c.Assert( 96 | HealthManagerState([]HealthManagerEntry{ 97 | NewHealthManagerEntry(false, "host1", 80), 98 | NewHealthManagerEntry(true, "host1", 81), 99 | NewHealthManagerEntry(false, "host2", 80), 100 | }).Equal(HealthManagerState([]HealthManagerEntry{ 101 | NewHealthManagerEntry(true, "host1", 81), 102 | NewHealthManagerEntry(false, "host2", 80), 103 | NewHealthManagerEntry(false, "host1", 80), 104 | })), IsTrue) 105 | 106 | // different items. 107 | c.Assert( 108 | HealthManagerState([]HealthManagerEntry{ 109 | NewHealthManagerEntry(false, "host1", 80), 110 | NewHealthManagerEntry(true, "host1", 81), 111 | NewHealthManagerEntry(false, "host2", 80), 112 | }).Equal(HealthManagerState([]HealthManagerEntry{ 113 | NewHealthManagerEntry(true, "host1", 81), 114 | NewHealthManagerEntry(false, "host2", 80), 115 | NewHealthManagerEntry(false, "host3", 80), 116 | })), IsFalse) 117 | c.Assert( 118 | HealthManagerState([]HealthManagerEntry{ 119 | NewHealthManagerEntry(false, "host1", 80), 120 | NewHealthManagerEntry(true, "host1", 81), 121 | NewHealthManagerEntry(false, "host2", 80), 122 | }).Equal(HealthManagerState([]HealthManagerEntry{ 123 | NewHealthManagerEntry(true, "host1", 81), 124 | NewHealthManagerEntry(false, "host2", 80), 125 | NewHealthManagerEntry(false, "host1", 81), 126 | })), IsFalse) 127 | } 128 | -------------------------------------------------------------------------------- /kglb/control_plane/health_checker_factory.go: -------------------------------------------------------------------------------- 1 | package control_plane 2 | 3 | import ( 4 | "context" 5 | "net" 6 | "strconv" 7 | 8 | "dropbox/dlog" 9 | "dropbox/exclog" 10 | "dropbox/kglb/common" 11 | "dropbox/kglb/utils/fwmark" 12 | "dropbox/kglb/utils/health_checker" 13 | pb "dropbox/proto/kglb" 14 | "godropbox/errors" 15 | ) 16 | 17 | // Interface to create and update HealthChecker instances based on 18 | // BalancerConfig proto. 19 | type HealthCheckerFactory interface { 20 | // Returns new instance of HealthChecker by provided configuration. 21 | Checker(conf *pb.BalancerConfig) (health_checker.HealthChecker, error) 22 | } 23 | 24 | type BaseHealthCheckerFactoryParams struct { 25 | // Fwmark Manager. 26 | FwmarkManager *fwmark.Manager 27 | 28 | // Optional fields to explicitly specify source IP of outgoing health check 29 | // related connections, otherwise they will be assigned automatically by OS. 30 | SourceIPv4 net.IP 31 | SourceIPv6 net.IP 32 | } 33 | 34 | type baseHealthCheckerFactory struct { 35 | params *BaseHealthCheckerFactoryParams 36 | 37 | // func to establish new TCP connection to simplify testing. 38 | tcpConnFunc func( 39 | ctx context.Context, 40 | network string, 41 | localIp net.IP, 42 | address string, 43 | fwmark uint32) (net.Conn, error) 44 | } 45 | 46 | // Returns new instances of health checker factory. 47 | func NewHealthCheckerFactory(params BaseHealthCheckerFactoryParams) *baseHealthCheckerFactory { 48 | return newHealthCheckerFactory(fwmark.NewTcpConnection, ¶ms) 49 | } 50 | 51 | // Returns HealthChecker instance based on configuration. 52 | func (b *baseHealthCheckerFactory) Checker(conf *pb.BalancerConfig) (health_checker.HealthChecker, error) { 53 | var err error 54 | var dialContext health_checker.DialContextFunc 55 | 56 | // create custom dialContext if fwmarks are enabled for healthchecks 57 | if conf.GetEnableFwmarks() { 58 | dialContext, err = b.fwMarkDialContext(conf.GetLbService()) 59 | if err != nil { 60 | return nil, errors.Wrapf(err, "failed to create fwMarkDialContext for %+v", conf) 61 | } 62 | } 63 | 64 | healthChecker, err := health_checker.NewHealthChecker(conf.GetUpstreamChecker(), dialContext) 65 | if err != nil { 66 | return nil, err 67 | } 68 | 69 | return healthChecker, nil 70 | } 71 | 72 | // for testing. 73 | func newHealthCheckerFactory( 74 | tcpConnFunc func( 75 | ctx context.Context, 76 | network string, 77 | localIp net.IP, 78 | address string, 79 | fwmark uint32) (net.Conn, error), 80 | params *BaseHealthCheckerFactoryParams) *baseHealthCheckerFactory { 81 | 82 | return &baseHealthCheckerFactory{ 83 | params: params, 84 | tcpConnFunc: tcpConnFunc, 85 | } 86 | } 87 | 88 | // Generate DialContext func for fwmark service based on LoadBalancerService config. 89 | func (b *baseHealthCheckerFactory) fwMarkDialContext(lb *pb.LoadBalancerService) ( 90 | func( 91 | ctx context.Context, 92 | network string, 93 | address string) (net.Conn, error), error) { 94 | 95 | vip, vport, err := common.GetVipFromLbService(lb) 96 | if err != nil { 97 | err = errors.Wrapf( 98 | err, 99 | "fails to extract vip from lb_service: %+v, err: ", 100 | lb) 101 | exclog.Report(err, exclog.Critical, "") 102 | return nil, err 103 | } 104 | 105 | // determining addr family to bind proper src addr. 106 | localIp := b.params.SourceIPv4 107 | if net.ParseIP(vip).To4() == nil { 108 | localIp = b.params.SourceIPv6 109 | } 110 | 111 | if localIp == nil { 112 | return nil, errors.Newf("Can't determine local ip for healthcheck: %+v", b.params) 113 | } 114 | 115 | // connect to VIP with fwmark when it's enabled, otherwise establish 116 | // connection to auto-resolved ip. 117 | return func( 118 | ctx context.Context, 119 | network, 120 | address string) (net.Conn, error) { 121 | 122 | host, portStr, err := net.SplitHostPort(address) 123 | if err != nil { 124 | return nil, errors.Wrapf(err, "fails to parse address: %s, err: ", address) 125 | } 126 | port, err := strconv.Atoi(portStr) 127 | if err != nil { 128 | return nil, errors.Wrapf(err, "fails to parse address: %s, err: ", address) 129 | } 130 | 131 | fwmarkId, err := b.params.FwmarkManager.GetAllocatedFwmark(host) 132 | if err != nil { 133 | return nil, errors.Wrapf(err, "fails to get allocated fwmark for: %s, err: ", host) 134 | } 135 | 136 | // might be helpful to debug fwmark mismatch in health checks 137 | // and fwmark ipvs services. 138 | dlog.V(2).Infof("using fwmark: %d for %s:%d, vip: %s", fwmarkId, host, port, vip) 139 | 140 | address = net.JoinHostPort(vip, strconv.Itoa(vport)) 141 | conn, err := b.tcpConnFunc(ctx, network, localIp, address, fwmarkId) 142 | if err != nil { 143 | // providing more details along with exact error associated with 144 | // connection to simplify debugging. 145 | err = errors.Wrapf(err, 146 | "fails to connect using fwmark: %d for [%s]:%d, vip: %s, err: %v", 147 | fwmarkId, host, port, address, err) 148 | } 149 | return conn, err 150 | }, nil 151 | } 152 | 153 | var _ HealthCheckerFactory = &baseHealthCheckerFactory{} 154 | -------------------------------------------------------------------------------- /kglb/common/pretty.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "fmt" 5 | "sort" 6 | "strings" 7 | 8 | kglb_pb "dropbox/proto/kglb" 9 | "godropbox/errors" 10 | ) 11 | 12 | // Prettify DataPlaneState. 13 | func PrettyDataPlaneState(state *kglb_pb.DataPlaneState) (string, error) { 14 | 15 | var out string 16 | 17 | out = out + "----- Balancers\n" 18 | out = out + `Name Prot LocalAddress:Port Scheduler Flags 19 | -> RemoteAddress:Port Forward Weight` + "\n" 20 | 21 | for _, balancer := range state.GetBalancers() { 22 | // pretty lb. 23 | lb, err := PrettyLoadBalancerService(balancer.GetLbService()) 24 | if err != nil { 25 | return "", err 26 | } 27 | out = out + balancer.GetName() + " " + lb + "\n" 28 | 29 | // pretty upstreams after sorting. 30 | upstreams := balancer.GetUpstreams() 31 | // sorting by address. 32 | sort.Slice(upstreams, func(i, j int) bool { 33 | return upstreams[i].GetAddress().String() < 34 | upstreams[j].GetAddress().String() 35 | }) 36 | for _, up := range upstreams { 37 | pretty, err := PrettyUpstreamState(up) 38 | if err != nil { 39 | return "", err 40 | } 41 | out = out + fmt.Sprintf(" -> %s\n", pretty) 42 | } 43 | } 44 | 45 | // dump set of DynamicRoute. 46 | out = out + "----- Dynamic Route Map\n" 47 | out = out + "LocalAsn PeerAsn Community Prefix/Prefixlen\n" 48 | for _, routing := range state.GetDynamicRoutes() { 49 | pretty, err := PrettyDynamicRoute(routing) 50 | if err != nil { 51 | return "", err 52 | } 53 | out = out + " - " + pretty + "\n" 54 | } 55 | 56 | // dump set of LinkAddress. 57 | out = out + "----- Address Map\n" 58 | for _, link := range state.GetLinkAddresses() { 59 | pretty, err := PrettyLinkAddress(link) 60 | if err != nil { 61 | return "", err 62 | } 63 | out = out + " - " + pretty + "\n" 64 | } 65 | 66 | return out, nil 67 | } 68 | 69 | // Pretty LinkAddress struct. 70 | func PrettyDynamicRoute(routing *kglb_pb.DynamicRoute) (string, error) { 71 | // extract bgp. 72 | switch routing.Attributes.(type) { 73 | case *kglb_pb.DynamicRoute_BgpAttributes: 74 | bgp := routing.GetBgpAttributes() 75 | return fmt.Sprintf( 76 | "%d %d %s %s/%d", 77 | bgp.GetLocalAsn(), 78 | bgp.GetPeerAsn(), 79 | // replacing space separator in community to improve eye view of 80 | // the string since space is used in arg separation as well. 81 | strings.Replace(bgp.GetCommunity(), " ", ",", -1), 82 | KglbAddrToNetIp(bgp.GetPrefix()), 83 | bgp.GetPrefixlen()), nil 84 | default: 85 | return "", errors.Newf("Unknown routing attribute: %+v", routing) 86 | } 87 | } 88 | 89 | // Pretty LinkAddress struct. 90 | func PrettyLinkAddress(link *kglb_pb.LinkAddress) (string, error) { 91 | return fmt.Sprintf( 92 | "%s%%%s", 93 | KglbAddrToNetIp(link.GetAddress()), 94 | link.GetLinkName()), nil 95 | } 96 | 97 | // Pretty UpstreamState 98 | func PrettyUpstreamState(up *kglb_pb.UpstreamState) (string, error) { 99 | fw, err := PrettyForwardMethod(up.GetForwardMethod()) 100 | if err != nil { 101 | return "", err 102 | } 103 | return fmt.Sprintf( 104 | "%s|%s:%d %s %d", 105 | up.GetHostname(), 106 | KglbAddrToNetIp(up.GetAddress()), 107 | up.GetPort(), 108 | fw, 109 | up.GetWeight()), nil 110 | } 111 | 112 | // Pretty LoadBalancerService 113 | func PrettyLoadBalancerService(lb *kglb_pb.LoadBalancerService) (string, error) { 114 | 115 | var out string 116 | 117 | ipvsService, err := GetIpvsServiceFromLbService(lb) 118 | if err != nil { 119 | return "", err 120 | } 121 | 122 | // 1. pretty service. 123 | switch attr := ipvsService.Attributes.(type) { 124 | case *kglb_pb.IpvsService_TcpAttributes: 125 | out += fmt.Sprintf( 126 | "TCP %s:%d", 127 | KglbAddrToNetIp(attr.TcpAttributes.GetAddress()), 128 | uint16(attr.TcpAttributes.Port)) 129 | case *kglb_pb.IpvsService_UdpAttributes: 130 | out += fmt.Sprintf( 131 | "UDP %s:%d", 132 | KglbAddrToNetIp(attr.UdpAttributes.GetAddress()), 133 | uint16(attr.UdpAttributes.Port)) 134 | case *kglb_pb.IpvsService_FwmarkAttributes: 135 | out += fmt.Sprintf("FWM %d", attr.FwmarkAttributes.Fwmark) 136 | default: 137 | return "", errors.Newf("Unknown attributes type: %s", attr) 138 | } 139 | 140 | // 2. pretty scheduler 141 | scheduler, err := PrettyIpvsScheduler(ipvsService.Scheduler) 142 | if err != nil { 143 | return "", err 144 | } 145 | out = out + " " + scheduler 146 | 147 | return out, nil 148 | } 149 | 150 | // Pretty ForwardMethod. 151 | func PrettyForwardMethod(fw kglb_pb.ForwardMethods) (string, error) { 152 | switch fw { 153 | case kglb_pb.ForwardMethods_TUNNEL: 154 | return "Tunnel", nil 155 | case kglb_pb.ForwardMethods_MASQ: 156 | return "Masq", nil 157 | default: 158 | return "", errors.Newf("unknown ForwardMethod: %v", fw) 159 | } 160 | } 161 | 162 | // Pretty IpvsSchceduler. 163 | func PrettyIpvsScheduler(scheduler kglb_pb.IpvsService_Scheduler) (string, error) { 164 | switch scheduler { 165 | case kglb_pb.IpvsService_RR: 166 | return "rr", nil 167 | case kglb_pb.IpvsService_WRR: 168 | return "wrr", nil 169 | case kglb_pb.IpvsService_IP_VS_SCH: 170 | return "ip_vs_sch", nil 171 | default: 172 | return "", errors.Newf("unknown kglb ipvs scheduler: %v", scheduler) 173 | } 174 | } 175 | -------------------------------------------------------------------------------- /kglb/utils/concurrency/concurrency_test.go: -------------------------------------------------------------------------------- 1 | package concurrency 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync/atomic" 7 | "time" 8 | 9 | . "gopkg.in/check.v1" 10 | 11 | . "godropbox/gocheck2" 12 | ) 13 | 14 | type ConcurrencySuite struct { 15 | } 16 | 17 | var _ = Suite(&ConcurrencySuite{}) 18 | 19 | func (m *ConcurrencySuite) TestCompleteTasksSingleWorker(c *C) { 20 | errChan := make(chan error, 1) 21 | ctx, cancelFunc := context.WithCancel(context.Background()) 22 | defer cancelFunc() 23 | 24 | go func() { 25 | err := CompleteTasks( 26 | ctx, 27 | 1, 28 | 10, 29 | func(workerId int, taskId int) { 30 | if workerId > 0 { 31 | errChan <- fmt.Errorf("unexpected workerId: %d!=0", workerId) 32 | } 33 | }) 34 | if err != nil { 35 | errChan <- err 36 | } 37 | close(errChan) 38 | }() 39 | 40 | select { 41 | case err, ok := <-errChan: 42 | c.Assert(err, NoErr) 43 | c.Assert(ok, IsFalse) 44 | case <-time.After(time.Second): 45 | c.Log("fails to wait completion") 46 | c.Fail() 47 | } 48 | } 49 | 50 | // Validate distribution of tasks per worker (10 tasks by 2 workers). 51 | func (m *ConcurrencySuite) TestCompleteTasksMultiWorkerEvenTasks(c *C) { 52 | errChan := make(chan error, 1) 53 | ctx, cancelFunc := context.WithCancel(context.Background()) 54 | defer cancelFunc() 55 | 56 | // keep track of executed tasks by each worker. 57 | var workerExecs [2]uint32 58 | 59 | go func() { 60 | err := CompleteTasks( 61 | ctx, 62 | 2, 63 | 10, 64 | func(workerId int, taskId int) { 65 | if workerId > 1 { 66 | errChan <- fmt.Errorf("unexpected workerId: %d>1", workerId) 67 | } 68 | atomic.AddUint32(&workerExecs[workerId], 1) 69 | }) 70 | if err != nil { 71 | errChan <- err 72 | } 73 | close(errChan) 74 | }() 75 | 76 | select { 77 | case err, ok := <-errChan: 78 | c.Assert(err, NoErr) 79 | c.Assert(ok, IsFalse) 80 | case <-time.After(time.Second): 81 | c.Log("fails to wait completion") 82 | c.Fail() 83 | } 84 | c.Assert(workerExecs[0], Equals, uint32(5)) 85 | c.Assert(workerExecs[1], Equals, uint32(5)) 86 | } 87 | 88 | // Validate distribution of tasks per worker (10 tasks by 3 workers). 89 | func (m *ConcurrencySuite) TestCompleteTasksMultiWorkerUnevenTasks(c *C) { 90 | errChan := make(chan error, 1) 91 | ctx, cancelFunc := context.WithCancel(context.Background()) 92 | defer cancelFunc() 93 | 94 | // keep track of executed tasks by each worker. 95 | var workerExecs [3]uint32 96 | 97 | go func() { 98 | err := CompleteTasks( 99 | ctx, 100 | 3, 101 | 10, 102 | func(workerId int, taskId int) { 103 | if workerId > 2 { 104 | errChan <- fmt.Errorf("unexpected workerId: %d>2", workerId) 105 | } 106 | atomic.AddUint32(&workerExecs[workerId], 1) 107 | }) 108 | if err != nil { 109 | errChan <- err 110 | } 111 | close(errChan) 112 | }() 113 | 114 | select { 115 | case err, ok := <-errChan: 116 | c.Assert(err, NoErr) 117 | c.Assert(ok, IsFalse) 118 | case <-time.After(time.Second): 119 | c.Log("fails to wait completion") 120 | c.Fail() 121 | } 122 | c.Assert(workerExecs[0], Equals, uint32(4)) 123 | c.Assert(workerExecs[1], Equals, uint32(4)) 124 | c.Assert(workerExecs[2], Equals, uint32(2)) 125 | } 126 | 127 | func (m *ConcurrencySuite) TestCompleteTasksUnlimited(c *C) { 128 | errChan := make(chan error, 1) 129 | ctx, cancelFunc := context.WithCancel(context.Background()) 130 | defer cancelFunc() 131 | 132 | const numTasks = 10 133 | // keep track of executed tasks by each worker. 134 | var workerExecs [numTasks]uint32 135 | 136 | go func() { 137 | err := CompleteTasks( 138 | ctx, 139 | 0, 140 | numTasks, 141 | func(workerId int, taskId int) { 142 | if workerId > 9 { 143 | errChan <- fmt.Errorf("unexpected workerId: %d>9", workerId) 144 | } 145 | atomic.AddUint32(&workerExecs[workerId], 1) 146 | }) 147 | if err != nil { 148 | errChan <- err 149 | } 150 | close(errChan) 151 | }() 152 | 153 | select { 154 | case err, ok := <-errChan: 155 | c.Assert(err, NoErr) 156 | c.Assert(ok, IsFalse) 157 | case <-time.After(time.Second): 158 | c.Log("fails to wait completion") 159 | c.Fail() 160 | } 161 | 162 | for i := 0; i < numTasks; i++ { 163 | c.Assert(workerExecs[i], Equals, uint32(1)) 164 | } 165 | } 166 | 167 | func (m *ConcurrencySuite) TestContextDeadline(c *C) { 168 | errChan := make(chan error, 1) 169 | ctx, cancelFunc := context.WithTimeout(context.Background(), time.Millisecond) 170 | defer cancelFunc() 171 | 172 | const numTasks = 10 173 | go func() { 174 | err := CompleteTasks( 175 | ctx, 176 | 10, 177 | numTasks, 178 | func(workerId int, taskId int) { 179 | time.Sleep(time.Millisecond * 100) 180 | }) 181 | if err != nil { 182 | errChan <- err 183 | } 184 | close(errChan) 185 | }() 186 | 187 | select { 188 | case err, ok := <-errChan: 189 | c.Assert(err, ErrorMatches, "context deadline exceeded") 190 | c.Assert(ok, IsTrue) 191 | case <-time.After(time.Second): 192 | c.Log("fails to wait completion") 193 | c.Fail() 194 | } 195 | } 196 | 197 | func (m *ConcurrencySuite) TestContextCancelled(c *C) { 198 | errChan := make(chan error) 199 | ctx, cancelFunc := context.WithCancel(context.Background()) 200 | 201 | const numTasks = 10 202 | go func() { 203 | err := CompleteTasks( 204 | ctx, 205 | numTasks, 206 | numTasks, 207 | func(workerId int, taskId int) { 208 | time.Sleep(time.Second) 209 | }) 210 | errChan <- err 211 | }() 212 | 213 | cancelFunc() 214 | 215 | select { 216 | case err := <-errChan: 217 | c.Assert(err, ErrorMatches, "context canceled") 218 | case <-time.After(time.Second): 219 | c.Log("context is not canceled") 220 | c.Fail() 221 | } 222 | } 223 | -------------------------------------------------------------------------------- /kglb/utils/health_checker/tcp_checker_test.go: -------------------------------------------------------------------------------- 1 | package health_checker 2 | 3 | import ( 4 | "context" 5 | "net" 6 | "net/http" 7 | "net/http/httptest" 8 | "strconv" 9 | "sync/atomic" 10 | "time" 11 | 12 | . "gopkg.in/check.v1" 13 | 14 | "dropbox/kglb/utils/concurrency" 15 | "dropbox/kglb/utils/fwmark" 16 | hc_pb "dropbox/proto/kglb/healthchecker" 17 | . "godropbox/gocheck2" 18 | ) 19 | 20 | type TcpCheckerSuite struct{} 21 | 22 | var _ = Suite(&TcpCheckerSuite{}) 23 | 24 | func (s *TcpCheckerSuite) SetUpTest(c *C) {} 25 | 26 | func (s *TcpCheckerSuite) TestCheckHealthTcp(c *C) { 27 | server := httptest.NewServer(&backendHandler{ 28 | handler: func(writer http.ResponseWriter, req *http.Request) { 29 | writer.WriteHeader(http.StatusOK) 30 | }, 31 | }) 32 | defer server.Close() 33 | 34 | host, portStr, err := net.SplitHostPort(server.Listener.Addr().String()) 35 | c.Assert(err, NoErr) 36 | port, err := strconv.Atoi(portStr) 37 | c.Assert(err, NoErr) 38 | 39 | params := &hc_pb.TcpCheckerAttributes{ 40 | CheckTimeoutMs: uint32(time.Second / time.Millisecond), 41 | } 42 | checker, err := NewTcpChecker(params, nil) 43 | c.Assert(err, NoErr) 44 | 45 | c.Assert(checker.Check(host, port), NoErr) 46 | } 47 | 48 | func (s *TcpCheckerSuite) TestCheckHealthTcpRefuse(c *C) { 49 | // assuming noone listening on that port on test server 50 | port := 65533 51 | host := "127.0.0.1" 52 | params := &hc_pb.TcpCheckerAttributes{ 53 | CheckTimeoutMs: uint32(time.Second / time.Millisecond), 54 | } 55 | checker, err := NewTcpChecker(params, nil) 56 | c.Assert(err, NoErr) 57 | 58 | c.Assert(checker.Check(host, port), NotNil) 59 | } 60 | 61 | func (s *TcpCheckerSuite) TestCheckHealthFwGen(c *C) { 62 | server := httptest.NewServer(&backendHandler{ 63 | handler: func(writer http.ResponseWriter, req *http.Request) { 64 | writer.WriteHeader(http.StatusOK) 65 | }, 66 | }) 67 | defer server.Close() 68 | 69 | host, portStr, err := net.SplitHostPort(server.Listener.Addr().String()) 70 | c.Assert(err, NoErr) 71 | port, err := strconv.Atoi(portStr) 72 | c.Assert(err, NoErr) 73 | 74 | isCalled := uint32(0) 75 | 76 | params := &hc_pb.TcpCheckerAttributes{ 77 | CheckTimeoutMs: uint32(time.Second / time.Millisecond), 78 | } 79 | checker, err := NewTcpChecker(params, func(ctx context.Context, network, address string) (net.Conn, error) { 80 | atomic.AddUint32(&isCalled, 1) 81 | return fwmark.NewTcpConnection(ctx, network, net.IP{}, address, 0) 82 | }) 83 | c.Assert(err, NoErr) 84 | c.Assert(checker.Check(host, port), NoErr) 85 | 86 | // check that FwmarkGenerator was called. 87 | c.Assert(int(atomic.LoadUint32(&isCalled)), Equals, 1) 88 | 89 | // repeating with src addr. 90 | params = &hc_pb.TcpCheckerAttributes{ 91 | CheckTimeoutMs: uint32(time.Second / time.Millisecond), 92 | } 93 | checker, err = NewTcpChecker(params, func(ctx context.Context, network, address string) (net.Conn, error) { 94 | atomic.AddUint32(&isCalled, 1) 95 | return fwmark.NewTcpConnection(ctx, network, net.ParseIP("127.0.0.1"), address, 0) 96 | }) 97 | 98 | c.Assert(err, NoErr) 99 | c.Assert(checker.Check(host, port), NoErr) 100 | 101 | // check that FwmarkGenerator was called. 102 | c.Assert(int(atomic.LoadUint32(&isCalled)), Equals, 2) 103 | } 104 | 105 | func (s *TcpCheckerSuite) TestIpv6(c *C) { 106 | server := NewBackendWithCustomAddr( 107 | c, 108 | "[::1]:15101", 109 | func(writer http.ResponseWriter, req *http.Request) { 110 | writer.WriteHeader(http.StatusOK) 111 | }) 112 | defer server.Close() 113 | 114 | time.Sleep(50 * time.Millisecond) 115 | 116 | params := &hc_pb.TcpCheckerAttributes{ 117 | CheckTimeoutMs: uint32(time.Second / time.Millisecond), 118 | } 119 | checker, err := NewTcpChecker(params, nil) 120 | 121 | c.Assert(err, NoErr) 122 | c.Assert(checker.Check("::1", 15101), NoErr) 123 | 124 | // repeating with dial context. 125 | checker, err = NewTcpChecker(params, func(ctx context.Context, network, address string) (net.Conn, error) { 126 | return fwmark.NewTcpConnection(ctx, network, net.IP{}, address, 0) 127 | }) 128 | c.Assert(err, NoErr) 129 | c.Assert(checker.Check("::1", 15101), NoErr) 130 | 131 | // repeating with specified src addr. 132 | checker, err = NewTcpChecker(params, func(ctx context.Context, network, address string) (net.Conn, error) { 133 | return fwmark.NewTcpConnection(ctx, network, net.ParseIP("::1"), address, 0) 134 | }) 135 | c.Assert(err, NoErr) 136 | c.Assert(checker.Check("::1", 15101), NoErr) 137 | } 138 | 139 | func (s *TcpCheckerSuite) TestCheckHealthTcpConcurrent(c *C) { 140 | server := httptest.NewServer(&backendHandler{ 141 | handler: func(writer http.ResponseWriter, req *http.Request) { 142 | writer.WriteHeader(http.StatusOK) 143 | }, 144 | }) 145 | defer server.Close() 146 | 147 | host, portStr, err := net.SplitHostPort(server.Listener.Addr().String()) 148 | c.Assert(err, NoErr) 149 | port, err := strconv.Atoi(portStr) 150 | c.Assert(err, NoErr) 151 | 152 | params := &hc_pb.TcpCheckerAttributes{ 153 | CheckTimeoutMs: uint32(time.Second / time.Millisecond), 154 | } 155 | checker, err := NewTcpChecker(params, nil) 156 | 157 | c.Assert(err, NoErr) 158 | 159 | numWorkers := 10 160 | errChan := make(chan error, numWorkers) 161 | err = concurrency.CompleteTasks( 162 | context.Background(), 163 | numWorkers, 164 | numWorkers, 165 | func(numWorker int, numTask int) { 166 | errChan <- checker.Check(host, port) 167 | }) 168 | c.Assert(err, NoErr) 169 | 170 | for i := 0; i < numWorkers; i++ { 171 | select { 172 | case err, ok := <-errChan: 173 | c.Assert(ok, IsTrue) 174 | c.Assert(err, NoErr) 175 | case <-time.After(time.Second): 176 | c.Log("check took too much time...") 177 | c.Fail() 178 | } 179 | } 180 | } 181 | -------------------------------------------------------------------------------- /kglb/data_plane/manager_balancer.go: -------------------------------------------------------------------------------- 1 | package data_plane 2 | 3 | import ( 4 | "dropbox/dlog" 5 | "dropbox/kglb/common" 6 | kglb_pb "dropbox/proto/kglb" 7 | "godropbox/errors" 8 | ) 9 | 10 | type BalancerManagerParams struct { 11 | Ipvs IpvsModule 12 | // DNS related module. 13 | Resolver ResolverModule 14 | } 15 | 16 | type BalancerManager struct { 17 | params *BalancerManagerParams 18 | } 19 | 20 | func NewBalancerManager( 21 | params BalancerManagerParams) (*BalancerManager, error) { 22 | 23 | return &BalancerManager{ 24 | params: ¶ms, 25 | }, nil 26 | } 27 | 28 | // Get existent balancers. 29 | func (m *BalancerManager) GetBalancers() ([]*kglb_pb.BalancerState, error) { 30 | // return value. 31 | balancers := []*kglb_pb.BalancerState{} 32 | 33 | ipvsServices, _, err := m.params.Ipvs.ListServices() 34 | if err != nil { 35 | return nil, errors.Wrap( 36 | err, 37 | "GetBalancer() fails because of Ipvs.ListServices(): ") 38 | } 39 | 40 | for _, ipvsService := range ipvsServices { 41 | // find service name. 42 | serviceName := m.params.Resolver.ServiceLookup(ipvsService) 43 | 44 | // query upstreams. 45 | configuredReals, _, err := m.params.Ipvs.GetRealServers(ipvsService) 46 | if err != nil { 47 | return nil, errors.Wrap( 48 | err, 49 | "GetBalancer() fails because of GetRealServers(): ") 50 | } 51 | 52 | balancers = append(balancers, &kglb_pb.BalancerState{ 53 | Name: serviceName, 54 | LbService: &kglb_pb.LoadBalancerService{ 55 | Service: &kglb_pb.LoadBalancerService_IpvsService{ 56 | IpvsService: ipvsService, 57 | }, 58 | }, 59 | Upstreams: configuredReals, 60 | }) 61 | } 62 | 63 | return balancers, nil 64 | } 65 | 66 | // Add set of new balancers. 67 | func (m *BalancerManager) AddBalancers(balancers []*kglb_pb.BalancerState) error { 68 | for _, balancer := range balancers { 69 | if err := m.AddBalancer(balancer); err != nil { 70 | return errors.Wrapf(err, "failed to add balancer: %+v", balancer) 71 | } else { 72 | dlog.Infof("Balancer has been added: %s", balancer.GetName()) 73 | } 74 | } 75 | return nil 76 | } 77 | 78 | // Add new balancer. 79 | func (m *BalancerManager) AddBalancer(balancer *kglb_pb.BalancerState) error { 80 | // 1. extracting ipvs service. 81 | ipvsService, err := common.GetIpvsServiceFromBalancer(balancer) 82 | if err != nil { 83 | return errors.Wrapf(err, "fails to extract ipvs service from balancer: ") 84 | } 85 | 86 | // 2. adding ipvs service. 87 | if err := m.params.Ipvs.AddService(ipvsService); err != nil { 88 | return errors.Wrapf(err, "failed to create IPVS service: ") 89 | } 90 | 91 | // 3. adding reals. 92 | reals := balancer.GetUpstreams() 93 | if err := m.params.Ipvs.AddRealServers(ipvsService, reals); err != nil { 94 | return errors.Wrapf(err, "failed to add upstreams: ") 95 | } 96 | 97 | return nil 98 | } 99 | 100 | // Delete set of existent balancers. 101 | func (m *BalancerManager) DeleteBalancers(balancers []*kglb_pb.BalancerState) error { 102 | for _, balancer := range balancers { 103 | if err := m.DeleteBalancer(balancer); err != nil { 104 | return errors.Wrapf(err, "failed to delete balancer: %+v", balancer) 105 | } else { 106 | dlog.Infof("Balancer has been deleted : %s", balancer.GetName()) 107 | } 108 | } 109 | return nil 110 | } 111 | 112 | func (m *BalancerManager) DeleteBalancer(balancer *kglb_pb.BalancerState) error { 113 | // 1. extracting ipvs service. 114 | ipvsService, err := common.GetIpvsServiceFromBalancer(balancer) 115 | if err != nil { 116 | return errors.Wrapf(err, "fails to extract ipvs service from balancer: ") 117 | } 118 | 119 | // 2. deleting ipvs service. 120 | if err := m.params.Ipvs.DeleteService(ipvsService); err != nil { 121 | return errors.Wrapf(err, "failed to delete IPVS service: ") 122 | } 123 | return nil 124 | } 125 | 126 | // Add new upstreams for specific LoadBalancerService. 127 | func (m *BalancerManager) AddUpstreams( 128 | lbService *kglb_pb.LoadBalancerService, 129 | upstreams []*kglb_pb.UpstreamState) error { 130 | 131 | // 1. extracting ipvs service. 132 | ipvsService, err := common.GetIpvsServiceFromLbService(lbService) 133 | if err != nil { 134 | return errors.Wrapf(err, "fails to extract ipvs service from LoadBalancerService: ") 135 | } 136 | 137 | // 2. adding upstreams. 138 | if err := m.params.Ipvs.AddRealServers(ipvsService, upstreams); err != nil { 139 | return errors.Wrapf(err, "failed to add upstreams: ") 140 | } 141 | return nil 142 | } 143 | 144 | // Delete upstreams from specific LoadBalancerService. 145 | func (m *BalancerManager) DeleteUpstreams( 146 | lbService *kglb_pb.LoadBalancerService, 147 | upstreams []*kglb_pb.UpstreamState) error { 148 | 149 | // 1. extracting ipvs service. 150 | ipvsService, err := common.GetIpvsServiceFromLbService(lbService) 151 | if err != nil { 152 | return errors.Wrapf(err, "fails to extract ipvs service from LoadBalancerService: ") 153 | } 154 | 155 | // 2. deleting upstreams. 156 | if err := m.params.Ipvs.DeleteRealServers(ipvsService, upstreams); err != nil { 157 | return errors.Wrapf(err, "failed to delete upstreams: ") 158 | } 159 | return nil 160 | } 161 | 162 | // Update upstreams for specific LoadBalancerService. 163 | func (m *BalancerManager) UpdateUpstreams( 164 | lbService *kglb_pb.LoadBalancerService, 165 | upstreams []*kglb_pb.UpstreamState) error { 166 | 167 | // 1. extracting ipvs service. 168 | ipvsService, err := common.GetIpvsServiceFromLbService(lbService) 169 | if err != nil { 170 | return errors.Wrapf(err, "fails to extract ipvs service from LoadBalancerService: ") 171 | } 172 | 173 | // 2. updating upstreams. 174 | if err := m.params.Ipvs.UpdateRealServers(ipvsService, upstreams); err != nil { 175 | return errors.Wrapf(err, "failed to update upstreams: ") 176 | } 177 | 178 | return nil 179 | } 180 | -------------------------------------------------------------------------------- /kglb/data_plane/mock_modules.go: -------------------------------------------------------------------------------- 1 | package data_plane 2 | 3 | import ( 4 | "net" 5 | 6 | kglb_pb "dropbox/proto/kglb" 7 | ) 8 | 9 | // set of fake modules with no functionality inside, but to simplify 10 | // overriding required functions. 11 | 12 | // Bgp 13 | type MockBgpModule struct { 14 | InitFunc func(asn uint32) error 15 | AdvertiseFunc func(bgpConfig *kglb_pb.BgpRouteAttributes) error 16 | WithdrawFunc func(bgpConfig *kglb_pb.BgpRouteAttributes) error 17 | ListPathsFunc func() ([]*kglb_pb.BgpRouteAttributes, error) 18 | IsSessionEstablishedFunc func() (bool, error) 19 | } 20 | 21 | func (m *MockBgpModule) Init(asn uint32) error { 22 | if m.AdvertiseFunc == nil { 23 | return notImplErr 24 | } 25 | 26 | return m.InitFunc(asn) 27 | } 28 | 29 | func (m *MockBgpModule) Advertise(bgpConfig *kglb_pb.BgpRouteAttributes) error { 30 | if m.AdvertiseFunc == nil { 31 | return notImplErr 32 | } 33 | 34 | return m.AdvertiseFunc(bgpConfig) 35 | } 36 | 37 | func (m *MockBgpModule) Withdraw(bgpConfig *kglb_pb.BgpRouteAttributes) error { 38 | if m.WithdrawFunc == nil { 39 | return notImplErr 40 | } 41 | 42 | return m.WithdrawFunc(bgpConfig) 43 | } 44 | 45 | func (m *MockBgpModule) ListPaths() ([]*kglb_pb.BgpRouteAttributes, error) { 46 | if m.WithdrawFunc == nil { 47 | return nil, notImplErr 48 | } 49 | 50 | return m.ListPathsFunc() 51 | } 52 | 53 | func (m *MockBgpModule) IsSessionEstablished() (bool, error) { 54 | if m.IsSessionEstablishedFunc == nil { 55 | return false, notImplErr 56 | } 57 | return m.IsSessionEstablishedFunc() 58 | } 59 | 60 | // AddressTableModule. 61 | type MockAddressTableModule struct { 62 | AddFunc func(addr net.IP, iface string) error 63 | DeleteFunc func(addr net.IP, iface string) error 64 | IsExistsFunc func(addr net.IP, iface string) (bool, error) 65 | ListFunc func(iface string) ([]net.IP, error) 66 | } 67 | 68 | func (m *MockAddressTableModule) Add(addr net.IP, iface string) error { 69 | if m.AddFunc == nil { 70 | return notImplErr 71 | } 72 | 73 | return m.AddFunc(addr, iface) 74 | } 75 | 76 | func (m *MockAddressTableModule) Delete(addr net.IP, iface string) error { 77 | if m.DeleteFunc == nil { 78 | return notImplErr 79 | } 80 | 81 | return m.DeleteFunc(addr, iface) 82 | } 83 | 84 | func (m *MockAddressTableModule) IsExists(addr net.IP, iface string) (bool, error) { 85 | if m.IsExistsFunc == nil { 86 | return false, notImplErr 87 | } 88 | 89 | return m.IsExistsFunc(addr, iface) 90 | } 91 | 92 | func (m *MockAddressTableModule) List(iface string) ([]net.IP, error) { 93 | 94 | if m.ListFunc == nil { 95 | return nil, notImplErr 96 | } 97 | 98 | return m.ListFunc(iface) 99 | } 100 | 101 | type MockIpvsModule struct { 102 | AddServiceFunc func(service *kglb_pb.IpvsService) error 103 | // Delete ipvs service. 104 | DeleteServiceFunc func(service *kglb_pb.IpvsService) error 105 | // Get list of existent ipvs Services. 106 | ListServicesFunc func() ([]*kglb_pb.IpvsService, []*kglb_pb.Stats, error) 107 | 108 | // Add destination to the specific ipvs service. 109 | AddRealServerFunc func(service *kglb_pb.IpvsService, dst *kglb_pb.UpstreamState) error 110 | // Delete destination from specific ipvs service. 111 | DeleteRealServerFunc func(service *kglb_pb.IpvsService, dst *kglb_pb.UpstreamState) error 112 | // Update destination for specific ipvs service. 113 | UpdateRealServerFunc func(service *kglb_pb.IpvsService, dst *kglb_pb.UpstreamState) error 114 | // Get list of destinations of specific ipvs service 115 | GetRealServersFunc func( 116 | service *kglb_pb.IpvsService) ([]*kglb_pb.UpstreamState, []*kglb_pb.Stats, error) 117 | } 118 | 119 | func (m *MockIpvsModule) AddService(service *kglb_pb.IpvsService) error { 120 | if m.AddServiceFunc != nil { 121 | return m.AddServiceFunc(service) 122 | } 123 | return notImplErr 124 | } 125 | 126 | // Delete ipvs service. 127 | func (m *MockIpvsModule) DeleteService(service *kglb_pb.IpvsService) error { 128 | if m.DeleteServiceFunc != nil { 129 | return m.DeleteServiceFunc(service) 130 | } 131 | return notImplErr 132 | } 133 | 134 | // Get list of existent ipvs Services. 135 | func (m *MockIpvsModule) ListServices() ([]*kglb_pb.IpvsService, []*kglb_pb.Stats, error) { 136 | if m.ListServicesFunc != nil { 137 | return m.ListServicesFunc() 138 | } 139 | return nil, nil, notImplErr 140 | } 141 | 142 | // Add destination to the specific ipvs service. 143 | func (m *MockIpvsModule) AddRealServers( 144 | service *kglb_pb.IpvsService, dsts []*kglb_pb.UpstreamState) error { 145 | if m.AddRealServerFunc != nil { 146 | for _, dst := range dsts { 147 | if err := m.AddRealServerFunc(service, dst); err != nil { 148 | return err 149 | } 150 | } 151 | return nil 152 | } 153 | return notImplErr 154 | } 155 | 156 | // Delete destination from specific ipvs service. 157 | func (m *MockIpvsModule) DeleteRealServers( 158 | service *kglb_pb.IpvsService, dsts []*kglb_pb.UpstreamState) error { 159 | if m.DeleteRealServerFunc != nil { 160 | for _, dst := range dsts { 161 | if err := m.DeleteRealServerFunc(service, dst); err != nil { 162 | return err 163 | } 164 | } 165 | return nil 166 | } 167 | return notImplErr 168 | } 169 | 170 | // Update destination for specific ipvs service. 171 | func (m *MockIpvsModule) UpdateRealServers( 172 | service *kglb_pb.IpvsService, dsts []*kglb_pb.UpstreamState) error { 173 | if m.UpdateRealServerFunc != nil { 174 | for _, dst := range dsts { 175 | if err := m.UpdateRealServerFunc(service, dst); err != nil { 176 | return err 177 | } 178 | } 179 | return nil 180 | } 181 | return notImplErr 182 | } 183 | 184 | // Get list of destinations of specific ipvs service 185 | func (m *MockIpvsModule) GetRealServers( 186 | service *kglb_pb.IpvsService) ([]*kglb_pb.UpstreamState, []*kglb_pb.Stats, error) { 187 | 188 | if m.GetRealServersFunc != nil { 189 | return m.GetRealServersFunc(service) 190 | } 191 | return nil, nil, notImplErr 192 | } 193 | 194 | var _ IpvsModule = &MockIpvsModule{} 195 | -------------------------------------------------------------------------------- /kglb/data_plane/manager_address.go: -------------------------------------------------------------------------------- 1 | package data_plane 2 | 3 | import ( 4 | "dropbox/exclog" 5 | "dropbox/kglb/common" 6 | kglb_pb "dropbox/proto/kglb" 7 | "dropbox/vortex2/v2stats" 8 | "godropbox/errors" 9 | ) 10 | 11 | type AddressManagerParams struct { 12 | // netlink/addresses. 13 | AddressTable AddressTableModule 14 | } 15 | 16 | type AddressManager struct { 17 | params *AddressManagerParams 18 | // map of added addresses to maintain link state. 19 | state map[string]*kglb_pb.LinkAddress 20 | } 21 | 22 | func NewAddressManager( 23 | params AddressManagerParams) (*AddressManager, error) { 24 | 25 | return &AddressManager{ 26 | params: ¶ms, 27 | state: make(map[string]*kglb_pb.LinkAddress), 28 | }, nil 29 | } 30 | 31 | func (m *AddressManager) AddAddresses(balancers []*kglb_pb.LinkAddress) error { 32 | for _, balancer := range balancers { 33 | err := m.AddAddress(balancer) 34 | if err != nil { 35 | return err 36 | } 37 | } 38 | return nil 39 | } 40 | 41 | // Add IP address to the loopback for the balancer if it's needed. 42 | func (m *AddressManager) AddAddress(linkAddress *kglb_pb.LinkAddress) (err error) { 43 | address := linkAddress.GetAddress() 44 | serviceAddr := common.KglbAddrToNetIp(address) 45 | 46 | defer func() { 47 | if err != nil { 48 | gauge, err2 := linkAddressGauge.V(v2stats.KV{ 49 | "address": serviceAddr.String(), 50 | "state": "add_failed", 51 | }) 52 | if err2 == nil { 53 | gauge.Set(1) 54 | } else { 55 | exclog.Report(errors.Wrap(err2, 56 | "unable to instantiate linkAddressGauge"), exclog.Critical, "") 57 | } 58 | } 59 | }() 60 | 61 | iface := linkAddress.GetLinkName() 62 | exists, err := m.params.AddressTable.IsExists(serviceAddr, iface) 63 | if err != nil { 64 | return errors.Wrapf( 65 | err, 66 | "failed to check if address is configured: %v: ", 67 | serviceAddr) 68 | } 69 | 70 | if !exists { 71 | err := m.params.AddressTable.Add(serviceAddr, iface) 72 | if err != nil { 73 | return errors.Wrapf( 74 | err, 75 | "failed to add VIP address to interface: %v", 76 | serviceAddr) 77 | } 78 | } 79 | 80 | gauge, err2 := linkAddressGauge.V(v2stats.KV{ 81 | "address": serviceAddr.String(), 82 | "state": "alive", 83 | }) 84 | if err2 == nil { 85 | gauge.Set(1) 86 | } else { 87 | exclog.Report(errors.Wrap(err2, 88 | "unable to instantiate linkAddressGauge"), exclog.Critical, "") 89 | } 90 | 91 | // updating internal map to track added addresses. 92 | m.state[linkAddress.String()] = linkAddress 93 | 94 | return nil 95 | } 96 | 97 | func (m *AddressManager) DeleteAddresses(addresses []*kglb_pb.LinkAddress) error { 98 | for _, address := range addresses { 99 | err := m.DeleteAddress(address) 100 | if err != nil { 101 | return err 102 | } 103 | } 104 | return nil 105 | } 106 | 107 | func (m *AddressManager) DeleteAddress(linkAddress *kglb_pb.LinkAddress) (err error) { 108 | address := linkAddress.GetAddress() 109 | serviceAddr := common.KglbAddrToNetIp(address) 110 | 111 | defer func() { 112 | if err != nil { 113 | gauge, err2 := linkAddressGauge.V(v2stats.KV{ 114 | "address": serviceAddr.String(), 115 | "state": "delete_failed", 116 | }) 117 | if err2 == nil { 118 | gauge.Set(1) 119 | } else { 120 | exclog.Report(errors.Wrap(err2, 121 | "unable to instantiate linkAddressGauge"), exclog.Critical, "") 122 | } 123 | } 124 | }() 125 | 126 | iface := linkAddress.GetLinkName() 127 | 128 | exists, err := m.params.AddressTable.IsExists(serviceAddr, iface) 129 | if err != nil { 130 | return errors.Wrapf( 131 | err, 132 | "failed to check if address is configured: %v: ", 133 | serviceAddr) 134 | } 135 | if exists { 136 | err := m.params.AddressTable.Delete(serviceAddr, iface) 137 | if err != nil { 138 | return errors.Wrapf( 139 | err, 140 | "failed to delete VIP address to interface: %v", 141 | serviceAddr) 142 | } 143 | } 144 | 145 | gauge, err2 := linkAddressGauge.V(v2stats.KV{ 146 | "address": serviceAddr.String(), 147 | "state": "alive", 148 | }) 149 | if err2 == nil { 150 | gauge.Set(0) 151 | } else { 152 | exclog.Report(errors.Wrap(err2, 153 | "unable to instantiate linkAddressGauge"), exclog.Critical, "") 154 | } 155 | 156 | // updating internal map to track added addresses. 157 | if _, ok := m.state[linkAddress.String()]; ok { 158 | delete(m.state, linkAddress.String()) 159 | } 160 | 161 | return nil 162 | } 163 | 164 | // deadcode: will be in use to dump addresses. 165 | func (m *AddressManager) ListAddresses(iface string) ([]*kglb_pb.LinkAddress, error) { 166 | addresses, err := m.params.AddressTable.List(iface) 167 | if err != nil { 168 | return nil, errors.Wrapf( 169 | err, 170 | "failed to check if address is configured:", 171 | ) 172 | } 173 | result := make([]*kglb_pb.LinkAddress, len(addresses)) 174 | for i, address := range addresses { 175 | result[i] = &kglb_pb.LinkAddress{LinkName: iface, Address: common.NetIpToKglbAddr(address)} 176 | } 177 | return result, nil 178 | } 179 | 180 | // returns list of addresses added through AddressManager API and which are still 181 | // active (not deleted). 182 | func (m *AddressManager) State() ([]*kglb_pb.LinkAddress, error) { 183 | var state []*kglb_pb.LinkAddress 184 | for _, link := range m.state { 185 | // double checking that address is still existent. 186 | serviceAddr := common.KglbAddrToNetIp(link.GetAddress()) 187 | exists, err := m.params.AddressTable.IsExists(serviceAddr, link.GetLinkName()) 188 | if err != nil { 189 | return nil, errors.Wrapf( 190 | err, 191 | "failed to check if address is configured: %+v: ", 192 | link) 193 | } 194 | 195 | gauge, err := linkAddressGauge.V(v2stats.KV{ 196 | "address": serviceAddr.String(), 197 | "state": "alive", 198 | }) 199 | if err != nil { 200 | exclog.Report(errors.Wrap(err, 201 | "unable to instantiate linkAddressGauge"), exclog.Critical, "") 202 | } 203 | if exists { 204 | gauge.Set(1) 205 | state = append(state, link) 206 | } else { 207 | gauge.Clear() 208 | } 209 | } 210 | 211 | return state, nil 212 | } 213 | -------------------------------------------------------------------------------- /kglb/utils/health_checker/dns_checker_test.go: -------------------------------------------------------------------------------- 1 | package health_checker 2 | 3 | import ( 4 | "context" 5 | "net" 6 | "strconv" 7 | "strings" 8 | "time" 9 | 10 | "github.com/miekg/dns" 11 | . "gopkg.in/check.v1" 12 | 13 | "dropbox/kglb/utils/fwmark" 14 | hc_pb "dropbox/proto/kglb/healthchecker" 15 | . "godropbox/gocheck2" 16 | ) 17 | 18 | type DnsCheckerSuite struct { 19 | } 20 | 21 | var _ = Suite(&DnsCheckerSuite{}) 22 | 23 | // Validating Timeout functionality. 24 | func (s *DnsCheckerSuite) TestTimeout(c *C) { 25 | lAddr, addrErr := net.ResolveUDPAddr("udp", "127.0.0.1:0") 26 | c.Assert(addrErr, NoErr) 27 | listener, addrErr := net.ListenUDP("udp", lAddr) 28 | c.Assert(addrErr, NoErr) 29 | defer listener.Close() 30 | 31 | // getting port 32 | addr := listener.LocalAddr() 33 | host, portStr, err := net.SplitHostPort(addr.String()) 34 | c.Assert(err, NoErr) 35 | port, err := strconv.Atoi(portStr) 36 | c.Assert(err, NoErr) 37 | 38 | params := &hc_pb.DnsCheckerAttributes{ 39 | Protocol: hc_pb.IPProtocol_UDP, 40 | QueryType: hc_pb.DnsCheckerAttributes_A, 41 | QueryString: ".", 42 | Rcode: 0, 43 | CheckTimeoutMs: 1000, 44 | } 45 | 46 | checker, err := NewDnsChecker(params, nil) 47 | c.Assert(err, NoErr) 48 | 49 | startTime := time.Now() 50 | // making request to custom listener which does nothing with received packet. 51 | err = checker.Check(host, port) 52 | elapsed := time.Since(startTime) 53 | c.Assert(err, NotNil) 54 | c.Assert(elapsed/time.Millisecond, GreaterThan, 50) 55 | } 56 | 57 | // Validating custom DialContext. 58 | func (s *DnsCheckerSuite) TestCustomDialerContext(c *C) { 59 | // construct dns server. 60 | dnsSrv := &DnsHandler{ 61 | Handler: func(w dns.ResponseWriter, r *dns.Msg) { 62 | // generate response. 63 | r.SetRcode(r, dns.RcodeSuccess) 64 | w.WriteMsg(r) 65 | }, 66 | } 67 | 68 | // start dns server. 69 | dnsSrv.ListenAndServe("udp", "127.1.2.3:1028") 70 | defer dnsSrv.Close() 71 | time.Sleep(50 * time.Millisecond) 72 | 73 | params := &hc_pb.DnsCheckerAttributes{ 74 | Protocol: hc_pb.IPProtocol_UDP, 75 | QueryType: hc_pb.DnsCheckerAttributes_A, 76 | QueryString: ".", 77 | Rcode: 0, 78 | CheckTimeoutMs: 51, 79 | } 80 | 81 | checker, err := NewDnsChecker(params, func(ctx context.Context, network, address string) (net.Conn, error) { 82 | dialer := net.Dialer{} 83 | return dialer.DialContext(ctx, network, "127.1.2.3:1028") 84 | }) 85 | c.Assert(err, NoErr) 86 | 87 | err = checker.Check("1.1.1.1", 1000) 88 | c.Assert(err, NoErr) 89 | } 90 | 91 | // Validating Timeout functionality. 92 | func (s *DnsCheckerSuite) TestBasicTcp(c *C) { 93 | // construct dns server. 94 | dnsSrv := &DnsHandler{ 95 | Handler: func(w dns.ResponseWriter, r *dns.Msg) { 96 | // generate response. 97 | r.SetRcode(r, dns.RcodeSuccess) 98 | w.WriteMsg(r) 99 | }, 100 | } 101 | 102 | // start dns server. 103 | dnsSrv.ListenAndServe("tcp", "127.1.2.3:1026") 104 | defer dnsSrv.Close() 105 | 106 | params := &hc_pb.DnsCheckerAttributes{ 107 | Protocol: hc_pb.IPProtocol_TCP, 108 | QueryType: hc_pb.DnsCheckerAttributes_A, 109 | QueryString: ".", 110 | Rcode: 0, 111 | CheckTimeoutMs: uint32(time.Second / time.Millisecond), 112 | } 113 | 114 | checker, err := NewDnsChecker(params, nil) 115 | c.Assert(err, NoErr) 116 | 117 | // repeating few times since server listener may not be ready fast enough. 118 | for i := 0; i < 5; i++ { 119 | if err := checker.Check("127.1.2.3", 1026); err != nil { 120 | time.Sleep(50 * time.Millisecond) 121 | } else { 122 | break 123 | } 124 | } 125 | err = checker.Check("127.1.2.3", 1026) 126 | c.Assert(err, NoErr) 127 | 128 | checker, err = NewDnsChecker(params, func(ctx context.Context, network, address string) (net.Conn, error) { 129 | return fwmark.NewTcpConnection(ctx, network, net.IP{}, "127.1.2.3:1026", 0) 130 | }) 131 | 132 | c.Assert(err, NoErr) 133 | err = checker.Check("127.1.2.3", 1026) 134 | c.Assert(err, NoErr) 135 | } 136 | 137 | func (s *DnsCheckerSuite) TestBasicUdp(c *C) { 138 | // construct dns server. 139 | dnsSrv := &DnsHandler{ 140 | Handler: func(w dns.ResponseWriter, r *dns.Msg) { 141 | // generate response. 142 | r.SetRcode(r, dns.RcodeSuccess) 143 | w.WriteMsg(r) 144 | }, 145 | } 146 | 147 | // start dns server. 148 | dnsSrv.ListenAndServe("udp", "127.1.2.3:1027") 149 | defer dnsSrv.Close() 150 | 151 | params := &hc_pb.DnsCheckerAttributes{ 152 | Protocol: hc_pb.IPProtocol_UDP, 153 | QueryType: hc_pb.DnsCheckerAttributes_A, 154 | QueryString: ".", 155 | Rcode: 0, 156 | CheckTimeoutMs: uint32(time.Second / time.Millisecond), 157 | } 158 | 159 | checker, err := NewDnsChecker(params, nil) 160 | c.Assert(err, NoErr) 161 | 162 | // repeating few times since server listener may not be ready fast enough. 163 | for i := 0; i < 5; i++ { 164 | err := checker.Check("127.1.2.3", 1027) 165 | if err != nil { 166 | time.Sleep(50 * time.Millisecond) 167 | } else { 168 | break 169 | } 170 | } 171 | err = checker.Check("127.1.2.3", 1027) 172 | c.Assert(err, NoErr) 173 | } 174 | 175 | func (s *DnsCheckerSuite) TestBasicFailure(c *C) { 176 | // construct dns server. 177 | dnsSrv := &DnsHandler{ 178 | Handler: func(w dns.ResponseWriter, r *dns.Msg) { 179 | // generate response. 180 | r.SetRcode(r, dns.RcodeServerFailure) 181 | w.WriteMsg(r) 182 | }, 183 | } 184 | 185 | // start dns server. 186 | dnsSrv.ListenAndServe("udp", "127.1.2.3:1028") 187 | defer dnsSrv.Close() 188 | 189 | params := &hc_pb.DnsCheckerAttributes{ 190 | Protocol: hc_pb.IPProtocol_UDP, 191 | QueryType: hc_pb.DnsCheckerAttributes_A, 192 | QueryString: ".", 193 | Rcode: 0, 194 | CheckTimeoutMs: uint32(time.Second / time.Millisecond), 195 | } 196 | 197 | checker, err := NewDnsChecker(params, nil) 198 | c.Assert(err, NoErr) 199 | 200 | // repeating few times since server listener may not be ready fast enough. 201 | for i := 0; i < 5; i++ { 202 | err := checker.Check("127.1.2.3", 1028) 203 | if err != nil { 204 | if strings.Contains(err.Error(), "incorrect rcode") { 205 | return 206 | } 207 | time.Sleep(50 * time.Millisecond) 208 | } 209 | } 210 | err = checker.Check("127.1.2.3", 1028) 211 | c.Assert(strings.Contains(err.Error(), "incorrect rcode"), IsTrue) 212 | } 213 | -------------------------------------------------------------------------------- /kglb/common/pretty_test.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | . "gopkg.in/check.v1" 5 | 6 | kglb_pb "dropbox/proto/kglb" 7 | ) 8 | 9 | type PrettySuite struct { 10 | } 11 | 12 | var _ = Suite(&PrettySuite{}) 13 | 14 | func (m *PrettySuite) TestPrettyScheduler(c *C) { 15 | pretty, err := PrettyIpvsScheduler(kglb_pb.IpvsService_RR) 16 | c.Assert(err, IsNil) 17 | c.Assert(pretty, Equals, "rr") 18 | } 19 | 20 | func (m *PrettySuite) TestPrettyForwardMethod(c *C) { 21 | pretty, err := PrettyForwardMethod(kglb_pb.ForwardMethods_TUNNEL) 22 | c.Assert(err, IsNil) 23 | c.Assert(pretty, Equals, "Tunnel") 24 | } 25 | 26 | func (m *PrettySuite) TestPrettyUpstreamState(c *C) { 27 | pretty, err := PrettyUpstreamState(&kglb_pb.UpstreamState{ 28 | Address: &kglb_pb.IP{Address: &kglb_pb.IP_Ipv4{Ipv4: "10.0.0.1"}}, 29 | Port: 443, 30 | Hostname: "hostname1", 31 | Weight: 50, 32 | ForwardMethod: kglb_pb.ForwardMethods_TUNNEL, 33 | }) 34 | c.Assert(err, IsNil) 35 | c.Assert(pretty, Equals, "hostname1|10.0.0.1:443 Tunnel 50") 36 | } 37 | 38 | func (m *PrettySuite) TestPrettyLoadBalancerService(c *C) { 39 | pretty, err := PrettyLoadBalancerService( 40 | &kglb_pb.LoadBalancerService{ 41 | Service: &kglb_pb.LoadBalancerService_IpvsService{ 42 | IpvsService: &kglb_pb.IpvsService{ 43 | Attributes: &kglb_pb.IpvsService_TcpAttributes{ 44 | TcpAttributes: &kglb_pb.IpvsTcpAttributes{ 45 | Address: &kglb_pb.IP{Address: &kglb_pb.IP_Ipv4{Ipv4: "172.0.0.1"}}, 46 | Port: 443, 47 | }, 48 | }, 49 | Scheduler: kglb_pb.IpvsService_RR, 50 | }, 51 | }, 52 | }) 53 | c.Assert(err, IsNil) 54 | c.Assert(pretty, Equals, "TCP 172.0.0.1:443 rr") 55 | } 56 | 57 | func (m *PrettySuite) TestPrettyLinkAddress(c *C) { 58 | pretty, err := PrettyLinkAddress(&kglb_pb.LinkAddress{ 59 | LinkName: "lo", 60 | Address: &kglb_pb.IP{Address: &kglb_pb.IP_Ipv4{Ipv4: "172.0.0.1"}}, 61 | }) 62 | c.Assert(err, IsNil) 63 | c.Assert(pretty, Equals, "172.0.0.1%lo") 64 | } 65 | 66 | func (m *PrettySuite) TestPrettyDynamicRoute(c *C) { 67 | pretty, err := PrettyDynamicRoute(&kglb_pb.DynamicRoute{ 68 | Attributes: &kglb_pb.DynamicRoute_BgpAttributes{ 69 | BgpAttributes: &kglb_pb.BgpRouteAttributes{ 70 | LocalAsn: 10, 71 | PeerAsn: 20, 72 | Community: "my_community", 73 | Prefix: &kglb_pb.IP{ 74 | Address: &kglb_pb.IP_Ipv4{Ipv4: "10.0.0.2"}, 75 | }, 76 | Prefixlen: 32, 77 | }, 78 | }, 79 | }) 80 | c.Assert(err, IsNil) 81 | c.Assert(pretty, Equals, "10 20 my_community 10.0.0.2/32") 82 | } 83 | 84 | func (m *PrettySuite) TestPrettyDataPlaneState(c *C) { 85 | pretty, err := PrettyDataPlaneState(&kglb_pb.DataPlaneState{ 86 | Balancers: []*kglb_pb.BalancerState{ 87 | { 88 | Name: "TestName1", 89 | LbService: &kglb_pb.LoadBalancerService{Service: &kglb_pb.LoadBalancerService_IpvsService{ 90 | IpvsService: &kglb_pb.IpvsService{ 91 | Attributes: &kglb_pb.IpvsService_TcpAttributes{ 92 | TcpAttributes: &kglb_pb.IpvsTcpAttributes{ 93 | Address: &kglb_pb.IP{Address: &kglb_pb.IP_Ipv4{Ipv4: "172.0.0.1"}}, 94 | Port: 443, 95 | }}, 96 | Scheduler: kglb_pb.IpvsService_RR, 97 | }}}, 98 | Upstreams: []*kglb_pb.UpstreamState{ 99 | { 100 | Address: &kglb_pb.IP{Address: &kglb_pb.IP_Ipv4{Ipv4: "10.0.0.2"}}, 101 | Port: 443, 102 | Hostname: "hostname2", 103 | Weight: 50, 104 | ForwardMethod: kglb_pb.ForwardMethods_TUNNEL, 105 | }, 106 | { 107 | Address: &kglb_pb.IP{Address: &kglb_pb.IP_Ipv4{Ipv4: "10.0.0.1"}}, 108 | Port: 443, 109 | Hostname: "hostname1", 110 | Weight: 50, 111 | ForwardMethod: kglb_pb.ForwardMethods_TUNNEL, 112 | }, 113 | }, 114 | }, 115 | { 116 | Name: "TestName2", 117 | LbService: &kglb_pb.LoadBalancerService{Service: &kglb_pb.LoadBalancerService_IpvsService{ 118 | IpvsService: &kglb_pb.IpvsService{ 119 | Attributes: &kglb_pb.IpvsService_TcpAttributes{ 120 | TcpAttributes: &kglb_pb.IpvsTcpAttributes{ 121 | Address: &kglb_pb.IP{Address: &kglb_pb.IP_Ipv4{Ipv4: "172.0.0.2"}}, 122 | Port: 443, 123 | }}, 124 | Scheduler: kglb_pb.IpvsService_RR, 125 | }}}, 126 | Upstreams: []*kglb_pb.UpstreamState{ 127 | { 128 | Address: &kglb_pb.IP{Address: &kglb_pb.IP_Ipv4{Ipv4: "10.0.0.2"}}, 129 | Port: 443, 130 | Hostname: "hostname2", 131 | Weight: 50, 132 | ForwardMethod: kglb_pb.ForwardMethods_TUNNEL, 133 | }, 134 | }, 135 | }, 136 | }, 137 | DynamicRoutes: []*kglb_pb.DynamicRoute{ 138 | { 139 | Attributes: &kglb_pb.DynamicRoute_BgpAttributes{ 140 | BgpAttributes: &kglb_pb.BgpRouteAttributes{ 141 | LocalAsn: 10, 142 | PeerAsn: 20, 143 | Community: "65101:30000 65102:10090 65103:10000", 144 | Prefix: &kglb_pb.IP{ 145 | Address: &kglb_pb.IP_Ipv4{Ipv4: "10.0.0.2"}, 146 | }, 147 | Prefixlen: 32, 148 | }, 149 | }, 150 | }, 151 | { 152 | Attributes: &kglb_pb.DynamicRoute_BgpAttributes{ 153 | BgpAttributes: &kglb_pb.BgpRouteAttributes{ 154 | LocalAsn: 30, 155 | PeerAsn: 40, 156 | Community: "65101:30000 65102:10090 65103:10000", 157 | Prefix: &kglb_pb.IP{ 158 | Address: &kglb_pb.IP_Ipv4{Ipv4: "10.0.0.5"}, 159 | }, 160 | Prefixlen: 32, 161 | }, 162 | }, 163 | }, 164 | }, 165 | LinkAddresses: []*kglb_pb.LinkAddress{ 166 | { 167 | LinkName: "lo", 168 | Address: &kglb_pb.IP{Address: &kglb_pb.IP_Ipv4{Ipv4: "172.0.0.1"}}, 169 | }, 170 | { 171 | LinkName: "lo", 172 | Address: &kglb_pb.IP{Address: &kglb_pb.IP_Ipv4{Ipv4: "172.0.0.2"}}, 173 | }, 174 | }, 175 | }) 176 | c.Assert(err, IsNil) 177 | c.Assert(pretty, Equals, `----- Balancers 178 | Name Prot LocalAddress:Port Scheduler Flags 179 | -> RemoteAddress:Port Forward Weight 180 | TestName1 TCP 172.0.0.1:443 rr 181 | -> hostname1|10.0.0.1:443 Tunnel 50 182 | -> hostname2|10.0.0.2:443 Tunnel 50 183 | TestName2 TCP 172.0.0.2:443 rr 184 | -> hostname2|10.0.0.2:443 Tunnel 50 185 | ----- Dynamic Route Map 186 | LocalAsn PeerAsn Community Prefix/Prefixlen 187 | - 10 20 65101:30000,65102:10090,65103:10000 10.0.0.2/32 188 | - 30 40 65101:30000,65102:10090,65103:10000 10.0.0.5/32 189 | ----- Address Map 190 | - 172.0.0.1%lo 191 | - 172.0.0.2%lo 192 | `) 193 | } 194 | -------------------------------------------------------------------------------- /kglb/common/data_types.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | 7 | "github.com/gogo/protobuf/proto" 8 | "godropbox/errors" 9 | 10 | kglb_pb "dropbox/proto/kglb" 11 | ) 12 | 13 | func NetIpToKglbAddr(netIP net.IP) *kglb_pb.IP { 14 | res := &kglb_pb.IP{} 15 | if ip := netIP.To4(); ip != nil { 16 | res.Address = &kglb_pb.IP_Ipv4{Ipv4: netIP.String()} 17 | } else { 18 | res.Address = &kglb_pb.IP_Ipv6{Ipv6: netIP.String()} 19 | } 20 | return res 21 | } 22 | 23 | func KglbAddrToAddressFamily(kglbIP *kglb_pb.IP) string { 24 | switch kglbIP.GetAddress().(type) { 25 | case *kglb_pb.IP_Ipv4: 26 | return "v4" 27 | case *kglb_pb.IP_Ipv6: 28 | return "v6" 29 | default: 30 | return "" 31 | } 32 | } 33 | 34 | func KglbAddrToNetIp(kglbIP *kglb_pb.IP) net.IP { 35 | switch addr := kglbIP.GetAddress().(type) { 36 | case *kglb_pb.IP_Ipv4: 37 | return net.ParseIP(addr.Ipv4) 38 | case *kglb_pb.IP_Ipv6: 39 | return net.ParseIP(addr.Ipv6) 40 | default: 41 | return nil 42 | } 43 | } 44 | 45 | func KglbAddrToString(kglbIP *kglb_pb.IP) string { 46 | switch addr := kglbIP.GetAddress().(type) { 47 | case *kglb_pb.IP_Ipv4: 48 | return net.ParseIP(addr.Ipv4).String() 49 | case *kglb_pb.IP_Ipv6: 50 | return fmt.Sprintf("[%s]", net.ParseIP(addr.Ipv6).String()) 51 | default: 52 | return "" 53 | } 54 | } 55 | 56 | func KglbAddrToNetIpNet(kglbIP *kglb_pb.IP) *net.IPNet { 57 | switch addr := kglbIP.GetAddress().(type) { 58 | case *kglb_pb.IP_Ipv4: 59 | return &net.IPNet{ 60 | IP: net.ParseIP(addr.Ipv4), 61 | Mask: net.CIDRMask(32, 32)} 62 | case *kglb_pb.IP_Ipv6: 63 | return &net.IPNet{ 64 | IP: net.ParseIP(addr.Ipv6), 65 | Mask: net.CIDRMask(128, 128)} 66 | default: 67 | return nil 68 | } 69 | } 70 | 71 | func KglbAddrToFamily(kglbIP *kglb_pb.IP) kglb_pb.AddressFamily { 72 | switch kglbIP.GetAddress().(type) { 73 | case *kglb_pb.IP_Ipv6: 74 | return kglb_pb.AddressFamily_AF_INET6 75 | default: 76 | return kglb_pb.AddressFamily_AF_INET 77 | } 78 | } 79 | 80 | func UpstreamsEqual(s, o *kglb_pb.UpstreamState) bool { 81 | s = proto.Clone(s).(*kglb_pb.UpstreamState) 82 | o = proto.Clone(o).(*kglb_pb.UpstreamState) 83 | s.Weight = 0 84 | o.Weight = 0 85 | s.Hostname = "" 86 | o.Hostname = "" 87 | return proto.Equal(s, o) 88 | } 89 | 90 | func IPVSServicesEqual(s, o *kglb_pb.IpvsService) bool { 91 | return proto.Equal(s, o) 92 | } 93 | 94 | func GetIpvsServiceFromLbService( 95 | lbService *kglb_pb.LoadBalancerService) (*kglb_pb.IpvsService, error) { 96 | 97 | var ipvsService *kglb_pb.IpvsService 98 | switch lbService.Service.(type) { 99 | case *kglb_pb.LoadBalancerService_IpvsService: 100 | ipvsService = lbService.GetIpvsService() 101 | default: 102 | return nil, errors.Newf("Unknown lb service type: %+v", lbService) 103 | } 104 | 105 | if ipvsService == nil { 106 | return nil, errors.Newf("missed ipvs service state: %+v", lbService) 107 | } 108 | return ipvsService, nil 109 | } 110 | 111 | func GetIpvsServiceFromBalancer( 112 | balancer *kglb_pb.BalancerState) (*kglb_pb.IpvsService, error) { 113 | 114 | // check arg first. 115 | if balancer == nil { 116 | return nil, errors.New("balancer state is empty.") 117 | } 118 | 119 | return GetIpvsServiceFromLbService(balancer.GetLbService()) 120 | } 121 | 122 | // Returns vip:port from LoadBalancerService proto. 123 | func GetVipFromLbService(lb *kglb_pb.LoadBalancerService) (string, int, error) { 124 | // getting IpvsService first. 125 | ipvsService, err := GetIpvsServiceFromLbService(lb) 126 | if err != nil { 127 | return "", 0, err 128 | } 129 | 130 | // extract vip, port, fwmark 131 | switch attr := ipvsService.Attributes.(type) { 132 | case *kglb_pb.IpvsService_TcpAttributes: 133 | ip, port := KglbAddrToNetIp(attr.TcpAttributes.GetAddress()).String(), 134 | int(attr.TcpAttributes.Port) 135 | return ip, port, nil 136 | case *kglb_pb.IpvsService_UdpAttributes: 137 | ip, port := KglbAddrToNetIp(attr.UdpAttributes.GetAddress()).String(), 138 | int(attr.UdpAttributes.Port) 139 | return ip, port, nil 140 | case *kglb_pb.IpvsService_FwmarkAttributes: 141 | return "fwmark", 142 | int(attr.FwmarkAttributes.GetFwmark()), 143 | nil 144 | default: 145 | return "", 0, fmt.Errorf( 146 | "unknown attributes type of ipvs service: %+v: ", 147 | ipvsService) 148 | } 149 | } 150 | 151 | // Generate unique key based on balancer config. It includes name and vip:vport, 152 | // and proto (tcp/udp) today which means control plane will need to recreate 153 | // balancer in case of dynamic update of one of that attribute. It is done in 154 | // this way to simplify logic of updating balancer without updating its 155 | // BalancerStats instance since all these attributes inside tags hierarchy. 156 | func GetKeyFromBalancerConfig(balancerConfig *kglb_pb.BalancerConfig) (string, error) { 157 | // balancer name. 158 | balancerName := balancerConfig.GetName() 159 | // key based on vip:vport and proto 160 | lbKey, err := GetKeyFromLbService(balancerConfig.GetLbService()) 161 | if err != nil { 162 | return "", err 163 | } 164 | 165 | return balancerName + "-" + lbKey, nil 166 | } 167 | 168 | // construct key for LoadBalancerService in "vip:vport-proto" format. 169 | func GetKeyFromLbService(lb *kglb_pb.LoadBalancerService) (string, error) { 170 | // getting IpvsService first. 171 | ipvsService, err := GetIpvsServiceFromLbService(lb) 172 | if err != nil { 173 | return "", err 174 | } 175 | 176 | // extract vip, port, fwmark 177 | switch attr := ipvsService.Attributes.(type) { 178 | case *kglb_pb.IpvsService_TcpAttributes: 179 | ip, port := KglbAddrToNetIp(attr.TcpAttributes.GetAddress()).String(), 180 | int(attr.TcpAttributes.Port) 181 | return fmt.Sprintf("%s:%d-tcp", ip, port), nil 182 | case *kglb_pb.IpvsService_UdpAttributes: 183 | ip, port := KglbAddrToNetIp(attr.UdpAttributes.GetAddress()).String(), 184 | int(attr.UdpAttributes.Port) 185 | return fmt.Sprintf("%s:%d-udp", ip, port), nil 186 | case *kglb_pb.IpvsService_FwmarkAttributes: 187 | af := "" 188 | if attr.FwmarkAttributes.AddressFamily == kglb_pb.AddressFamily_AF_INET { 189 | af = "v4" 190 | } else if attr.FwmarkAttributes.AddressFamily == kglb_pb.AddressFamily_AF_INET6 { 191 | af = "v6" 192 | } 193 | return fmt.Sprintf( 194 | "fwmark:%d:%s", 195 | int(attr.FwmarkAttributes.GetFwmark()), 196 | af), nil 197 | default: 198 | return "", fmt.Errorf( 199 | "unknown attributes type of ipvs service: %+v: ", 200 | ipvsService) 201 | } 202 | } 203 | -------------------------------------------------------------------------------- /kglb/utils/fwmark/fwmark_conn.go: -------------------------------------------------------------------------------- 1 | package fwmark 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net" 7 | "os" 8 | "syscall" 9 | "time" 10 | 11 | "golang.org/x/sys/unix" 12 | 13 | "godropbox/errors" 14 | ) 15 | 16 | const ( 17 | basePort = 50000 18 | ) 19 | 20 | // wrapper on top of net.Conn to properly close undertheath socket/file 21 | // constructed via FileConn. 22 | type connWrap struct { 23 | net.Conn 24 | file *os.File 25 | } 26 | 27 | func (c *connWrap) Close() error { 28 | err := c.Conn.Close() 29 | err2 := c.file.Close() 30 | if err == nil { 31 | err = err2 32 | } 33 | return err 34 | } 35 | 36 | type tcpConnParams struct { 37 | timeout time.Duration 38 | fwmark uint32 39 | localIp net.IP 40 | remoteIp net.IP // required field. 41 | remotePort int // required field. 42 | } 43 | 44 | // convert net.IP into sockaddr structure. 45 | func toSockaddr(ip net.IP, port int) syscall.Sockaddr { 46 | if ip.To4() != nil { 47 | addrBytes := [4]byte{} 48 | copy(addrBytes[:], ip.To4()) 49 | return &syscall.SockaddrInet4{Port: port, Addr: addrBytes} 50 | } else { 51 | addrBytes := [16]byte{} 52 | copy(addrBytes[:], ip.To16()) 53 | return &syscall.SockaddrInet6{Port: port, Addr: addrBytes} 54 | } 55 | } 56 | 57 | // establish tcp connection via blocking connect() call with provided socket (fd). 58 | func tcpConn(fd int, params *tcpConnParams) error { 59 | if len(params.remoteIp) == 0 || params.remoteIp.IsUnspecified() { 60 | return fmt.Errorf("remoteIp is required") 61 | } 62 | 63 | if params.remotePort == 0 { 64 | return fmt.Errorf("remotePort is required") 65 | } 66 | 67 | // bind local addr if it's provided. 68 | if len(params.localIp) != 0 && !params.localIp.IsUnspecified() { 69 | port := 0 70 | if params.fwmark != 0 { 71 | port = basePort + int(params.fwmark) 72 | // enable SO_REUSEADDR so we could use same local port for TCP sockets which goes to different 73 | // remote ip/port 74 | err := syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1) 75 | if err != nil { 76 | return errors.Wrapf( 77 | err, 78 | "SetsockoptInt() so_reuseaddr fails, dst: %v:%d, local: %v ", 79 | params.remoteIp.String(), 80 | params.remotePort, 81 | params.localIp.String()) 82 | } 83 | // set maximum outgoing segment size. we would account for aditional encap 84 | err = syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, syscall.TCP_MAXSEG, 1400) 85 | if err != nil { 86 | return errors.Wrapf( 87 | err, 88 | "SetsockoptInt() tcp_maxseg fails, dst: %v:%d, local: %v ", 89 | params.remoteIp.String(), 90 | params.remotePort, 91 | params.localIp.String()) 92 | } 93 | } 94 | lsa := toSockaddr(params.localIp, port) 95 | if err := syscall.Bind(fd, lsa); err != nil { 96 | return errors.Wrapf( 97 | err, 98 | "Bind() fails, dst: %v:%d, local: %s, lsa: %+v", 99 | params.remoteIp.String(), 100 | params.remotePort, 101 | params.localIp.String(), 102 | lsa) 103 | } 104 | } 105 | 106 | if params.fwmark != 0 { 107 | err := syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_MARK, int(params.fwmark)) 108 | if err != nil { 109 | return errors.Wrapf( 110 | err, 111 | "SetsockoptInt() fails, dst: %v:%d, local: %v ", 112 | params.remoteIp.String(), 113 | params.remotePort, 114 | params.localIp.String()) 115 | } 116 | } 117 | 118 | if params.timeout > 0 { 119 | tv := syscall.NsecToTimeval(params.timeout.Nanoseconds()) 120 | for _, opt := range []int{syscall.SO_RCVTIMEO, syscall.SO_SNDTIMEO} { 121 | if err := syscall.SetsockoptTimeval(fd, syscall.SOL_SOCKET, opt, &tv); err != nil { 122 | return errors.Wrapf( 123 | err, 124 | "fails to apply %d, dst: %v:%d, local: %v ", 125 | opt, 126 | params.remoteIp.String(), 127 | params.remotePort, 128 | params.localIp.String()) 129 | } 130 | } 131 | if err := syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, unix.TCP_USER_TIMEOUT, int(params.timeout/time.Millisecond)); err != nil { 132 | return errors.Wrapf( 133 | err, 134 | "fails to apply TCP_USER_TIMEOUT, dst: %v:%d, local: %v ", 135 | params.remoteIp.String(), 136 | params.remotePort, 137 | params.localIp.String()) 138 | } 139 | } 140 | 141 | // sockaddr structure for dst ip. 142 | rsa := toSockaddr(params.remoteIp, params.remotePort) 143 | 144 | // TODO(dkopytkov): simplify whole this logic via dialer.Control after 145 | // migration to go1.12 146 | return syscall.Connect(fd, rsa) 147 | } 148 | 149 | func newTcpConnection(ctx context.Context, fd int, params tcpConnParams) (*connWrap, error) { 150 | if deadline, ok := ctx.Deadline(); ok { 151 | dur := time.Until(deadline) 152 | if dur <= 0 { 153 | return nil, fmt.Errorf("deadline has already passed: dst: %v:%d, local: %v", 154 | params.remoteIp, params.remotePort, params.localIp.String()) 155 | } 156 | params.timeout = dur 157 | } 158 | 159 | if err := tcpConn(fd, ¶ms); err != nil { 160 | return nil, err 161 | } 162 | 163 | file := os.NewFile(uintptr(fd), "") 164 | fileConn, err := net.FileConn(file) 165 | return &connWrap{ 166 | Conn: fileConn, 167 | file: file, 168 | }, err 169 | } 170 | 171 | // Establishes TCP connection to the address provided in dstAddress with a socket 172 | // marked with fwmark. When srcAddress is provided the socket will be bind on 173 | // that address, otherwise src address will be assign automatically by OS. 174 | func NewTcpConnection( 175 | ctx context.Context, 176 | network string, 177 | localIp net.IP, 178 | dstAddress string, 179 | fwmark uint32) (net.Conn, error) { 180 | 181 | switch network { 182 | case "tcp", "tcp4", "tcp6": 183 | default: 184 | return nil, errors.Newf( 185 | "unknown network %s, dst: %s, local_ip: %v", 186 | network, 187 | dstAddress, 188 | localIp.String()) 189 | } 190 | 191 | raddr, err := net.ResolveTCPAddr(network, dstAddress) 192 | if err != nil { 193 | return nil, errors.Wrapf(err, "ResolveTCPAddr() fails, dst: %s: ", dstAddress) 194 | } 195 | 196 | // golang uses network "tcp" even for IPv6 connections 197 | af := syscall.AF_INET6 198 | if raddr.IP.To4() != nil { 199 | af = syscall.AF_INET 200 | } 201 | 202 | fd, err := syscall.Socket(af, syscall.SOCK_STREAM, syscall.IPPROTO_TCP) 203 | if err != nil { 204 | return nil, errors.Wrapf(err, "Socket() fails, dst: %s: ", dstAddress) 205 | } 206 | 207 | c, err := newTcpConnection(ctx, fd, tcpConnParams{ 208 | fwmark: fwmark, 209 | localIp: localIp, 210 | remoteIp: raddr.IP, 211 | remotePort: raddr.Port, 212 | }) 213 | if err != nil { 214 | _ = syscall.Close(fd) 215 | return nil, err 216 | } 217 | return c, nil 218 | } 219 | -------------------------------------------------------------------------------- /kglb/data_plane/cache_resolver.go: -------------------------------------------------------------------------------- 1 | package data_plane 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "strings" 7 | "sync" 8 | 9 | "dropbox/dlog" 10 | "dropbox/exclog" 11 | "dropbox/kglb/common" 12 | kglb_pb "dropbox/proto/kglb" 13 | "godropbox/errors" 14 | ) 15 | 16 | var ( 17 | // value of nknown service_name tag in case of failed resolution. 18 | defaultService = "default" 19 | // value of hostname tag in case of failed resolution. 20 | defaultHostname = "default" 21 | // value of real_cluster tag in case of failed resolution. 22 | defaultRealCluster = "default" 23 | ) 24 | 25 | // Implements Resolver interface based on cache approach. 26 | type CacheResolver struct { 27 | mutex sync.RWMutex 28 | // cache map protected by mutex. 29 | // ip -> hostname map 30 | reverseCache map[string]string 31 | // hostname -> ip map 32 | hostnameCache map[string]*HostnameCacheEntry 33 | // vip:vport -> alias 34 | serviceCache map[string]string 35 | } 36 | 37 | func NewCacheResolver() (*CacheResolver, error) { 38 | return &CacheResolver{ 39 | reverseCache: make(map[string]string), 40 | hostnameCache: make(map[string]*HostnameCacheEntry), 41 | serviceCache: make(map[string]string), 42 | }, nil 43 | } 44 | 45 | // lookup (hostname resolution). 46 | func (r *CacheResolver) Lookup(name string) (*HostnameCacheEntry, error) { 47 | r.mutex.RLock() 48 | defer r.mutex.RUnlock() 49 | 50 | if val, ok := r.hostnameCache[name]; ok { 51 | return val, nil 52 | } 53 | 54 | err := errors.Newf("missed lookup cache: %v, %v", name, r.hostnameCache) 55 | exclog.Report(err, exclog.Noncritical, "") 56 | return nil, err 57 | } 58 | 59 | // cluster name for hostname. 60 | func (r *CacheResolver) Cluster(hostname string) string { 61 | if len(hostname) == 0 || !strings.Contains(hostname, "-") { 62 | return defaultRealCluster 63 | } 64 | 65 | slices := strings.Split(hostname, "-") 66 | if len(slices) > 0 { 67 | return slices[0] 68 | } 69 | return defaultRealCluster 70 | } 71 | 72 | // reverse lookup (ip -> hostname resolution). 73 | func (r *CacheResolver) ReverseLookup(ip net.IP) (string, error) { 74 | r.mutex.RLock() 75 | defer r.mutex.RUnlock() 76 | 77 | if val, ok := r.reverseCache[ip.String()]; ok { 78 | return val, nil 79 | } 80 | 81 | err := errors.Newf("missed reverse lookup cache: %v", ip.String()) 82 | exclog.Report(err, exclog.Noncritical, "") 83 | 84 | return defaultHostname, nil 85 | } 86 | 87 | // vip:vport to human readable alias conversion. 88 | func (r *CacheResolver) ServiceLookup(srv *kglb_pb.IpvsService) string { 89 | if srv == nil { 90 | return defaultService 91 | } 92 | 93 | // extract vip, port, fwmark 94 | key := r.keyByService(srv) 95 | 96 | r.mutex.RLock() 97 | defer r.mutex.RUnlock() 98 | 99 | if val, ok := r.serviceCache[key]; ok { 100 | return val 101 | } 102 | 103 | dlog.Errorf("missed service lookup cache: %v", key) 104 | 105 | return defaultService 106 | } 107 | 108 | // update map of internal naming/aliases to skip reverse-lookup since 109 | // control plane sends all required information including hostname and aliases 110 | // inside the config. 111 | func (r *CacheResolver) UpdateCache(state *kglb_pb.DataPlaneState) { 112 | if state == nil { 113 | dlog.Info( 114 | "Skipping updating CacheResolver cache since state is empty") 115 | return 116 | } 117 | r.mutex.Lock() 118 | defer r.mutex.Unlock() 119 | 120 | cacheUpdated := false 121 | 122 | for _, service := range state.Balancers { 123 | alias := service.GetName() 124 | key := r.keyByService(service.GetLbService().GetIpvsService()) 125 | if entry, ok := r.serviceCache[key]; !ok { 126 | dlog.Infof( 127 | "Adding service cache record: %s <-> %s", 128 | key, 129 | alias) 130 | } else if entry != alias { 131 | dlog.Infof( 132 | "Updating service cache record: %s: %s -> %s", 133 | key, 134 | entry, 135 | alias) 136 | } 137 | r.serviceCache[key] = alias 138 | 139 | // update real server hostname <-> ip caches. 140 | for _, upstreamState := range service.Upstreams { 141 | hostname := upstreamState.GetHostname() 142 | if len(hostname) == 0 { 143 | exclog.Report( 144 | errors.Newf("missed hostname field: %+v", upstreamState), 145 | exclog.Noncritical, "") 146 | continue 147 | } 148 | address := common.KglbAddrToNetIp(upstreamState.GetAddress()) 149 | if len(address) == 0 { 150 | exclog.Report( 151 | errors.Newf("missed address field: %+v", upstreamState), 152 | exclog.Noncritical, "") 153 | continue 154 | } 155 | 156 | if _, ok := r.hostnameCache[hostname]; !ok { 157 | r.hostnameCache[hostname] = &HostnameCacheEntry{} 158 | } 159 | 160 | if address.To4() != nil && !r.hostnameCache[hostname].IPv4.Equal(address) { 161 | dlog.Infof( 162 | "Adding hostname v4 cache record: %s <-> %s", 163 | hostname, 164 | address.String()) 165 | r.hostnameCache[hostname].IPv4 = address 166 | cacheUpdated = true 167 | } else if address.To4() == nil && !r.hostnameCache[hostname].IPv6.Equal(address) { 168 | dlog.Infof( 169 | "Adding hostname v6 cache record: %s <-> %s", 170 | hostname, 171 | address.String()) 172 | r.hostnameCache[hostname].IPv6 = address 173 | cacheUpdated = true 174 | } 175 | 176 | if cachedHostname, ok := r.reverseCache[address.String()]; !ok || cachedHostname != hostname { 177 | dlog.Infof( 178 | "Adding reverse cache record: %s <-> %s", 179 | address.String(), 180 | hostname) 181 | r.reverseCache[address.String()] = hostname 182 | cacheUpdated = true 183 | } 184 | } 185 | } 186 | 187 | if cacheUpdated { 188 | // log size of caches to control sizes in alerts since update adds values 189 | // without removing old keys. 190 | dlog.Infof( 191 | "service_cache_size: %d, hostname_cache_size: %d, reverse_cache_size: %d", 192 | len(r.serviceCache), 193 | len(r.hostnameCache), 194 | len(r.reverseCache)) 195 | } 196 | } 197 | 198 | // generate key based on IpvsService settings to use it as a key for serviceCache map. 199 | func (r *CacheResolver) keyByService(srv *kglb_pb.IpvsService) string { 200 | // extract vip, port, fwmark 201 | switch attr := srv.Attributes.(type) { 202 | case *kglb_pb.IpvsService_TcpAttributes: 203 | return fmt.Sprintf( 204 | "tcp-%s:%d", 205 | common.KglbAddrToNetIp(attr.TcpAttributes.GetAddress()), 206 | attr.TcpAttributes.Port) 207 | case *kglb_pb.IpvsService_UdpAttributes: 208 | return fmt.Sprintf( 209 | "udp-%s:%d", 210 | common.KglbAddrToNetIp(attr.UdpAttributes.GetAddress()), 211 | attr.UdpAttributes.Port) 212 | case *kglb_pb.IpvsService_FwmarkAttributes: 213 | return fmt.Sprintf( 214 | "fwmark-%d", 215 | attr.FwmarkAttributes.GetFwmark()) 216 | default: 217 | err := errors.Newf( 218 | "service lookup fails because of unknown attributes type: %s, service: %+v", 219 | attr, 220 | srv) 221 | exclog.Report(err, exclog.Noncritical, "") 222 | } 223 | return defaultService 224 | } 225 | --------------------------------------------------------------------------------