├── .gitignore ├── code-of-conduct.md ├── OWNERS ├── examples └── kubernetes │ ├── kubeconfig │ ├── kubia.yaml │ ├── konnectivity-test-client.yaml │ ├── token_generation.sh │ ├── konnectivity-agent.yaml │ ├── konnectivity-server.yaml │ └── README.md ├── SECURITY_CONTACTS ├── hack └── go-license-header.txt ├── tests ├── readiness_test.go ├── reconnect_test.go ├── custom_alpn_test.go ├── tcp_server_test.go ├── concurrent_test.go ├── concurrent_client_request_test.go ├── ha_proxy_server_test.go └── agent_disconnect_test.go ├── konnectivity-client ├── go.mod ├── proto │ └── client │ │ └── client.proto ├── pkg │ └── client │ │ ├── conn.go │ │ ├── client.go │ │ └── client_test.go └── go.sum ├── cloudbuild.yaml ├── pkg ├── util │ ├── flags.go │ ├── handlers.go │ ├── url.go │ ├── net_test.go │ ├── net.go │ └── certificates.go ├── server │ ├── readiness_manager.go │ ├── default_route_backend_manager.go │ ├── desthost_backend_manager.go │ ├── backend_manager_test.go │ ├── tunnel.go │ ├── metrics │ │ └── metrics.go │ ├── server_test.go │ └── backend_manager.go ├── features │ └── features.go └── agent │ ├── metrics │ └── metrics.go │ ├── clientset.go │ └── client_test.go ├── proto ├── agent │ ├── agent.proto │ ├── agent.pb.go │ └── mocks │ │ └── agent_mock.go └── header │ └── header.go ├── artifacts └── images │ ├── agent-build.Dockerfile │ ├── server-build.Dockerfile │ ├── test-server-build.Dockerfile │ └── client-build.Dockerfile ├── cmd ├── agent │ ├── main.go │ └── app │ │ ├── server.go │ │ └── options │ │ └── options.go ├── server │ └── main.go └── test-server │ └── main.go ├── CONTRIBUTING.md ├── go.mod ├── RELEASE.md ├── README.md └── LICENSE /.gitignore: -------------------------------------------------------------------------------- 1 | /bin 2 | /certs/ 3 | /cfssl 4 | /cfssljson 5 | /easy-rsa-master/ 6 | /easy-rsa.tar.gz 7 | /easy-rsa 8 | -------------------------------------------------------------------------------- /code-of-conduct.md: -------------------------------------------------------------------------------- 1 | # Kubernetes Community Code of Conduct 2 | 3 | Please refer to our [Kubernetes Community Code of Conduct](https://git.k8s.io/community/code-of-conduct.md) 4 | -------------------------------------------------------------------------------- /OWNERS: -------------------------------------------------------------------------------- 1 | # See the OWNERS docs at https://go.k8s.io/owners 2 | 3 | reviewers: 4 | - caesarxuchao 5 | - dberkov 6 | - jefftree 7 | - jkh52 8 | approvers: 9 | - cheftako 10 | - mcrute 11 | - anfernee 12 | - andrewsykim 13 | - caesarxuchao 14 | emeritus_approvers: 15 | - Sh4d1 16 | -------------------------------------------------------------------------------- /examples/kubernetes/kubeconfig: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Config 3 | users: 4 | - name: konnectivity-server 5 | user: 6 | token: ${SERVER_TOKEN} 7 | clusters: 8 | - name: local 9 | cluster: 10 | insecure-skip-tls-verify: true 11 | server: https://localhost:443 12 | contexts: 13 | - context: 14 | cluster: local 15 | user: konnectivity-server 16 | name: konnectivity-server 17 | current-context: konnectivity-server 18 | -------------------------------------------------------------------------------- /examples/kubernetes/kubia.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: kubia 5 | labels: 6 | app: kubia 7 | spec: 8 | containers: 9 | - image: luksa/kubia 10 | name: kubia 11 | ports: 12 | - containerPort: 8080 13 | protocol: TCP 14 | --- 15 | apiVersion: v1 16 | kind: Service 17 | metadata: 18 | name: kubia 19 | spec: 20 | ports: 21 | - port: 80 22 | targetPort: 8080 23 | selector: 24 | app: kubia 25 | 26 | -------------------------------------------------------------------------------- /SECURITY_CONTACTS: -------------------------------------------------------------------------------- 1 | # Defined below are the security contacts for this repo. 2 | # 3 | # They are the contact point for the Product Security Committee to reach out 4 | # to for triaging and handling of incoming issues. 5 | # 6 | # The below names agree to abide by the 7 | # [Embargo Policy](https://git.k8s.io/security/private-distributors-list.md#embargo-policy) 8 | # and will be removed and replaced if they violate that agreement. 9 | # 10 | # DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, FOLLOW THE 11 | # INSTRUCTIONS AT https://kubernetes.io/security/ 12 | 13 | cheftako 14 | mcrute 15 | anfernee 16 | andrewsykim 17 | -------------------------------------------------------------------------------- /hack/go-license-header.txt: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | -------------------------------------------------------------------------------- /tests/readiness_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | ) 7 | 8 | func TestReadiness(t *testing.T) { 9 | stopCh := make(chan struct{}) 10 | defer close(stopCh) 11 | 12 | proxy, server, cleanup, err := runGRPCProxyServerWithServerCount(1) 13 | if err != nil { 14 | t.Fatal(err) 15 | } 16 | defer cleanup() 17 | 18 | ready, _ := server.Readiness.Ready() 19 | if ready { 20 | t.Fatalf("expected not ready") 21 | } 22 | 23 | runAgent(proxy.agent, stopCh) 24 | 25 | // Wait for agent to register on proxy server 26 | time.Sleep(time.Second) 27 | 28 | ready, _ = server.Readiness.Ready() 29 | if !ready { 30 | t.Fatalf("expected ready") 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /konnectivity-client/go.mod: -------------------------------------------------------------------------------- 1 | module sigs.k8s.io/apiserver-network-proxy/konnectivity-client 2 | 3 | go 1.17 4 | 5 | // Prefer to keep requirements compatible with the oldest supported 6 | // k/k minor version, to prevent client backport issues. 7 | require ( 8 | github.com/golang/protobuf v1.4.3 9 | google.golang.org/grpc v1.27.1 10 | k8s.io/klog/v2 v2.0.0 11 | ) 12 | 13 | require ( 14 | github.com/go-logr/logr v0.1.0 // indirect 15 | golang.org/x/net v0.0.0-20200822124328-c89045814202 // indirect 16 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd // indirect 17 | golang.org/x/text v0.3.0 // indirect 18 | google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 // indirect 19 | google.golang.org/protobuf v1.26.0-rc.1 // indirect 20 | ) 21 | -------------------------------------------------------------------------------- /cloudbuild.yaml: -------------------------------------------------------------------------------- 1 | timeout: 9000s 2 | options: 3 | substitution_option: ALLOW_LOOSE 4 | steps: 5 | - name: 'gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20211118-2f2d816b90' 6 | entrypoint: bash 7 | env: 8 | - TAG=$_GIT_TAG 9 | - BASE_REF=$_PULL_BASE_REF 10 | - DOCKER_CLI_EXPERIMENTAL=enabled 11 | # default cloudbuild has HOME=/builder/home and docker buildx is in /root/.docker/cli-plugins/docker-buildx 12 | # set the home to /root explicitly to if using docker buildx 13 | - HOME=/root 14 | args: 15 | - '-c' 16 | - | 17 | gcloud auth configure-docker \ 18 | && make release-staging 19 | substitutions: 20 | # _GIT_TAG will be filled with a git-based tag for the image, of the form vYYYYMMDD-hash, and 21 | # can be used as a substitution 22 | _GIT_TAG: '12345' 23 | _PULL_BASE_REF: 'dev' 24 | -------------------------------------------------------------------------------- /pkg/util/flags.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package util 18 | 19 | import "strings" 20 | 21 | // Normalize replaces underscores with hyphens 22 | // we should always use hyphens instead of underscores when registering component flags 23 | func Normalize(s string) string { 24 | return strings.Replace(s, "_", "-", -1) 25 | } 26 | -------------------------------------------------------------------------------- /proto/agent/agent.proto: -------------------------------------------------------------------------------- 1 | // Copyright The Kubernetes Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | syntax = "proto3"; 16 | 17 | option go_package = "sigs.k8s.io/apiserver-network-proxy/proto/agent"; 18 | 19 | import "konnectivity-client/proto/client/client.proto"; 20 | 21 | service AgentService { 22 | // Agent Identifier? 23 | rpc Connect(stream Packet) returns (stream Packet) {} 24 | } 25 | -------------------------------------------------------------------------------- /pkg/util/handlers.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package util 18 | 19 | import "net/http" 20 | 21 | // RedirectTo redirects request to a certain destination. 22 | func RedirectTo(to string) func(http.ResponseWriter, *http.Request) { 23 | return func(rw http.ResponseWriter, req *http.Request) { 24 | http.Redirect(rw, req, to, http.StatusMovedPermanently) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /artifacts/images/agent-build.Dockerfile: -------------------------------------------------------------------------------- 1 | # Build the proxy-agent binary 2 | FROM golang:1.17.5 as builder 3 | 4 | # Copy in the go src 5 | WORKDIR /go/src/sigs.k8s.io/apiserver-network-proxy 6 | 7 | # Copy the Go Modules manifests 8 | COPY go.mod go.mod 9 | COPY go.sum go.sum 10 | 11 | # This is required before go mod download because we have a 12 | # replace directive for konnectivity-client in go.mod 13 | # The download will fail without the directory present 14 | COPY konnectivity-client/ konnectivity-client/ 15 | 16 | # Cache dependencies 17 | RUN go mod download 18 | 19 | # Copy the sources 20 | COPY pkg/ pkg/ 21 | COPY cmd/ cmd/ 22 | COPY proto/ proto/ 23 | 24 | # Build 25 | ARG ARCH 26 | RUN CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} go build -a -ldflags '-extldflags "-static"' -o proxy-agent sigs.k8s.io/apiserver-network-proxy/cmd/agent 27 | 28 | # Copy the loader into a thin image 29 | FROM scratch 30 | WORKDIR / 31 | COPY --from=builder /go/src/sigs.k8s.io/apiserver-network-proxy/proxy-agent . 32 | ENTRYPOINT ["/proxy-agent"] 33 | -------------------------------------------------------------------------------- /artifacts/images/server-build.Dockerfile: -------------------------------------------------------------------------------- 1 | # Build the proxy-server binary 2 | FROM golang:1.17.5 as builder 3 | 4 | # Copy in the go src 5 | WORKDIR /go/src/sigs.k8s.io/apiserver-network-proxy 6 | 7 | # Copy the Go Modules manifests 8 | COPY go.mod go.mod 9 | COPY go.sum go.sum 10 | 11 | # This is required before go mod download because we have a 12 | # replace directive for konnectivity-client in go.mod 13 | # The download will fail without the directory present 14 | COPY konnectivity-client/ konnectivity-client/ 15 | 16 | # Cache dependencies 17 | RUN go mod download 18 | 19 | # Copy the sources 20 | COPY pkg/ pkg/ 21 | COPY cmd/ cmd/ 22 | COPY proto/ proto/ 23 | 24 | # Build 25 | ARG ARCH 26 | RUN CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} go build -a -ldflags '-extldflags "-static"' -o proxy-server sigs.k8s.io/apiserver-network-proxy/cmd/server 27 | 28 | # Copy the loader into a thin image 29 | FROM scratch 30 | WORKDIR / 31 | COPY --from=builder /go/src/sigs.k8s.io/apiserver-network-proxy/proxy-server . 32 | ENTRYPOINT ["/proxy-server"] 33 | -------------------------------------------------------------------------------- /artifacts/images/test-server-build.Dockerfile: -------------------------------------------------------------------------------- 1 | # Build the http test server binary 2 | FROM golang:1.17.5 as builder 3 | 4 | # Copy in the go src 5 | WORKDIR /go/src/sigs.k8s.io/apiserver-network-proxy 6 | 7 | # Copy the Go Modules manifests 8 | COPY go.mod go.mod 9 | COPY go.sum go.sum 10 | 11 | 12 | # This is required before go mod download because we have a 13 | # replace directive for konnectivity-client in go.mod 14 | # The download will fail without the directory present 15 | COPY konnectivity-client/ konnectivity-client/ 16 | 17 | # Cache dependencies 18 | RUN go mod download 19 | 20 | # Copy the sources 21 | COPY pkg/ pkg/ 22 | COPY cmd/ cmd/ 23 | 24 | # Build 25 | ARG ARCH 26 | RUN CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} go build -a -ldflags '-extldflags "-static"' -o http-test-server sigs.k8s.io/apiserver-network-proxy/cmd/test-server 27 | 28 | # Copy the loader into a thin image 29 | FROM scratch 30 | WORKDIR / 31 | COPY --from=builder /go/src/sigs.k8s.io/apiserver-network-proxy/http-test-server . 32 | ENTRYPOINT ["/http-test-server"] 33 | -------------------------------------------------------------------------------- /artifacts/images/client-build.Dockerfile: -------------------------------------------------------------------------------- 1 | # Build the client binary 2 | FROM golang:1.17.5 as builder 3 | 4 | # Copy in the go src 5 | WORKDIR /go/src/sigs.k8s.io/apiserver-network-proxy 6 | 7 | # Copy the Go Modules manifests 8 | COPY go.mod go.mod 9 | COPY go.sum go.sum 10 | 11 | # This is required before go mod download because we have a 12 | # replace directive for konnectivity-client in go.mod 13 | # The download will fail without the directory present 14 | COPY konnectivity-client/ konnectivity-client/ 15 | 16 | # Cache dependencies 17 | RUN go mod download 18 | 19 | # Copy the sources 20 | COPY pkg/ pkg/ 21 | COPY cmd/ cmd/ 22 | COPY proto/ proto/ 23 | 24 | # Build 25 | ARG ARCH 26 | RUN CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} go build -a -ldflags '-extldflags "-static"' -o proxy-test-client sigs.k8s.io/apiserver-network-proxy/cmd/client 27 | 28 | # Copy the loader into a thin image 29 | FROM scratch 30 | WORKDIR / 31 | COPY --from=builder /go/src/sigs.k8s.io/apiserver-network-proxy/proxy-test-client . 32 | ENTRYPOINT ["/proxy-test-client"] 33 | -------------------------------------------------------------------------------- /pkg/server/readiness_manager.go: -------------------------------------------------------------------------------- 1 | /* 2 | copyright 2020 the kubernetes authors. 3 | 4 | licensed under the apache license, version 2.0 (the "license"); 5 | you may not use this file except in compliance with the license. 6 | you may obtain a copy of the license at 7 | 8 | http://www.apache.org/licenses/license-2.0 9 | 10 | unless required by applicable law or agreed to in writing, software 11 | distributed under the license is distributed on an "as is" basis, 12 | without warranties or conditions of any kind, either express or implied. 13 | see the license for the specific language governing permissions and 14 | limitations under the license. 15 | */ 16 | 17 | package server 18 | 19 | // ReadinessManager supports checking if the proxy server is ready. 20 | type ReadinessManager interface { 21 | // Ready returns if the proxy server is ready. If not, also return an 22 | // error message. 23 | Ready() (bool, string) 24 | } 25 | 26 | var _ ReadinessManager = &DefaultBackendStorage{} 27 | 28 | func (s *DefaultBackendStorage) Ready() (bool, string) { 29 | if s.NumBackends() == 0 { 30 | return false, "no connection to any proxy agent" 31 | } 32 | return true, "" 33 | } 34 | -------------------------------------------------------------------------------- /pkg/util/url.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package util 18 | 19 | import ( 20 | "bytes" 21 | "fmt" 22 | "net/url" 23 | "strings" 24 | ) 25 | 26 | // PrettyPrintURL decodes the URL encoded input string and print each key=val 27 | // pair separated by comma 28 | func PrettyPrintURL(urlEncode string) string { 29 | decoded, _ := url.ParseQuery(urlEncode) 30 | var buf bytes.Buffer 31 | for key, val := range decoded { 32 | for _, subVal := range val { 33 | buf.WriteString(fmt.Sprintf("%s=%s,", key, subVal)) 34 | } 35 | } 36 | return strings.TrimSuffix(buf.String(), ",") 37 | } 38 | -------------------------------------------------------------------------------- /proto/header/header.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package header 18 | 19 | const ( 20 | ServerCount = "serverCount" 21 | ServerID = "serverID" 22 | AgentID = "agentID" 23 | AgentIdentifiers = "agentIdentifiers" 24 | // AuthenticationTokenContextKey will be used as a key to store authentication tokens in grpc call 25 | // (https://tools.ietf.org/html/rfc6750#section-2.1) 26 | AuthenticationTokenContextKey = "Authorization" 27 | 28 | // AuthenticationTokenContextSchemePrefix has a prefix for auth token's content. 29 | // (https://tools.ietf.org/html/rfc6750#section-2.1) 30 | AuthenticationTokenContextSchemePrefix = "Bearer " 31 | 32 | // UserAgent is used to provide the client information in a proxy request 33 | UserAgent = "user-agent" 34 | ) 35 | -------------------------------------------------------------------------------- /examples/kubernetes/konnectivity-test-client.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: konnectivity-test-client 5 | namespace: kube-system 6 | annotations: 7 | scheduler.alpha.kubernetes.io/critical-pod: '' 8 | seccomp.security.alpha.kubernetes.io/pod: 'docker/default' 9 | spec: 10 | hostNetwork: true 11 | restartPolicy: Never 12 | containers: 13 | - name: konnectivity-test-client 14 | image: ${TEST_CLIENT_IMAGE}:${TAG} 15 | resources: 16 | requests: 17 | cpu: 1m 18 | command: [ "/proxy-test-client"] 19 | args: [ 20 | "--log-file=/var/log/konnectivity-test-client.log", 21 | "--logtostderr=false", 22 | "--proxy-uds=/etc/srv/kubernetes/konnectivity-server/konnectivity-server.socket", 23 | "--proxy-host=", 24 | "--proxy-port=0", 25 | "--mode=http-connect", 26 | "--request-port=80", 27 | "--request-host=${KUBIA_IP}", 28 | ] 29 | volumeMounts: 30 | - name: konnectivity-test-log 31 | mountPath: /var/log/konnectivity-test-client.log 32 | readOnly: false 33 | - name: konnectivity-home 34 | mountPath: /etc/srv/kubernetes/konnectivity-server 35 | volumes: 36 | - name: konnectivity-test-log 37 | hostPath: 38 | path: /var/log/konnectivity-test-client.log 39 | type: FileOrCreate 40 | - name: konnectivity-home 41 | hostPath: 42 | path: /etc/srv/kubernetes/konnectivity-server 43 | type: DirectoryOrCreate 44 | -------------------------------------------------------------------------------- /examples/kubernetes/token_generation.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # https://github.com/kubernetes/kubernetes/blob/84beab6f26609527752ff441c155813b783fe145/cluster/gce/gci/configure-helper.sh#L44 4 | # secure_random generates a secure random string of bytes. This function accepts 5 | # a number of secure bytes desired and returns a base64 encoded string with at 6 | # least the requested entropy. Rather than directly reading from /dev/urandom, 7 | # we use uuidgen which calls getrandom(2). getrandom(2) verifies that the 8 | # entropy pool has been initialized sufficiently for the desired operation 9 | # before reading from /dev/urandom. 10 | # 11 | # ARGS: 12 | # #1: number of secure bytes to generate. We round up to the nearest factor of 32. 13 | function secure_random { 14 | local infobytes="${1}" 15 | if ((infobytes <= 0)); then 16 | echo "Invalid argument to secure_random: infobytes='${infobytes}'" 1>&2 17 | return 1 18 | fi 19 | 20 | local out="" 21 | for (( i = 0; i < "${infobytes}"; i += 32 )); do 22 | # uuids have 122 random bits, sha256 sums have 256 bits, so concatenate 23 | # three uuids and take their sum. The sum is encoded in ASCII hex, hence the 24 | # 64 character cut. 25 | out+="$( 26 | ( 27 | uuidgen --random; 28 | uuidgen --random; 29 | uuidgen --random; 30 | ) | sha256sum \ 31 | | head -c 64 32 | )"; 33 | done 34 | # Finally, convert the ASCII hex to base64 to increase the density. 35 | echo -n "${out}" | xxd -r -p | base64 -w 0 36 | } 37 | 38 | secure_random $1 -------------------------------------------------------------------------------- /examples/kubernetes/konnectivity-agent.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: konnectivity-agent 5 | namespace: kube-system 6 | --- 7 | apiVersion: v1 8 | kind: Pod 9 | metadata: 10 | name: konnectivity-agent 11 | namespace: kube-system 12 | annotations: 13 | scheduler.alpha.kubernetes.io/critical-pod: '' 14 | seccomp.security.alpha.kubernetes.io/pod: 'docker/default' 15 | spec: 16 | hostNetwork: true 17 | containers: 18 | - name: konnectivity-agent-container 19 | image: ${AGENT_IMAGE}:${TAG} 20 | resources: 21 | requests: 22 | cpu: 50m 23 | limits: 24 | memory: 30Mi 25 | command: [ "/proxy-agent"] 26 | args: [ 27 | "--logtostderr=true", 28 | "--ca-cert=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", 29 | "--proxy-server-host=${CLUSTER_IP}", 30 | "--proxy-server-port=8091", 31 | "--service-account-token-path=/var/run/secrets/tokens/konnectivity-agent-token", 32 | ] 33 | livenessProbe: 34 | httpGet: 35 | scheme: HTTP 36 | port: 8093 37 | path: /healthz 38 | initialDelaySeconds: 15 39 | timeoutSeconds: 15 40 | volumeMounts: 41 | - mountPath: /var/run/secrets/tokens 42 | name: konnectivity-agent-token 43 | serviceAccountName: konnectivity-agent 44 | volumes: 45 | - name: konnectivity-agent-token 46 | projected: 47 | sources: 48 | - serviceAccountToken: 49 | path: konnectivity-agent-token 50 | audience: system:konnectivity-server 51 | -------------------------------------------------------------------------------- /pkg/features/features.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package features 18 | 19 | import ( 20 | "k8s.io/apimachinery/pkg/util/runtime" 21 | "k8s.io/component-base/featuregate" 22 | ) 23 | 24 | const ( 25 | // NodeToMasterTraffic enables the traffic initiated in the agents side 26 | // to flow to the server side e.g. Kubelet to KAS and pods to KAS traffic 27 | // (KEP-2025). 28 | // TODO (#issues/232) Determine how to safely fix the feature gate name. 29 | NodeToMasterTraffic featuregate.Feature = "NodeToMasterTraffic" 30 | ) 31 | 32 | var ( 33 | // DefaultMutableFeatureGate is a mutable version of DefaultFeatureGate. 34 | DefaultMutableFeatureGate featuregate.MutableFeatureGate = featuregate.NewFeatureGate() 35 | ) 36 | 37 | func init() { 38 | runtime.Must(DefaultMutableFeatureGate.Add(defaultFeatureGates)) 39 | } 40 | 41 | var defaultFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ 42 | NodeToMasterTraffic: {Default: false, PreRelease: featuregate.Alpha}, 43 | } 44 | -------------------------------------------------------------------------------- /cmd/agent/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "flag" 21 | "fmt" 22 | "os" 23 | 24 | "k8s.io/klog/v2" 25 | 26 | "sigs.k8s.io/apiserver-network-proxy/cmd/agent/app" 27 | "sigs.k8s.io/apiserver-network-proxy/cmd/agent/app/options" 28 | "sigs.k8s.io/apiserver-network-proxy/pkg/util" 29 | ) 30 | 31 | func main() { 32 | agent := &app.Agent{} 33 | o := options.NewGrpcProxyAgentOptions() 34 | command := app.NewAgentCommand(agent, o) 35 | flags := command.Flags() 36 | flags.AddFlagSet(o.Flags()) 37 | local := flag.NewFlagSet(os.Args[0], flag.ExitOnError) 38 | klog.InitFlags(local) 39 | err := local.Set("v", "4") 40 | if err != nil { 41 | fmt.Fprintf(os.Stderr, "error setting klog flags: %v", err) 42 | } 43 | local.VisitAll(func(fl *flag.Flag) { 44 | fl.Name = util.Normalize(fl.Name) 45 | flags.AddGoFlag(fl) 46 | }) 47 | if err := command.Execute(); err != nil { 48 | klog.Errorf("error: %v\n", err) 49 | klog.Flush() 50 | os.Exit(1) 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /cmd/server/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "flag" 21 | "fmt" 22 | "os" 23 | 24 | "k8s.io/klog/v2" 25 | 26 | "sigs.k8s.io/apiserver-network-proxy/cmd/server/app" 27 | "sigs.k8s.io/apiserver-network-proxy/cmd/server/app/options" 28 | "sigs.k8s.io/apiserver-network-proxy/pkg/util" 29 | ) 30 | 31 | func main() { 32 | // flag.CommandLine.Parse(os.Args[1:]) 33 | proxy := &app.Proxy{} 34 | o := options.NewProxyRunOptions() 35 | command := app.NewProxyCommand(proxy, o) 36 | flags := command.Flags() 37 | flags.AddFlagSet(o.Flags()) 38 | local := flag.NewFlagSet(os.Args[0], flag.ExitOnError) 39 | klog.InitFlags(local) 40 | err := local.Set("v", "4") 41 | if err != nil { 42 | fmt.Fprintf(os.Stderr, "error setting klog flags: %v", err) 43 | } 44 | local.VisitAll(func(fl *flag.Flag) { 45 | fl.Name = util.Normalize(fl.Name) 46 | flags.AddGoFlag(fl) 47 | }) 48 | if err := command.Execute(); err != nil { 49 | klog.Errorf("error: %v\n", err) 50 | klog.Flush() 51 | os.Exit(1) 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /tests/reconnect_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "net" 5 | "testing" 6 | 7 | "github.com/google/uuid" 8 | "google.golang.org/grpc" 9 | "google.golang.org/grpc/metadata" 10 | "sigs.k8s.io/apiserver-network-proxy/proto/agent" 11 | agentproto "sigs.k8s.io/apiserver-network-proxy/proto/agent" 12 | "sigs.k8s.io/apiserver-network-proxy/proto/header" 13 | ) 14 | 15 | func TestClientReconnects(t *testing.T) { 16 | connections := make(chan struct{}) 17 | s := &testAgentServerImpl{ 18 | onConnect: func(stream agent.AgentService_ConnectServer) error { 19 | stream.SetHeader(metadata.New(map[string]string{ 20 | header.ServerID: uuid.Must(uuid.NewRandom()).String(), 21 | header.ServerCount: "1", 22 | })) 23 | connections <- struct{}{} 24 | return nil 25 | }, 26 | } 27 | 28 | svr := grpc.NewServer() 29 | agentproto.RegisterAgentServiceServer(svr, s) 30 | lis, err := net.Listen("tcp", "127.0.0.1:0") 31 | if err != nil { 32 | t.Fatal(err) 33 | } 34 | go func() { 35 | svr.Serve(lis) 36 | }() 37 | 38 | stopCh := make(chan struct{}) 39 | defer close(stopCh) 40 | runAgentWithID("test-id", lis.Addr().String(), stopCh) 41 | 42 | <-connections 43 | svr.Stop() 44 | 45 | lis2, err := net.Listen("tcp", lis.Addr().String()) 46 | if err != nil { 47 | t.Fatal(err) 48 | } 49 | svr2 := grpc.NewServer() 50 | agentproto.RegisterAgentServiceServer(svr2, s) 51 | go func() { 52 | if err := svr2.Serve(lis2); err != nil { 53 | panic(err) 54 | } 55 | }() 56 | 57 | <-connections 58 | } 59 | 60 | type testAgentServerImpl struct { 61 | onConnect func(agent.AgentService_ConnectServer) error 62 | } 63 | 64 | func (t *testAgentServerImpl) Connect(svr agent.AgentService_ConnectServer) error { 65 | return t.onConnect(svr) 66 | } 67 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Welcome to Kubernetes. We are excited about the prospect of you joining our [community](https://git.k8s.io/community)! The Kubernetes community abides by the CNCF [code of conduct](code-of-conduct.md). Here is an excerpt: 4 | 5 | _As contributors and maintainers of this project, and in the interest of fostering an open and welcoming community, we pledge to respect all people who contribute through reporting issues, posting feature requests, updating documentation, submitting pull requests or patches, and other activities._ 6 | 7 | ## Getting Started 8 | 9 | We have full documentation on how to get started contributing here: 10 | 11 | 14 | 15 | - [Contributor License Agreement](https://git.k8s.io/community/CLA.md) Kubernetes projects require that you sign a Contributor License Agreement (CLA) before we can accept your pull requests 16 | - [Kubernetes Contributor Guide](https://git.k8s.io/community/contributors/guide) - Main contributor documentation, or you can just jump directly to the [contributing section](https://git.k8s.io/community/contributors/guide#contributing) 17 | - [Contributor Cheat Sheet](https://git.k8s.io/community/contributors/guide/contributor-cheatsheet.md) - Common resources for existing developers 18 | 19 | ## Mentorship 20 | 21 | - [Mentoring Initiatives](https://git.k8s.io/community/mentoring) - We have a diverse set of mentorship programs available that are always looking for volunteers! 22 | 23 | ## Contact Information 24 | 25 | - [Slack channel](https://kubernetes.slack.com/messages/sig-cloud-provider) 26 | - [Mailing list](https://groups.google.com/forum/#!forum/kubernetes-sig-cloud-provider) 27 | -------------------------------------------------------------------------------- /pkg/server/default_route_backend_manager.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package server 18 | 19 | import ( 20 | "context" 21 | 22 | "k8s.io/klog/v2" 23 | "sigs.k8s.io/apiserver-network-proxy/pkg/agent" 24 | ) 25 | 26 | type DefaultRouteBackendManager struct { 27 | *DefaultBackendStorage 28 | } 29 | 30 | var _ BackendManager = &DefaultRouteBackendManager{} 31 | 32 | func NewDefaultRouteBackendManager() *DefaultRouteBackendManager { 33 | return &DefaultRouteBackendManager{ 34 | DefaultBackendStorage: NewDefaultBackendStorage( 35 | []agent.IdentifierType{agent.DefaultRoute})} 36 | } 37 | 38 | // Backend tries to get a backend associating to the request destination host. 39 | func (dibm *DefaultRouteBackendManager) Backend(ctx context.Context) (Backend, error) { 40 | dibm.mu.RLock() 41 | defer dibm.mu.RUnlock() 42 | if len(dibm.backends) == 0 { 43 | return nil, &ErrNotFound{} 44 | } 45 | if len(dibm.defaultRouteAgentIDs) == 0 { 46 | return nil, &ErrNotFound{} 47 | } 48 | agentID := dibm.defaultRouteAgentIDs[dibm.random.Intn(len(dibm.defaultRouteAgentIDs))] 49 | klog.V(4).InfoS("Picked agent as backend", "agentID", agentID) 50 | return dibm.backends[agentID][0], nil 51 | } 52 | -------------------------------------------------------------------------------- /pkg/util/net_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package util 18 | 19 | import ( 20 | "testing" 21 | ) 22 | 23 | const ( 24 | failed = "\u2717" 25 | succeed = "\u2713" 26 | ) 27 | 28 | func TestRemovePortFromHost(t *testing.T) { 29 | tests := []struct { 30 | name string 31 | origHost string 32 | expect string 33 | }{ 34 | {"Domain&Port", "localhost:8080", "localhost"}, 35 | {"Domain", "localhost", "localhost"}, 36 | {"IPv4&Port", "192.168.0.1:8080", "192.168.0.1"}, 37 | {"ShortestIPv6", "::", "::"}, 38 | {"IPv6", "9878::7675:1292:9183:7562", "9878::7675:1292:9183:7562"}, 39 | {"IPv6&Port", "[alsk:1204:1020::1292]:8080", "alsk:1204:1020::1292"}, 40 | {"FQDN", " www.kubernetes.test:8080", " www.kubernetes.test"}, 41 | } 42 | 43 | for _, tt := range tests { 44 | st := tt 45 | tf := func(t *testing.T) { 46 | t.Parallel() 47 | t.Logf("\tTestCase: %s", st.name) 48 | { 49 | get := RemovePortFromHost(st.origHost) 50 | if get != st.expect { 51 | t.Fatalf("\t%s\texpect %v, but get %v", failed, st.expect, get) 52 | } 53 | t.Logf("\t%s\texpect %v, get %v", succeed, st.expect, get) 54 | 55 | } 56 | } 57 | t.Run(st.name, tf) 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /pkg/server/desthost_backend_manager.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package server 18 | 19 | import ( 20 | "context" 21 | 22 | "k8s.io/klog/v2" 23 | "sigs.k8s.io/apiserver-network-proxy/pkg/agent" 24 | ) 25 | 26 | type DestHostBackendManager struct { 27 | *DefaultBackendStorage 28 | } 29 | 30 | var _ BackendManager = &DestHostBackendManager{} 31 | 32 | func NewDestHostBackendManager() *DestHostBackendManager { 33 | return &DestHostBackendManager{ 34 | DefaultBackendStorage: NewDefaultBackendStorage( 35 | []agent.IdentifierType{agent.IPv4, agent.IPv6, agent.Host})} 36 | } 37 | 38 | // Backend tries to get a backend associating to the request destination host. 39 | func (dibm *DestHostBackendManager) Backend(ctx context.Context) (Backend, error) { 40 | dibm.mu.RLock() 41 | defer dibm.mu.RUnlock() 42 | if len(dibm.backends) == 0 { 43 | return nil, &ErrNotFound{} 44 | } 45 | destHost := ctx.Value(destHost).(string) 46 | if destHost != "" { 47 | bes, exist := dibm.backends[destHost] 48 | if exist && len(bes) > 0 { 49 | klog.V(5).InfoS("Get the backend through the DestHostBackendManager", "destHost", destHost) 50 | return dibm.backends[destHost][0], nil 51 | } 52 | } 53 | return nil, &ErrNotFound{} 54 | } 55 | -------------------------------------------------------------------------------- /tests/custom_alpn_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "context" 5 | "crypto/tls" 6 | "encoding/pem" 7 | "io/ioutil" 8 | "net/http" 9 | "net/http/httptest" 10 | "os" 11 | "strings" 12 | "sync/atomic" 13 | "testing" 14 | 15 | "google.golang.org/grpc" 16 | "google.golang.org/grpc/credentials" 17 | "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client" 18 | "sigs.k8s.io/apiserver-network-proxy/pkg/util" 19 | ) 20 | 21 | func TestCustomALPN(t *testing.T) { 22 | const proto = "test-proto" 23 | protoUsed := int32(0) 24 | 25 | svr := httptest.NewUnstartedServer(http.DefaultServeMux) 26 | svr.TLS = &tls.Config{NextProtos: []string{proto}, MinVersion: tls.VersionTLS13} 27 | svr.Config.TLSNextProto = map[string]func(*http.Server, *tls.Conn, http.Handler){ 28 | proto: func(svr *http.Server, conn *tls.Conn, handle http.Handler) { 29 | atomic.AddInt32(&protoUsed, 1) 30 | }, 31 | } 32 | svr.StartTLS() 33 | 34 | ca, err := ioutil.TempFile("", "") 35 | if err != nil { 36 | t.Fatal(err) 37 | } 38 | defer ca.Close() 39 | defer os.Remove(ca.Name()) 40 | 41 | err = pem.Encode(ca, &pem.Block{ 42 | Type: "CERTIFICATE", 43 | Bytes: svr.TLS.Certificates[0].Certificate[0], 44 | }) 45 | if err != nil { 46 | t.Fatal(err) 47 | } 48 | ca.Close() 49 | 50 | tlsConfig, err := util.GetClientTLSConfig(ca.Name(), "", "", "", []string{proto}) 51 | if err != nil { 52 | t.Fatal(err) 53 | } 54 | 55 | addr := strings.TrimPrefix(svr.URL, "https://") 56 | conn, err := grpc.Dial(addr, grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig))) 57 | if err != nil { 58 | t.Fatal(err) 59 | } 60 | grpcClient := client.NewProxyServiceClient(conn) 61 | 62 | grpcClient.Proxy(context.Background()) 63 | if atomic.LoadInt32(&protoUsed) != 1 { 64 | t.Error("expected custom ALPN protocol to have been used") 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /pkg/util/net.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package util 18 | 19 | import ( 20 | "crypto/tls" 21 | "strings" 22 | ) 23 | 24 | // containIPv6Addr checks if the given host identity contains an 25 | // IPv6 address 26 | func containIPv6Addr(host string) bool { 27 | // the shortest IPv6 address is :: 28 | return len(strings.Split(host, ":")) > 2 29 | } 30 | 31 | // containPortIPv6 checks if the host that contains an IPv6 address 32 | // also contains a port number 33 | func containPortIPv6(host string) bool { 34 | // based on to RFC 3986, section 3.2.2, host identified by an 35 | // IPv6 is distinguished by enclosing the IP literal within square 36 | // brackets ("[" and "]") 37 | return strings.ContainsRune(host, '[') 38 | } 39 | 40 | // RemovePortFromHost removes port number from the host address that 41 | // may be of the form ":" where the can be an either 42 | // an IPv4/6 address or a domain name 43 | func RemovePortFromHost(host string) string { 44 | if !containIPv6Addr(host) { 45 | return strings.Split(host, ":")[0] 46 | } 47 | if containPortIPv6(host) { 48 | host = host[:strings.LastIndexByte(host, ':')] 49 | } 50 | return strings.Trim(host, "[]") 51 | } 52 | 53 | // GetAcceptedCiphers returns all the ciphers supported by the crypto/tls package 54 | func GetAcceptedCiphers() map[string]uint16 { 55 | acceptedCiphers := make(map[string]uint16, len(tls.CipherSuites())) 56 | for _, v := range tls.CipherSuites() { 57 | acceptedCiphers[v.Name] = v.ID 58 | } 59 | return acceptedCiphers 60 | } 61 | -------------------------------------------------------------------------------- /pkg/util/certificates.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package util 18 | 19 | import ( 20 | "crypto/tls" 21 | "crypto/x509" 22 | "fmt" 23 | "io/ioutil" 24 | "path/filepath" 25 | ) 26 | 27 | // getCACertPool loads CA certificates to pool 28 | func getCACertPool(caFile string) (*x509.CertPool, error) { 29 | certPool := x509.NewCertPool() 30 | caCert, err := ioutil.ReadFile(filepath.Clean(caFile)) 31 | if err != nil { 32 | return nil, fmt.Errorf("failed to read CA cert %s: %v", caFile, err) 33 | } 34 | ok := certPool.AppendCertsFromPEM(caCert) 35 | if !ok { 36 | return nil, fmt.Errorf("failed to append CA cert to the cert pool") 37 | } 38 | return certPool, nil 39 | } 40 | 41 | // GetClientTLSConfig returns tlsConfig based on x509 certs 42 | func GetClientTLSConfig(caFile, certFile, keyFile, serverName string, protos []string) (*tls.Config, error) { 43 | certPool, err := getCACertPool(caFile) 44 | if err != nil { 45 | return nil, err 46 | } 47 | 48 | tlsConfig := &tls.Config{ 49 | RootCAs: certPool, 50 | MinVersion: tls.VersionTLS12, 51 | } 52 | if len(protos) != 0 { 53 | tlsConfig.NextProtos = protos 54 | } 55 | if certFile == "" && keyFile == "" { 56 | // return TLS config based on CA only 57 | return tlsConfig, nil 58 | } 59 | 60 | cert, err := tls.LoadX509KeyPair(certFile, keyFile) 61 | if err != nil { 62 | return nil, fmt.Errorf("failed to load X509 key pair %s and %s: %v", certFile, keyFile, err) 63 | } 64 | 65 | tlsConfig.ServerName = serverName 66 | tlsConfig.Certificates = []tls.Certificate{cert} 67 | return tlsConfig, nil 68 | } 69 | -------------------------------------------------------------------------------- /examples/kubernetes/konnectivity-server.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: konnectivity-server 5 | namespace: kube-system 6 | annotations: 7 | scheduler.alpha.kubernetes.io/critical-pod: '' 8 | seccomp.security.alpha.kubernetes.io/pod: 'docker/default' 9 | spec: 10 | hostNetwork: true 11 | containers: 12 | - name: konnectivity-server-container 13 | image: ${PROXY_IMAGE}:${TAG} 14 | resources: 15 | requests: 16 | cpu: 1m 17 | command: [ "/proxy-server"] 18 | args: [ 19 | "--log-file=/var/log/konnectivity-server.log", 20 | "--logtostderr=false", 21 | "--log-file-max-size=0", 22 | "--uds-name=/etc/srv/kubernetes/konnectivity-server/konnectivity-server.socket", 23 | "--cluster-cert=/etc/srv/kubernetes/pki/apiserver.crt", 24 | "--cluster-key=/etc/srv/kubernetes/pki/apiserver.key", 25 | "--server-port=0", 26 | "--agent-port=8091", 27 | "--health-port=8092", 28 | "--admin-port=8093", 29 | "--keepalive-time=1h", 30 | "--mode=http-connect", 31 | "--agent-namespace=kube-system", 32 | "--agent-service-account=konnectivity-agent", 33 | "--kubeconfig=/etc/srv/kubernetes/konnectivity-server/kubeconfig", 34 | "--authentication-audience=system:konnectivity-server", 35 | ] 36 | livenessProbe: 37 | httpGet: 38 | scheme: HTTP 39 | host: 127.0.0.1 40 | port: 8092 41 | path: /healthz 42 | initialDelaySeconds: 10 43 | timeoutSeconds: 60 44 | ports: 45 | - name: serverport 46 | containerPort: 8090 47 | hostPort: 8090 48 | - name: agentport 49 | containerPort: 8091 50 | hostPort: 8091 51 | - name: healthport 52 | containerPort: 8092 53 | hostPort: 8092 54 | - name: adminport 55 | containerPort: 8093 56 | hostPort: 8093 57 | volumeMounts: 58 | - name: varlogkonnectivityserver 59 | mountPath: /var/log/konnectivity-server.log 60 | readOnly: false 61 | - name: pki 62 | mountPath: /etc/srv/kubernetes/pki 63 | readOnly: true 64 | - name: konnectivity-home 65 | mountPath: /etc/srv/kubernetes/konnectivity-server 66 | volumes: 67 | - name: varlogkonnectivityserver 68 | hostPath: 69 | path: /var/log/konnectivity-server.log 70 | type: FileOrCreate 71 | - name: pki 72 | hostPath: 73 | path: /etc/srv/kubernetes/pki 74 | - name: konnectivity-home 75 | hostPath: 76 | path: /etc/srv/kubernetes/konnectivity-server 77 | type: DirectoryOrCreate 78 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module sigs.k8s.io/apiserver-network-proxy 2 | 3 | go 1.17 4 | 5 | require ( 6 | github.com/golang/mock v1.4.4 7 | github.com/golang/protobuf v1.4.3 8 | github.com/google/uuid v1.1.2 9 | github.com/prometheus/client_golang v1.7.1 10 | github.com/spf13/cobra v0.0.3 11 | github.com/spf13/pflag v1.0.5 12 | golang.org/x/net v0.0.0-20201110031124-69a78807bb2b 13 | google.golang.org/grpc v1.42.0 14 | k8s.io/api v0.20.10 15 | k8s.io/apimachinery v0.20.10 16 | k8s.io/client-go v0.20.10 17 | k8s.io/component-base v0.20.10 18 | k8s.io/klog/v2 v2.4.0 19 | sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.0 20 | ) 21 | 22 | require ( 23 | github.com/beorn7/perks v1.0.1 // indirect 24 | github.com/cespare/xxhash/v2 v2.1.1 // indirect 25 | github.com/davecgh/go-spew v1.1.1 // indirect 26 | github.com/evanphx/json-patch v4.9.0+incompatible // indirect 27 | github.com/go-logr/logr v0.2.0 // indirect 28 | github.com/gogo/protobuf v1.3.2 // indirect 29 | github.com/google/gofuzz v1.1.0 // indirect 30 | github.com/googleapis/gnostic v0.4.1 // indirect 31 | github.com/imdario/mergo v0.3.5 // indirect 32 | github.com/inconshreveable/mousetrap v1.0.0 // indirect 33 | github.com/json-iterator/go v1.1.10 // indirect 34 | github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect 35 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 36 | github.com/modern-go/reflect2 v1.0.1 // indirect 37 | github.com/pkg/errors v0.9.1 // indirect 38 | github.com/prometheus/client_model v0.2.0 // indirect 39 | github.com/prometheus/common v0.10.0 // indirect 40 | github.com/prometheus/procfs v0.2.0 // indirect 41 | golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 // indirect 42 | golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect 43 | golang.org/x/sys v0.0.0-20201112073958-5cba982894dd // indirect 44 | golang.org/x/text v0.3.4 // indirect 45 | golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect 46 | google.golang.org/appengine v1.6.5 // indirect 47 | google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 // indirect 48 | google.golang.org/protobuf v1.26.0-rc.1 // indirect 49 | gopkg.in/inf.v0 v0.9.1 // indirect 50 | gopkg.in/yaml.v2 v2.2.8 // indirect 51 | k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd // indirect 52 | k8s.io/utils v0.0.0-20201110183641-67b214c5f920 // indirect 53 | sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect 54 | sigs.k8s.io/yaml v1.2.0 // indirect 55 | ) 56 | 57 | replace sigs.k8s.io/apiserver-network-proxy/konnectivity-client => ./konnectivity-client 58 | -------------------------------------------------------------------------------- /konnectivity-client/proto/client/client.proto: -------------------------------------------------------------------------------- 1 | // Copyright The Kubernetes Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | syntax = "proto3"; 16 | 17 | // Retransmit? 18 | // Sliding windows? 19 | 20 | option go_package = "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client"; 21 | 22 | service ProxyService { 23 | rpc Proxy(stream Packet) returns (stream Packet) {} 24 | } 25 | 26 | enum PacketType { 27 | DIAL_REQ = 0; 28 | DIAL_RSP = 1; 29 | CLOSE_REQ = 2; 30 | CLOSE_RSP = 3; 31 | DATA = 4; 32 | DIAL_CLS = 5; 33 | } 34 | 35 | enum Error { 36 | EOF = 0; 37 | // ... 38 | } 39 | 40 | message Packet { 41 | PacketType type = 1; 42 | 43 | oneof payload { 44 | DialRequest dialRequest = 2; 45 | DialResponse dialResponse = 3; 46 | Data data = 4; 47 | CloseRequest closeRequest = 5; 48 | CloseResponse closeResponse = 6; 49 | CloseDial closeDial = 7; 50 | } 51 | } 52 | 53 | message DialRequest { 54 | // tcp or udp? 55 | string protocol = 1; 56 | 57 | // node:port 58 | string address = 2; 59 | 60 | // random id for client, maybe should be longer 61 | int64 random = 3; 62 | } 63 | 64 | message DialResponse { 65 | // error failed reason; enum? 66 | string error = 1; 67 | 68 | // connectID indicates the identifier of the connection 69 | int64 connectID = 2; 70 | 71 | // random copied from DialRequest 72 | int64 random = 3; 73 | } 74 | 75 | message CloseRequest { 76 | // connectID of the stream to close 77 | int64 connectID = 1; 78 | } 79 | 80 | message CloseResponse { 81 | // error message 82 | string error = 1; 83 | 84 | // connectID indicates the identifier of the connection 85 | int64 connectID = 2; 86 | } 87 | 88 | message CloseDial { 89 | // random id of the DialRequest 90 | int64 random = 1; 91 | } 92 | 93 | message Data { 94 | // connectID to connect to 95 | int64 connectID = 1; 96 | 97 | // error message if error happens 98 | string error = 2; 99 | 100 | // stream data 101 | bytes data = 3; 102 | } 103 | -------------------------------------------------------------------------------- /tests/tcp_server_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "context" 5 | "net" 6 | "testing" 7 | "time" 8 | 9 | "google.golang.org/grpc" 10 | "k8s.io/klog/v2" 11 | "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client" 12 | ) 13 | 14 | func echo(conn net.Conn) { 15 | var data [256]byte 16 | 17 | for { 18 | n, err := conn.Read(data[:]) 19 | if err != nil { 20 | klog.Info(err) 21 | return 22 | } 23 | 24 | _, err = conn.Write(data[:n]) 25 | if err != nil { 26 | klog.Info(err) 27 | return 28 | } 29 | } 30 | } 31 | 32 | func TestEchoServer(t *testing.T) { 33 | ctx := context.Background() 34 | ln, err := net.Listen("tcp", "") 35 | if err != nil { 36 | t.Error(err) 37 | } 38 | 39 | go func() { 40 | for { 41 | conn, err := ln.Accept() 42 | if err != nil { 43 | klog.Info(err) 44 | break 45 | } 46 | go echo(conn) 47 | } 48 | }() 49 | 50 | stopCh := make(chan struct{}) 51 | defer close(stopCh) 52 | 53 | proxy, cleanup, err := runGRPCProxyServer() 54 | if err != nil { 55 | t.Fatal(err) 56 | } 57 | defer cleanup() 58 | 59 | runAgent(proxy.agent, stopCh) 60 | 61 | // Wait for agent to register on proxy server 62 | time.Sleep(time.Second) 63 | 64 | // run test client 65 | tunnel, err := client.CreateSingleUseGrpcTunnel(ctx, proxy.front, grpc.WithInsecure()) 66 | if err != nil { 67 | t.Fatal(err) 68 | } 69 | 70 | conn, err := tunnel.DialContext(ctx, "tcp", ln.Addr().String()) 71 | if err != nil { 72 | t.Error(err) 73 | } 74 | 75 | msg := "1234567890123456789012345" 76 | n, err := conn.Write([]byte(msg)) 77 | if err != nil { 78 | t.Error(err) 79 | } 80 | if n != len(msg) { 81 | t.Errorf("expect write %d; got %d", len(msg), n) 82 | } 83 | 84 | var data [10]byte 85 | 86 | n, err = conn.Read(data[:]) 87 | if err != nil { 88 | t.Error(err) 89 | } 90 | if string(data[:n]) != msg[:10] { 91 | t.Errorf("expect %s; got %s", msg[:10], string(data[:n])) 92 | } 93 | 94 | n, err = conn.Read(data[:]) 95 | if err != nil { 96 | t.Error(err) 97 | } 98 | if string(data[:n]) != msg[10:20] { 99 | t.Errorf("expect %s; got %s", msg[10:20], string(data[:n])) 100 | } 101 | 102 | msg2 := "1234567" 103 | n, err = conn.Write([]byte(msg2)) 104 | if err != nil { 105 | t.Error(err) 106 | } 107 | if n != len(msg2) { 108 | t.Errorf("expect write %d; got %d", len(msg2), n) 109 | } 110 | 111 | n, err = conn.Read(data[:]) 112 | if err != nil { 113 | t.Error(err) 114 | } 115 | if string(data[:n]) != msg[20:] { 116 | t.Errorf("expect %s; got %s", msg[20:], string(data[:n])) 117 | } 118 | 119 | n, err = conn.Read(data[:]) 120 | if err != nil { 121 | t.Error(err) 122 | } 123 | if string(data[:n]) != msg2 { 124 | t.Errorf("expect %s; got %s", msg, string(data[:n])) 125 | } 126 | 127 | if err := conn.Close(); err != nil { 128 | t.Error(err) 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /RELEASE.md: -------------------------------------------------------------------------------- 1 | # Releasing a new version of apiserver-network-proxy 2 | 3 | Please note this guide is only intended for the admins of this repository, and requires write access. 4 | 5 | Creating a new release of network proxy involves releasing a new version of the client library (konnectivity-client) and new images for the proxy agent and server. Generally we also want to upgrade kubernetes/kubernetes with the latest version of the images and library, but this is a GCE specific change. 6 | 7 | 1. The first step involves creating a new git tag for the release, following semvar for go libraries. A tag is required for both the repository and the konnectivity-client library. For example releasing the `0.0.15` version will have two tags `v0.0.15` and `konnectivity-client/v0.0.15` on the appropriate commit. 8 | 9 | The exact commands are 10 | 11 | ``` 12 | # Check out the appropriate commit (usually head of master) 13 | git tag -a v0.0.15 14 | git tag konnectivity-client/v0.0.15 15 | git push upstream v0.0.15 16 | git push upstream konnectivity-client/v0.0.15 17 | ``` 18 | 19 | Once the two tags are created, the konnectivity-client can be imported as a library in kubernetes/kubernetes and other go programs. 20 | 21 | 2. To publish the proxy server and proxy agent images, they must be promoted from the k8s staging repo. An example PR can be seen here: [https://github.com/kubernetes/k8s.io/pull/1602](https://github.com/kubernetes/k8s.io/pull/1602) 22 | 23 | The SHA in the PR corresponds to the SHA of the image within the k8s staging repo. (This is under the **Name** column) 24 | 25 | The images can be found here for the [proxy server](http://console.cloud.google.com/gcr/images/k8s-staging-kas-network-proxy/GLOBAL/proxy-server?gcrImageListsize=30) and [proxy agent](http://console.cloud.google.com/gcr/images/k8s-staging-kas-network-proxy/GLOBAL/proxy-agent?gcrImageListsize=30). 26 | 27 | Please ensure that the commit shown on the tag of the image matches the one shown in the tags page in the network proxy repo. 28 | 29 | 30 | 31 | 3. Finally, update kubernetes/kubernetes with the new client library and images. 32 | 33 | An example PR can be found here: [https://github.com/kubernetes/kubernetes/pull/94983](https://github.com/kubernetes/kubernetes/pull/94983) 34 | 35 | Image paths must be bumped and are located in: 36 | 37 | - cluster/gce/addons/konnectivity-agent/konnectivity-agent-ds.yaml 38 | - cluster/gce/manifests/konnectivity-server.yaml 39 | 40 | To update the library, no go.mod files need to be manually modified and running these commands in k/k will perform the necessary go.mod changes. 41 | 42 | ``` 43 | ./hack/pin-dependency.sh sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15 44 | make clean generated_files 45 | ./hack/update-vendor.sh 46 | ``` 47 | -------------------------------------------------------------------------------- /pkg/agent/metrics/metrics.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package metrics 18 | 19 | import ( 20 | "time" 21 | 22 | "github.com/prometheus/client_golang/prometheus" 23 | ) 24 | 25 | type Direction string 26 | 27 | const ( 28 | namespace = "konnectivity_network_proxy" 29 | subsystem = "agent" 30 | 31 | // DirectionToServer indicates that the agent attempts to send a packet 32 | // to the proxy server. 33 | DirectionToServer Direction = "to_server" 34 | // DirectionFromServer indicates that the agent attempts to receive a 35 | // packet from the proxy server. 36 | DirectionFromServer Direction = "from_server" 37 | ) 38 | 39 | var ( 40 | // Use buckets ranging from 5 ms to 30 seconds. 41 | latencyBuckets = []float64{0.005, 0.025, 0.1, 0.5, 2.5, 10, 30} 42 | 43 | // Metrics provides access to all dial metrics. 44 | Metrics = newAgentMetrics() 45 | ) 46 | 47 | // AgentMetrics includes all the metrics of the proxy agent. 48 | type AgentMetrics struct { 49 | latencies *prometheus.HistogramVec 50 | failures *prometheus.CounterVec 51 | } 52 | 53 | // newAgentMetrics create a new AgentMetrics, configured with default metric names. 54 | func newAgentMetrics() *AgentMetrics { 55 | latencies := prometheus.NewHistogramVec( 56 | prometheus.HistogramOpts{ 57 | Namespace: namespace, 58 | Subsystem: subsystem, 59 | Name: "dial_duration_seconds", 60 | Help: "Latency of dial to the remote endpoint in seconds", 61 | Buckets: latencyBuckets, 62 | }, 63 | []string{}, 64 | ) 65 | failures := prometheus.NewCounterVec( 66 | prometheus.CounterOpts{ 67 | Namespace: namespace, 68 | Subsystem: subsystem, 69 | Name: "server_connection_failure_count", 70 | Help: "Count of failures to send to or receive from the proxy server, labeled by the direction (from_server or to_server)", 71 | }, 72 | []string{"direction"}, 73 | ) 74 | prometheus.MustRegister(failures) 75 | prometheus.MustRegister(latencies) 76 | return &AgentMetrics{failures: failures, latencies: latencies} 77 | } 78 | 79 | // Reset resets the metrics. 80 | func (a *AgentMetrics) Reset() { 81 | a.failures.Reset() 82 | a.latencies.Reset() 83 | } 84 | 85 | // ObserveFailure records a failure to send to or receive from the proxy 86 | // server, labeled by the direction. 87 | func (a *AgentMetrics) ObserveFailure(direction Direction) { 88 | a.failures.WithLabelValues(string(direction)).Inc() 89 | } 90 | 91 | // ObserveDialLatency records the latency of dial to the remote endpoint. 92 | func (a *AgentMetrics) ObserveDialLatency(elapsed time.Duration) { 93 | a.latencies.WithLabelValues().Observe(elapsed.Seconds()) 94 | } 95 | -------------------------------------------------------------------------------- /examples/kubernetes/README.md: -------------------------------------------------------------------------------- 1 | # HTTP-Connect Client using UDS Proxy with dial back Agent runs on kubernetes cluster 2 | 3 | # Push all images to container registry 4 | ```console 5 | TAG=$(git rev-parse HEAD) 6 | make docker-push TAG=${TAG} 7 | ``` 8 | 9 | # Start a test web-server as a kubernetes pod & service 10 | ```bash 11 | kubectl apply -f examples/kubernetes/kubia.yaml 12 | ``` 13 | 14 | # Initialize environment variables 15 | *CLUSTER_CERT* and *CLUSTER_KEY* are certificates used for starting [kubernetes API server](https://kubernetes.io/docs/concepts/cluster-administration/certificates/) 16 | 17 | ```bash 18 | CLUSTER_IP=$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}' | sed -n "s/https\:\/\/\(\S*\).*$/\1/p") 19 | KUBIA_IP=$(kubectl get svc kubia -o=jsonpath='{.spec.clusterIP}') 20 | PROXY_IMAGE=$(docker images | grep "proxy-server-" -m1 | awk '{print $1}') 21 | AGENT_IMAGE=$(docker images | grep "proxy-agent-" -m1 | awk '{print $1}') 22 | TEST_CLIENT_IMAGE=$(docker images | grep "proxy-test-client-" -m1 | awk '{print $1}') 23 | SERVER_TOKEN=$(./examples/kubernetes/token_generation.sh 32) 24 | CLUSTER_CERT= 25 | CLUSTER_KEY= 26 | ``` 27 | 28 | #### GCE sample configuration 29 | ```bash 30 | CLUSTER_CERT=/etc/srv/kubernetes/pki/apiserver.crt 31 | CLUSTER_KEY=/etc/srv/kubernetes/pki/apiserver.key 32 | ``` 33 | 34 | # Register SERVER_TOKEN in [static-token-file](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#static-token-file) 35 | Append the output of the following line to the [static-token-file](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#static-token-file) and restart **kube-apiserver** on the control plane. 36 | ```bash 37 | echo "${SERVER_TOKEN},system:konnectivity-server,uid:system:konnectivity-server" 38 | ``` 39 | 40 | #### GCE sample configuration 41 | 1. [static-token-file](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#static-token-file) location is: **/etc/srv/kubernetes/known_tokens.csv** 42 | 43 | 1. Restart kube-apiserver 44 | ```bash 45 | K8S_API_PID=$(sudo crictl ps | grep kube-apiserver | awk '{ print $1; }') 46 | sudo crictl stop ${K8S_API_PID} 47 | ``` 48 | 49 | # Save following config at /etc/srv/kubernetes/konnectivity-server/kubeconfig on control plane VM 50 | ```bash 51 | SERVER_TOKEN=${SERVER_TOKEN} envsubst < examples/kubernetes/kubeconfig 52 | ``` 53 | 54 | # Create a clusterrolebinding allowing proxy-server authenticate proxy-client 55 | ```bash 56 | kubectl create clusterrolebinding --user system:konnectivity-server --clusterrole system:auth-delegator system:konnectivity-server 57 | ``` 58 | 59 | # Start **proxy-server** 60 | 61 | - as a [static pod](https://kubernetes.io/docs/tasks/configure-pod-container/static-pod/) with following configuration 62 | ```bash 63 | TAG=${TAG} PROXY_IMAGE=${PROXY_IMAGE} CLUSTER_CERT=${CLUSTER_CERT} CLUSTER_KEY=${CLUSTER_KEY} envsubst < examples/kubernetes/konnectivity-server.yaml 64 | ``` 65 | 66 | - as a kubernetes pod with following configuration 67 | ```bash 68 | TAG=${TAG} PROXY_IMAGE=${PROXY_IMAGE} CLUSTER_CERT=${CLUSTER_CERT} CLUSTER_KEY=${CLUSTER_KEY} envsubst < examples/kubernetes/konnectivity-server.yaml | kubectl apply -f - 69 | ``` 70 | 71 | #### GKE specific configuration 72 | */etc/kubernetes/manifests* is a folder where .yaml file needs to be created for static pod 73 | 74 | # Start **proxy-agent** as a kubernetes pod 75 | ```bash 76 | TAG=${TAG} AGENT_IMAGE=${AGENT_IMAGE} CLUSTER_IP=${CLUSTER_IP} envsubst < examples/kubernetes/konnectivity-agent.yaml | kubectl apply -f - 77 | ``` 78 | 79 | # Run **test-client** as a [static pod](https://kubernetes.io/docs/tasks/configure-pod-container/static-pod/) with following configuration on same machine where **proxy-server** runs 80 | ```bash 81 | TAG=${TAG} KUBIA_IP=${KUBIA_IP} TEST_CLIENT_IMAGE=${TEST_CLIENT_IMAGE} envsubst < examples/kubernetes/konnectivity-test-client.yaml 82 | ``` 83 | 84 | Last row in the following log file **/var/log/konnectivity-test-client.log** supposed to be: **You've hit kubia** 85 | 86 | #### GKE specific configuration 87 | */etc/kubernetes/manifests* is a folder where .yaml file needs to be created for static pod 88 | -------------------------------------------------------------------------------- /konnectivity-client/pkg/client/conn.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package client 18 | 19 | import ( 20 | "errors" 21 | "io" 22 | "net" 23 | "time" 24 | 25 | "k8s.io/klog/v2" 26 | "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client" 27 | ) 28 | 29 | // CloseTimeout is the timeout to wait CLOSE_RSP packet after a 30 | // successful delivery of CLOSE_REQ. 31 | const CloseTimeout = 10 * time.Second 32 | 33 | // conn is an implementation of net.Conn, where the data is transported 34 | // over an established tunnel defined by a gRPC service ProxyService. 35 | type conn struct { 36 | stream client.ProxyService_ProxyClient 37 | connID int64 38 | random int64 39 | readCh chan []byte 40 | closeCh chan string 41 | rdata []byte 42 | } 43 | 44 | var _ net.Conn = &conn{} 45 | 46 | // Write sends the data thru the connection over proxy service 47 | func (c *conn) Write(data []byte) (n int, err error) { 48 | req := &client.Packet{ 49 | Type: client.PacketType_DATA, 50 | Payload: &client.Packet_Data{ 51 | Data: &client.Data{ 52 | ConnectID: c.connID, 53 | Data: data, 54 | }, 55 | }, 56 | } 57 | 58 | klog.V(5).InfoS("[tracing] send req", "type", req.Type) 59 | 60 | err = c.stream.Send(req) 61 | if err != nil { 62 | return 0, err 63 | } 64 | return len(data), err 65 | } 66 | 67 | // Read receives data from the connection over proxy service 68 | func (c *conn) Read(b []byte) (n int, err error) { 69 | var data []byte 70 | 71 | if c.rdata != nil { 72 | data = c.rdata 73 | } else { 74 | data = <-c.readCh 75 | } 76 | 77 | if data == nil { 78 | return 0, io.EOF 79 | } 80 | 81 | if len(data) > len(b) { 82 | copy(b, data[:len(b)]) 83 | c.rdata = data[len(b):] 84 | return len(b), nil 85 | } 86 | 87 | c.rdata = nil 88 | copy(b, data) 89 | 90 | return len(data), nil 91 | } 92 | 93 | func (c *conn) LocalAddr() net.Addr { 94 | return nil 95 | } 96 | 97 | func (c *conn) RemoteAddr() net.Addr { 98 | return nil 99 | } 100 | 101 | func (c *conn) SetDeadline(t time.Time) error { 102 | return errors.New("not implemented") 103 | } 104 | 105 | func (c *conn) SetReadDeadline(t time.Time) error { 106 | return errors.New("not implemented") 107 | } 108 | 109 | func (c *conn) SetWriteDeadline(t time.Time) error { 110 | return errors.New("not implemented") 111 | } 112 | 113 | // Close closes the connection. It also sends CLOSE_REQ packet over 114 | // proxy service to notify remote to drop the connection. 115 | func (c *conn) Close() error { 116 | klog.V(4).Infoln("closing connection") 117 | var req *client.Packet 118 | if c.connID != 0 { 119 | req = &client.Packet{ 120 | Type: client.PacketType_CLOSE_REQ, 121 | Payload: &client.Packet_CloseRequest{ 122 | CloseRequest: &client.CloseRequest{ 123 | ConnectID: c.connID, 124 | }, 125 | }, 126 | } 127 | } else { 128 | // Never received a DIAL response so no connection ID. 129 | req = &client.Packet{ 130 | Type: client.PacketType_DIAL_CLS, 131 | Payload: &client.Packet_CloseDial{ 132 | CloseDial: &client.CloseDial{ 133 | Random: c.random, 134 | }, 135 | }, 136 | } 137 | } 138 | 139 | klog.V(5).InfoS("[tracing] send req", "type", req.Type) 140 | 141 | if err := c.stream.Send(req); err != nil { 142 | return err 143 | } 144 | 145 | select { 146 | case errMsg := <-c.closeCh: 147 | if errMsg != "" { 148 | return errors.New(errMsg) 149 | } 150 | return nil 151 | case <-time.After(CloseTimeout): 152 | } 153 | 154 | return errors.New("close timeout") 155 | } 156 | -------------------------------------------------------------------------------- /tests/concurrent_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "context" 5 | "io/ioutil" 6 | "net/http" 7 | "net/http/httptest" 8 | "sync" 9 | "testing" 10 | "time" 11 | 12 | "google.golang.org/grpc" 13 | "k8s.io/apimachinery/pkg/util/wait" 14 | "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client" 15 | ) 16 | 17 | func TestProxy_Concurrency(t *testing.T) { 18 | ctx := context.Background() 19 | length := 1 << 20 20 | chunks := 10 21 | server := httptest.NewServer(newSizedServer(length, chunks)) 22 | defer server.Close() 23 | 24 | stopCh := make(chan struct{}) 25 | defer close(stopCh) 26 | 27 | proxy, cleanup, err := runGRPCProxyServer() 28 | if err != nil { 29 | t.Fatal(err) 30 | } 31 | defer cleanup() 32 | 33 | runAgent(proxy.agent, stopCh) 34 | 35 | // Wait for agent to register on proxy server 36 | time.Sleep(time.Second) 37 | 38 | // run test client 39 | tunnel, err := client.CreateSingleUseGrpcTunnel(ctx, proxy.front, grpc.WithInsecure()) 40 | if err != nil { 41 | t.Fatal(err) 42 | } 43 | 44 | var wg sync.WaitGroup 45 | verify := func() { 46 | defer wg.Done() 47 | 48 | c := &http.Client{ 49 | Transport: &http.Transport{ 50 | DialContext: tunnel.DialContext, 51 | }, 52 | } 53 | 54 | r, err := c.Get(server.URL) 55 | if err != nil { 56 | t.Error(err) 57 | } 58 | 59 | data, err := ioutil.ReadAll(r.Body) 60 | if err != nil { 61 | t.Error(err) 62 | } 63 | defer r.Body.Close() 64 | 65 | if len(data) != length*chunks { 66 | t.Errorf("expect data length %d; got %d", length*chunks, len(data)) 67 | } 68 | } 69 | 70 | concurrency := 10 71 | wg.Add(concurrency) 72 | for i := 0; i < concurrency; i++ { 73 | go verify() 74 | } 75 | wg.Wait() 76 | } 77 | 78 | // This test verifies that when one stream between a proxy agent and the proxy 79 | // server terminates, the proxy server does not terminate other frontends 80 | // supported by the same proxy agent but on different streams. 81 | func TestAgent_MultipleConn(t *testing.T) { 82 | testcases := []struct { 83 | name string 84 | proxyServerFunction func() (proxy, func(), error) 85 | clientFunction func(context.Context, string, string) (*http.Client, error) 86 | }{ 87 | { 88 | name: "grpc", 89 | proxyServerFunction: runGRPCProxyServer, 90 | clientFunction: createGrpcTunnelClient, 91 | }, 92 | { 93 | name: "http-connect", 94 | proxyServerFunction: runHTTPConnProxyServer, 95 | clientFunction: createHTTPConnectClient, 96 | }, 97 | } 98 | 99 | for _, tc := range testcases { 100 | t.Run(tc.name, func(t *testing.T) { 101 | ctx := context.Background() 102 | echoServer := newEchoServer("hello") 103 | echoServer.wchan = make(chan struct{}) 104 | server := httptest.NewServer(echoServer) 105 | defer server.Close() 106 | 107 | stopCh := make(chan struct{}) 108 | stopCh2 := make(chan struct{}) 109 | 110 | proxy, cleanup, err := tc.proxyServerFunction() 111 | if err != nil { 112 | t.Fatal(err) 113 | } 114 | defer cleanup() 115 | 116 | runAgentWithID("multipleAgentConn", proxy.agent, stopCh) 117 | defer close(stopCh) 118 | 119 | // Wait for agent to register on proxy server 120 | wait.Poll(100*time.Millisecond, 5*time.Second, func() (bool, error) { 121 | ready, _ := proxy.server.Readiness.Ready() 122 | return ready, nil 123 | }) 124 | 125 | // run test client 126 | c, err := tc.clientFunction(ctx, proxy.front, server.URL) 127 | 128 | fcnStopCh := make(chan struct{}) 129 | 130 | go func() { 131 | _, err := clientRequest(c, server.URL) 132 | if err != nil { 133 | t.Errorf("expected no error on proxy request, got %v", err) 134 | } 135 | close(fcnStopCh) 136 | }() 137 | 138 | // Running an agent with the same ID simulates a second connection from the same agent. 139 | // This simulates the scenario where a proxy agent established connections with HA proxy server 140 | // and creates multiple connections with the same proxy server 141 | runAgentWithID("multipleAgentConn", proxy.agent, stopCh2) 142 | close(stopCh2) 143 | // Wait for the server to run cleanup routine 144 | time.Sleep(1 * time.Second) 145 | close(echoServer.wchan) 146 | 147 | <-fcnStopCh 148 | }) 149 | } 150 | } 151 | -------------------------------------------------------------------------------- /cmd/agent/app/server.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "crypto/tls" 5 | "fmt" 6 | "net" 7 | "net/http" 8 | "net/http/pprof" 9 | "runtime" 10 | 11 | "github.com/prometheus/client_golang/prometheus/promhttp" 12 | "github.com/spf13/cobra" 13 | "google.golang.org/grpc" 14 | "google.golang.org/grpc/credentials" 15 | "google.golang.org/grpc/keepalive" 16 | "k8s.io/klog/v2" 17 | 18 | "sigs.k8s.io/apiserver-network-proxy/cmd/agent/app/options" 19 | "sigs.k8s.io/apiserver-network-proxy/pkg/util" 20 | ) 21 | 22 | func NewAgentCommand(a *Agent, o *options.GrpcProxyAgentOptions) *cobra.Command { 23 | cmd := &cobra.Command{ 24 | Use: "agent", 25 | Long: `A gRPC agent, Connects to the proxy and then allows traffic to be forwarded to it.`, 26 | RunE: func(cmd *cobra.Command, args []string) error { 27 | return a.run(o) 28 | }, 29 | } 30 | 31 | return cmd 32 | } 33 | 34 | type Agent struct { 35 | } 36 | 37 | func (a *Agent) run(o *options.GrpcProxyAgentOptions) error { 38 | o.Print() 39 | if err := o.Validate(); err != nil { 40 | return fmt.Errorf("failed to validate agent options with %v", err) 41 | } 42 | 43 | stopCh := make(chan struct{}) 44 | if err := a.runProxyConnection(o, stopCh); err != nil { 45 | return fmt.Errorf("failed to run proxy connection with %v", err) 46 | } 47 | 48 | if err := a.runHealthServer(o); err != nil { 49 | return fmt.Errorf("failed to run health server with %v", err) 50 | } 51 | 52 | if err := a.runAdminServer(o); err != nil { 53 | return fmt.Errorf("failed to run admin server with %v", err) 54 | } 55 | 56 | <-stopCh 57 | 58 | return nil 59 | } 60 | 61 | func (a *Agent) runProxyConnection(o *options.GrpcProxyAgentOptions, stopCh <-chan struct{}) error { 62 | var tlsConfig *tls.Config 63 | var err error 64 | if tlsConfig, err = util.GetClientTLSConfig(o.CaCert, o.AgentCert, o.AgentKey, o.ProxyServerHost, o.AlpnProtos); err != nil { 65 | return err 66 | } 67 | dialOptions := []grpc.DialOption{ 68 | grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)), 69 | grpc.WithKeepaliveParams(keepalive.ClientParameters{ 70 | Time: o.KeepaliveTime, 71 | PermitWithoutStream: true, 72 | }), 73 | } 74 | cc := o.ClientSetConfig(dialOptions...) 75 | cs := cc.NewAgentClientSet(stopCh) 76 | cs.Serve() 77 | 78 | return nil 79 | } 80 | 81 | func (a *Agent) runHealthServer(o *options.GrpcProxyAgentOptions) error { 82 | livenessHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 83 | fmt.Fprintf(w, "ok") 84 | }) 85 | readinessHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 86 | fmt.Fprintf(w, "ok") 87 | }) 88 | 89 | muxHandler := http.NewServeMux() 90 | muxHandler.Handle("/metrics", promhttp.Handler()) 91 | muxHandler.HandleFunc("/healthz", livenessHandler) 92 | // "/ready" is deprecated but being maintained for backward compatibility 93 | muxHandler.HandleFunc("/ready", readinessHandler) 94 | muxHandler.HandleFunc("/readyz", readinessHandler) 95 | healthServer := &http.Server{ 96 | Addr: fmt.Sprintf(":%d", o.HealthServerPort), 97 | Handler: muxHandler, 98 | MaxHeaderBytes: 1 << 20, 99 | } 100 | 101 | go func() { 102 | err := healthServer.ListenAndServe() 103 | if err != nil { 104 | klog.ErrorS(err, "health server could not listen") 105 | } 106 | klog.V(0).Infoln("Health server stopped listening") 107 | }() 108 | 109 | return nil 110 | } 111 | 112 | func (a *Agent) runAdminServer(o *options.GrpcProxyAgentOptions) error { 113 | muxHandler := http.NewServeMux() 114 | muxHandler.Handle("/metrics", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 115 | host, _, err := net.SplitHostPort(r.Host) 116 | // The port number may be omitted if the admin server is running on port 117 | // 80, the default port for HTTP 118 | if err != nil { 119 | host = r.Host 120 | } 121 | http.Redirect(w, r, fmt.Sprintf("%s:%d%s", host, o.HealthServerPort, r.URL.Path), http.StatusMovedPermanently) 122 | })) 123 | if o.EnableProfiling { 124 | muxHandler.HandleFunc("/debug/pprof", util.RedirectTo("/debug/pprof/")) 125 | muxHandler.HandleFunc("/debug/pprof/", pprof.Index) 126 | if o.EnableContentionProfiling { 127 | runtime.SetBlockProfileRate(1) 128 | } 129 | } 130 | 131 | adminServer := &http.Server{ 132 | Addr: fmt.Sprintf("127.0.0.1:%d", o.AdminServerPort), 133 | Handler: muxHandler, 134 | MaxHeaderBytes: 1 << 20, 135 | } 136 | 137 | go func() { 138 | err := adminServer.ListenAndServe() 139 | if err != nil { 140 | klog.ErrorS(err, "admin server could not listen") 141 | } 142 | klog.V(0).Infoln("Admin server stopped listening") 143 | }() 144 | 145 | return nil 146 | } 147 | -------------------------------------------------------------------------------- /pkg/server/backend_manager_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package server 18 | 19 | import ( 20 | "reflect" 21 | "testing" 22 | 23 | pkgagent "sigs.k8s.io/apiserver-network-proxy/pkg/agent" 24 | "sigs.k8s.io/apiserver-network-proxy/proto/agent" 25 | ) 26 | 27 | type fakeAgentServiceConnectServer struct { 28 | agent.AgentService_ConnectServer 29 | } 30 | 31 | func TestAddRemoveBackends(t *testing.T) { 32 | conn1 := new(fakeAgentServiceConnectServer) 33 | conn12 := new(fakeAgentServiceConnectServer) 34 | conn2 := new(fakeAgentServiceConnectServer) 35 | conn22 := new(fakeAgentServiceConnectServer) 36 | conn3 := new(fakeAgentServiceConnectServer) 37 | 38 | p := NewDefaultBackendManager() 39 | 40 | p.AddBackend("agent1", pkgagent.UID, conn1) 41 | p.RemoveBackend("agent1", pkgagent.UID, conn1) 42 | expectedBackends := make(map[string][]*backend) 43 | expectedAgentIDs := []string{} 44 | if e, a := expectedBackends, p.backends; !reflect.DeepEqual(e, a) { 45 | t.Errorf("expected %v, got %v", e, a) 46 | } 47 | if e, a := expectedAgentIDs, p.agentIDs; !reflect.DeepEqual(e, a) { 48 | t.Errorf("expected %v, got %v", e, a) 49 | } 50 | 51 | p = NewDefaultBackendManager() 52 | p.AddBackend("agent1", pkgagent.UID, conn1) 53 | p.AddBackend("agent1", pkgagent.UID, conn12) 54 | // Adding the same connection again should be a no-op. 55 | p.AddBackend("agent1", pkgagent.UID, conn12) 56 | p.AddBackend("agent2", pkgagent.UID, conn2) 57 | p.AddBackend("agent2", pkgagent.UID, conn22) 58 | p.AddBackend("agent3", pkgagent.UID, conn3) 59 | p.RemoveBackend("agent2", pkgagent.UID, conn22) 60 | p.RemoveBackend("agent2", pkgagent.UID, conn2) 61 | p.RemoveBackend("agent1", pkgagent.UID, conn1) 62 | // This is invalid. agent1 doesn't have conn3. This should be a no-op. 63 | p.RemoveBackend("agent1", pkgagent.UID, conn3) 64 | expectedBackends = map[string][]*backend{ 65 | "agent1": {newBackend(conn12)}, 66 | "agent3": {newBackend(conn3)}, 67 | } 68 | expectedAgentIDs = []string{"agent1", "agent3"} 69 | if e, a := expectedBackends, p.backends; !reflect.DeepEqual(e, a) { 70 | t.Errorf("expected %v, got %v", e, a) 71 | } 72 | if e, a := expectedAgentIDs, p.agentIDs; !reflect.DeepEqual(e, a) { 73 | t.Errorf("expected %v, got %v", e, a) 74 | } 75 | } 76 | 77 | func TestAddRemoveBackendsWithDefaultRoute(t *testing.T) { 78 | conn1 := new(fakeAgentServiceConnectServer) 79 | conn12 := new(fakeAgentServiceConnectServer) 80 | conn2 := new(fakeAgentServiceConnectServer) 81 | conn22 := new(fakeAgentServiceConnectServer) 82 | conn3 := new(fakeAgentServiceConnectServer) 83 | 84 | p := NewDefaultRouteBackendManager() 85 | 86 | p.AddBackend("agent1", pkgagent.DefaultRoute, conn1) 87 | p.RemoveBackend("agent1", pkgagent.DefaultRoute, conn1) 88 | expectedBackends := make(map[string][]*backend) 89 | expectedAgentIDs := []string{} 90 | if e, a := expectedBackends, p.backends; !reflect.DeepEqual(e, a) { 91 | t.Errorf("expected %v, got %v", e, a) 92 | } 93 | if e, a := expectedAgentIDs, p.agentIDs; !reflect.DeepEqual(e, a) { 94 | t.Errorf("expected %v, got %v", e, a) 95 | } 96 | if e, a := expectedAgentIDs, p.defaultRouteAgentIDs; !reflect.DeepEqual(e, a) { 97 | t.Errorf("expected %v, got %v", e, a) 98 | } 99 | 100 | p = NewDefaultRouteBackendManager() 101 | p.AddBackend("agent1", pkgagent.DefaultRoute, conn1) 102 | p.AddBackend("agent1", pkgagent.DefaultRoute, conn12) 103 | // Adding the same connection again should be a no-op. 104 | p.AddBackend("agent1", pkgagent.DefaultRoute, conn12) 105 | p.AddBackend("agent2", pkgagent.DefaultRoute, conn2) 106 | p.AddBackend("agent2", pkgagent.DefaultRoute, conn22) 107 | p.AddBackend("agent3", pkgagent.DefaultRoute, conn3) 108 | p.RemoveBackend("agent2", pkgagent.DefaultRoute, conn22) 109 | p.RemoveBackend("agent2", pkgagent.DefaultRoute, conn2) 110 | p.RemoveBackend("agent1", pkgagent.DefaultRoute, conn1) 111 | // This is invalid. agent1 doesn't have conn3. This should be a no-op. 112 | p.RemoveBackend("agent1", pkgagent.DefaultRoute, conn3) 113 | 114 | expectedBackends = map[string][]*backend{ 115 | "agent1": {newBackend(conn12)}, 116 | "agent3": {newBackend(conn3)}, 117 | } 118 | expectedDefaultRouteAgentIDs := []string{"agent1", "agent3"} 119 | 120 | if e, a := expectedBackends, p.backends; !reflect.DeepEqual(e, a) { 121 | t.Errorf("expected %v, got %v", e, a) 122 | } 123 | if e, a := expectedDefaultRouteAgentIDs, p.defaultRouteAgentIDs; !reflect.DeepEqual(e, a) { 124 | t.Errorf("expected %v, got %v", e, a) 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /tests/concurrent_client_request_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "io/ioutil" 8 | "net/http" 9 | "net/http/httptest" 10 | "sync" 11 | "testing" 12 | "time" 13 | 14 | "google.golang.org/grpc" 15 | "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client" 16 | pkgagent "sigs.k8s.io/apiserver-network-proxy/pkg/agent" 17 | "sigs.k8s.io/apiserver-network-proxy/pkg/server" 18 | "sigs.k8s.io/apiserver-network-proxy/proto/agent" 19 | ) 20 | 21 | type simpleServer struct { 22 | receivedSecondReq chan struct{} 23 | } 24 | 25 | // ServeHTTP blocks the response to the request whose body is "1" until a 26 | // request whose body is "2" is handled. 27 | func (s *simpleServer) ServeHTTP(w http.ResponseWriter, req *http.Request) { 28 | bytes, err := ioutil.ReadAll(req.Body) 29 | if err != nil { 30 | w.Write([]byte(err.Error())) 31 | } 32 | if string(bytes) == "2" { 33 | close(s.receivedSecondReq) 34 | w.Write([]byte("2")) 35 | } 36 | if string(bytes) == "1" { 37 | <-s.receivedSecondReq 38 | w.Write([]byte("1")) 39 | } 40 | } 41 | 42 | // TODO: test http-connect as well. 43 | func getTestClient(front string, t *testing.T) *http.Client { 44 | ctx := context.Background() 45 | tunnel, err := client.CreateSingleUseGrpcTunnel(ctx, front, grpc.WithInsecure()) 46 | if err != nil { 47 | t.Fatal(err) 48 | } 49 | 50 | return &http.Client{ 51 | Transport: &http.Transport{ 52 | DialContext: tunnel.DialContext, 53 | }, 54 | Timeout: 2 * time.Second, 55 | } 56 | } 57 | 58 | // singleTimeManager makes sure that a backend only serves one request. 59 | type singleTimeManager struct { 60 | mu sync.Mutex 61 | backends map[string]agent.AgentService_ConnectServer 62 | used map[string]struct{} 63 | } 64 | 65 | func (s *singleTimeManager) AddBackend(agentID string, _ pkgagent.IdentifierType, conn agent.AgentService_ConnectServer) server.Backend { 66 | s.mu.Lock() 67 | defer s.mu.Unlock() 68 | s.backends[agentID] = conn 69 | return conn 70 | } 71 | 72 | func (s *singleTimeManager) RemoveBackend(agentID string, _ pkgagent.IdentifierType, conn agent.AgentService_ConnectServer) { 73 | s.mu.Lock() 74 | defer s.mu.Unlock() 75 | v, ok := s.backends[agentID] 76 | if !ok { 77 | panic(fmt.Errorf("no backends found for %s", agentID)) 78 | } 79 | if v != conn { 80 | panic(fmt.Errorf("recorded connection %v does not match conn %v", v, conn)) 81 | } 82 | delete(s.backends, agentID) 83 | } 84 | 85 | func (s *singleTimeManager) Backend(_ context.Context) (server.Backend, error) { 86 | s.mu.Lock() 87 | defer s.mu.Unlock() 88 | for k, v := range s.backends { 89 | if _, ok := s.used[k]; !ok { 90 | s.used[k] = struct{}{} 91 | return v, nil 92 | } 93 | } 94 | return nil, fmt.Errorf("cannot find backend to a new agent") 95 | } 96 | 97 | func (s *singleTimeManager) GetBackend(agentID string) server.Backend { 98 | return nil 99 | } 100 | 101 | func (s *singleTimeManager) NumBackends() int { 102 | return 0 103 | } 104 | 105 | func newSingleTimeGetter(m *server.DefaultBackendManager) *singleTimeManager { 106 | return &singleTimeManager{ 107 | used: make(map[string]struct{}), 108 | backends: make(map[string]agent.AgentService_ConnectServer), 109 | } 110 | } 111 | 112 | var _ server.BackendManager = &singleTimeManager{} 113 | 114 | func (s *singleTimeManager) Ready() (bool, string) { 115 | return true, "" 116 | } 117 | 118 | func TestConcurrentClientRequest(t *testing.T) { 119 | s := httptest.NewServer(&simpleServer{receivedSecondReq: make(chan struct{})}) 120 | defer s.Close() 121 | 122 | proxy, ps, cleanup, err := runGRPCProxyServerWithServerCount(1) 123 | if err != nil { 124 | t.Fatal(err) 125 | } 126 | defer cleanup() 127 | ps.BackendManagers = []server.BackendManager{newSingleTimeGetter(server.NewDefaultBackendManager())} 128 | 129 | stopCh := make(chan struct{}) 130 | defer close(stopCh) 131 | // Run two agents 132 | runAgent(proxy.agent, stopCh) 133 | runAgent(proxy.agent, stopCh) 134 | 135 | client1 := getTestClient(proxy.front, t) 136 | client2 := getTestClient(proxy.front, t) 137 | var wg sync.WaitGroup 138 | wg.Add(2) 139 | go func() { 140 | defer wg.Done() 141 | r, err := client1.Post(s.URL, "text/plain", bytes.NewBufferString("1")) 142 | if err != nil { 143 | t.Error(err) 144 | return 145 | } 146 | data, err := ioutil.ReadAll(r.Body) 147 | if err != nil { 148 | t.Error(err) 149 | } 150 | r.Body.Close() 151 | 152 | if string(data) != "1" { 153 | t.Errorf("expect %v; got %v", "1", string(data)) 154 | } 155 | }() 156 | // give client1 some time to establish the connection. 157 | time.Sleep(1 * time.Second) 158 | go func() { 159 | defer wg.Done() 160 | r, err := client2.Post(s.URL, "text/plain", bytes.NewBufferString("2")) 161 | if err != nil { 162 | t.Error(err) 163 | return 164 | } 165 | data, err := ioutil.ReadAll(r.Body) 166 | if err != nil { 167 | t.Error(err) 168 | } 169 | r.Body.Close() 170 | 171 | if string(data) != "2" { 172 | t.Errorf("expect %v; got %v", "2", string(data)) 173 | } 174 | }() 175 | wg.Wait() 176 | } 177 | -------------------------------------------------------------------------------- /pkg/server/tunnel.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package server 18 | 19 | import ( 20 | "fmt" 21 | "io" 22 | "math/rand" 23 | "net/http" 24 | "sync" 25 | "time" 26 | 27 | "k8s.io/klog/v2" 28 | "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client" 29 | "sigs.k8s.io/apiserver-network-proxy/pkg/server/metrics" 30 | ) 31 | 32 | // Tunnel implements Proxy based on HTTP Connect, which tunnels the traffic to 33 | // the agent registered in ProxyServer. 34 | type Tunnel struct { 35 | Server *ProxyServer 36 | } 37 | 38 | func (t *Tunnel) ServeHTTP(w http.ResponseWriter, r *http.Request) { 39 | metrics.Metrics.HTTPConnectionInc() 40 | defer metrics.Metrics.HTTPConnectionDec() 41 | 42 | klog.V(2).InfoS("Received request for host", "method", r.Method, "host", r.Host, "userAgent", r.UserAgent()) 43 | if r.TLS != nil { 44 | klog.V(2).InfoS("TLS", "commonName", r.TLS.PeerCertificates[0].Subject.CommonName) 45 | } 46 | if r.Method != http.MethodConnect { 47 | http.Error(w, "this proxy only supports CONNECT passthrough", http.StatusMethodNotAllowed) 48 | return 49 | } 50 | 51 | hijacker, ok := w.(http.Hijacker) 52 | if !ok { 53 | http.Error(w, "hijacking not supported", http.StatusInternalServerError) 54 | return 55 | } 56 | w.WriteHeader(http.StatusOK) 57 | 58 | conn, bufrw, err := hijacker.Hijack() 59 | if err != nil { 60 | http.Error(w, err.Error(), http.StatusInternalServerError) 61 | return 62 | } 63 | var closeOnce sync.Once 64 | defer closeOnce.Do(func() { conn.Close() }) 65 | 66 | random := rand.Int63() /* #nosec G404 */ 67 | dialRequest := &client.Packet{ 68 | Type: client.PacketType_DIAL_REQ, 69 | Payload: &client.Packet_DialRequest{ 70 | DialRequest: &client.DialRequest{ 71 | Protocol: "tcp", 72 | Address: r.Host, 73 | Random: random, 74 | }, 75 | }, 76 | } 77 | 78 | klog.V(4).Infof("Set pending(rand=%d) to %v", random, w) 79 | backend, err := t.Server.getBackend(r.Host) 80 | if err != nil { 81 | http.Error(w, fmt.Sprintf("currently no tunnels available: %v", err), http.StatusInternalServerError) 82 | return 83 | } 84 | closed := make(chan struct{}) 85 | connected := make(chan struct{}) 86 | connection := &ProxyClientConnection{ 87 | Mode: "http-connect", 88 | HTTP: io.ReadWriter(conn), // pass as ReadWriter so the caller must close with CloseHTTP 89 | CloseHTTP: func() error { 90 | closeOnce.Do(func() { conn.Close() }) 91 | close(closed) 92 | return nil 93 | }, 94 | connected: connected, 95 | start: time.Now(), 96 | backend: backend, 97 | } 98 | t.Server.PendingDial.Add(random, connection) 99 | if err := backend.Send(dialRequest); err != nil { 100 | klog.ErrorS(err, "failed to tunnel dial request") 101 | return 102 | } 103 | ctxt := backend.Context() 104 | if ctxt.Err() != nil { 105 | klog.ErrorS(err, "context reports failure") 106 | } 107 | 108 | select { 109 | case <-ctxt.Done(): 110 | klog.V(5).Infoln("context reports done") 111 | default: 112 | } 113 | 114 | select { 115 | case <-connection.connected: // Waiting for response before we begin full communication. 116 | case <-closed: // Connection was closed before being established 117 | } 118 | 119 | defer func() { 120 | packet := &client.Packet{ 121 | Type: client.PacketType_CLOSE_REQ, 122 | Payload: &client.Packet_CloseRequest{ 123 | CloseRequest: &client.CloseRequest{ 124 | ConnectID: connection.connectID, 125 | }, 126 | }, 127 | } 128 | 129 | if err = backend.Send(packet); err != nil { 130 | klog.V(2).InfoS("failed to send close request packet", "host", r.Host, "agentID", connection.agentID, "connectionID", connection.connectID) 131 | } 132 | conn.Close() 133 | }() 134 | 135 | klog.V(3).InfoS("Starting proxy to host", "host", r.Host) 136 | pkt := make([]byte, 1<<15) // Match GRPC Window size 137 | 138 | connID := connection.connectID 139 | agentID := connection.agentID 140 | var acc int 141 | 142 | for { 143 | n, err := bufrw.Read(pkt[:]) 144 | acc += n 145 | if err == io.EOF { 146 | klog.V(1).InfoS("EOF from host", "host", r.Host) 147 | break 148 | } 149 | if err != nil { 150 | klog.ErrorS(err, "Received failure on connection") 151 | break 152 | } 153 | 154 | packet := &client.Packet{ 155 | Type: client.PacketType_DATA, 156 | Payload: &client.Packet_Data{ 157 | Data: &client.Data{ 158 | ConnectID: connID, 159 | Data: pkt[:n], 160 | }, 161 | }, 162 | } 163 | err = backend.Send(packet) 164 | if err != nil { 165 | klog.ErrorS(err, "error sending packet") 166 | break 167 | } 168 | klog.V(5).InfoS("Forwarding data on tunnel to agent", 169 | "bytes", n, 170 | "totalBytes", acc, 171 | "agentID", connection.agentID, 172 | "connectionID", connection.connectID) 173 | } 174 | 175 | klog.V(5).InfoS("Stopping transfer to host", "host", r.Host, "agentID", agentID, "connectionID", connID) 176 | } 177 | -------------------------------------------------------------------------------- /tests/ha_proxy_server_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "io/ioutil" 7 | "log" 8 | "math/rand" 9 | "net" 10 | "net/http" 11 | "net/http/httptest" 12 | "sync" 13 | "testing" 14 | "time" 15 | 16 | "google.golang.org/grpc" 17 | "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client" 18 | ) 19 | 20 | type tcpLB struct { 21 | t *testing.T 22 | mu sync.RWMutex 23 | backends []string 24 | } 25 | 26 | func copy(wc io.WriteCloser, r io.Reader) { 27 | defer wc.Close() 28 | io.Copy(wc, r) 29 | } 30 | 31 | func (lb *tcpLB) handleConnection(in net.Conn, backend string) { 32 | out, err := net.Dial("tcp", backend) 33 | if err != nil { 34 | lb.t.Log(err) 35 | return 36 | } 37 | go copy(out, in) 38 | go copy(in, out) 39 | } 40 | 41 | func (lb *tcpLB) serve(stopCh chan struct{}) { 42 | ln, err := net.Listen("tcp", "127.0.0.1:8000") 43 | if err != nil { 44 | log.Fatalf("failed to bind: %s", err) 45 | } 46 | for { 47 | select { 48 | case <-stopCh: 49 | return 50 | default: 51 | } 52 | conn, err := ln.Accept() 53 | if err != nil { 54 | log.Printf("failed to accept: %s", err) 55 | continue 56 | } 57 | // go lb.handleConnection(conn, lb.randomBackend()) 58 | back := lb.randomBackend() 59 | go lb.handleConnection(conn, back) 60 | } 61 | } 62 | 63 | func (lb *tcpLB) addBackend(backend string) { 64 | lb.mu.Lock() 65 | defer lb.mu.Unlock() 66 | lb.backends = append(lb.backends, backend) 67 | } 68 | 69 | func (lb *tcpLB) removeBackend(backend string) { 70 | lb.mu.Lock() 71 | defer lb.mu.Unlock() 72 | for i := range lb.backends { 73 | if lb.backends[i] == backend { 74 | lb.backends = append(lb.backends[:i], lb.backends[i+1:]...) 75 | return 76 | } 77 | } 78 | } 79 | 80 | func (lb *tcpLB) randomBackend() string { 81 | lb.mu.RLock() 82 | defer lb.mu.RUnlock() 83 | i := rand.Intn(len(lb.backends)) /* #nosec G404 */ 84 | return lb.backends[i] 85 | } 86 | 87 | const haServerCount = 3 88 | 89 | func setupHAProxyServer(t *testing.T) ([]proxy, []func()) { 90 | proxy1, _, cleanup1, err := runGRPCProxyServerWithServerCount(haServerCount) 91 | if err != nil { 92 | t.Fatal(err) 93 | } 94 | 95 | proxy2, _, cleanup2, err := runGRPCProxyServerWithServerCount(haServerCount) 96 | if err != nil { 97 | t.Fatal(err) 98 | } 99 | 100 | proxy3, _, cleanup3, err := runGRPCProxyServerWithServerCount(haServerCount) 101 | if err != nil { 102 | t.Fatal(err) 103 | } 104 | return []proxy{proxy1, proxy2, proxy3}, []func(){cleanup1, cleanup2, cleanup3} 105 | } 106 | 107 | func TestBasicHAProxyServer_GRPC(t *testing.T) { 108 | server := httptest.NewServer(newEchoServer("hello")) 109 | defer server.Close() 110 | 111 | stopCh := make(chan struct{}) 112 | defer close(stopCh) 113 | 114 | proxy, cleanups := setupHAProxyServer(t) 115 | 116 | lb := tcpLB{ 117 | backends: []string{ 118 | proxy[0].agent, 119 | proxy[1].agent, 120 | proxy[2].agent, 121 | }, 122 | t: t, 123 | } 124 | go lb.serve(stopCh) 125 | 126 | clientset := runAgent(":8000", stopCh) 127 | 128 | var ready bool 129 | var hc, cc int 130 | for i := 0; i < 3; i++ { 131 | time.Sleep(1 * time.Second) 132 | hc, cc = clientset.HealthyClientsCount(), clientset.ClientsCount() 133 | t.Logf("got %d clients, %d of them are healthy", hc, cc) 134 | if hc == 3 && cc == 3 { 135 | ready = true 136 | break 137 | } 138 | } 139 | if !ready { 140 | t.Fatalf("expected to get 3 clients, got %d clients, %d healthy clients", hc, cc) 141 | } 142 | 143 | // run test client 144 | testProxyServer(t, proxy[0].front, server.URL) 145 | testProxyServer(t, proxy[1].front, server.URL) 146 | testProxyServer(t, proxy[2].front, server.URL) 147 | 148 | t.Logf("basic HA proxy server test passed") 149 | 150 | // interrupt the HA server 151 | lb.removeBackend(proxy[0].agent) 152 | cleanups[0]() 153 | 154 | proxy4, _, cleanup4, err := runGRPCProxyServerWithServerCount(haServerCount) 155 | if err != nil { 156 | t.Fatal(err) 157 | } 158 | lb.addBackend(proxy4.agent) 159 | defer func() { 160 | cleanups[1]() 161 | cleanups[2]() 162 | cleanup4() 163 | }() 164 | // give the agent some time to detect the disconnection 165 | time.Sleep(1 * time.Second) 166 | 167 | ready = false 168 | for i := 0; i < 3; i++ { 169 | time.Sleep(1 * time.Second) 170 | hc, cc = clientset.HealthyClientsCount(), clientset.ClientsCount() 171 | t.Logf("got %d clients, %d of them are healthy", hc, cc) 172 | if hc == 3 && (cc == 3 || cc == 4) { 173 | ready = true 174 | break 175 | } 176 | } 177 | if !ready { 178 | t.Fatalf("expected to get 3 clients, got %d clients, %d healthy clients", hc, cc) 179 | } 180 | // run test client 181 | testProxyServer(t, proxy[1].front, server.URL) 182 | testProxyServer(t, proxy[2].front, server.URL) 183 | testProxyServer(t, proxy4.front, server.URL) 184 | } 185 | 186 | func testProxyServer(t *testing.T, front string, target string) { 187 | ctx := context.Background() 188 | tunnel, err := client.CreateSingleUseGrpcTunnel(ctx, front, grpc.WithInsecure()) 189 | if err != nil { 190 | t.Fatal(err) 191 | } 192 | 193 | c := &http.Client{ 194 | Transport: &http.Transport{ 195 | DialContext: tunnel.DialContext, 196 | }, 197 | Timeout: 1 * time.Second, 198 | } 199 | 200 | r, err := c.Get(target) 201 | if err != nil { 202 | t.Fatal(err) 203 | } 204 | 205 | data, err := ioutil.ReadAll(r.Body) 206 | if err != nil { 207 | t.Error(err) 208 | } 209 | 210 | if string(data) != "hello" { 211 | t.Errorf("expect %v; got %v", "hello", string(data)) 212 | } 213 | } 214 | -------------------------------------------------------------------------------- /pkg/server/metrics/metrics.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package metrics 18 | 19 | import ( 20 | "time" 21 | 22 | "github.com/prometheus/client_golang/prometheus" 23 | ) 24 | 25 | const ( 26 | namespace = "konnectivity_network_proxy" 27 | subsystem = "server" 28 | 29 | // Proxy is the ProxyService method used to handle incoming streams. 30 | Proxy = "Proxy" 31 | 32 | // Connect is the AgentService method used to establish next hop. 33 | Connect = "Connect" 34 | ) 35 | 36 | var ( 37 | // Use buckets ranging from 10 ns to 12.5 seconds. 38 | latencyBuckets = []float64{0.000001, 0.00001, 0.0001, 0.005, 0.025, 0.1, 0.5, 2.5, 12.5} 39 | 40 | // Metrics provides access to all dial metrics. 41 | Metrics = newServerMetrics() 42 | ) 43 | 44 | // ServerMetrics includes all the metrics of the proxy server. 45 | type ServerMetrics struct { 46 | latencies *prometheus.HistogramVec 47 | frontendLatencies *prometheus.HistogramVec 48 | connections *prometheus.GaugeVec 49 | httpConnections prometheus.Gauge 50 | backend *prometheus.GaugeVec 51 | pendingDials *prometheus.GaugeVec 52 | } 53 | 54 | // newServerMetrics create a new ServerMetrics, configured with default metric names. 55 | func newServerMetrics() *ServerMetrics { 56 | latencies := prometheus.NewHistogramVec( 57 | prometheus.HistogramOpts{ 58 | Namespace: namespace, 59 | Subsystem: subsystem, 60 | Name: "dial_duration_seconds", 61 | Help: "Latency of dial to the remote endpoint in seconds", 62 | Buckets: latencyBuckets, 63 | }, 64 | []string{}, 65 | ) 66 | frontendLatencies := prometheus.NewHistogramVec( 67 | prometheus.HistogramOpts{ 68 | Namespace: namespace, 69 | Subsystem: subsystem, 70 | Name: "frontend_write_duration_seconds", 71 | Help: "Latency of write to the frontend in seconds", 72 | Buckets: latencyBuckets, 73 | }, 74 | []string{}, 75 | ) 76 | connections := prometheus.NewGaugeVec( 77 | prometheus.GaugeOpts{ 78 | Namespace: namespace, 79 | Subsystem: subsystem, 80 | Name: "grpc_connections", 81 | Help: "Number of current grpc connections, partitioned by service method.", 82 | }, 83 | []string{ 84 | "service_method", 85 | }, 86 | ) 87 | httpConnections := prometheus.NewGauge( 88 | prometheus.GaugeOpts{ 89 | Namespace: namespace, 90 | Subsystem: subsystem, 91 | Name: "http_connections", 92 | Help: "Number of current HTTP CONNECT connections", 93 | }, 94 | ) 95 | backend := prometheus.NewGaugeVec( 96 | prometheus.GaugeOpts{ 97 | Namespace: namespace, 98 | Subsystem: subsystem, 99 | Name: "ready_backend_connections", 100 | Help: "Number of konnectivity agent connected to the proxy server", 101 | }, 102 | []string{}, 103 | ) 104 | pendingDials := prometheus.NewGaugeVec( 105 | prometheus.GaugeOpts{ 106 | Namespace: namespace, 107 | Subsystem: subsystem, 108 | Name: "pending_backend_dials", 109 | Help: "Current number of pending backend dial requests", 110 | }, 111 | []string{}, 112 | ) 113 | 114 | prometheus.MustRegister(latencies) 115 | prometheus.MustRegister(frontendLatencies) 116 | prometheus.MustRegister(connections) 117 | prometheus.MustRegister(httpConnections) 118 | prometheus.MustRegister(backend) 119 | prometheus.MustRegister(pendingDials) 120 | return &ServerMetrics{ 121 | latencies: latencies, 122 | frontendLatencies: frontendLatencies, 123 | connections: connections, 124 | httpConnections: httpConnections, 125 | backend: backend, 126 | pendingDials: pendingDials, 127 | } 128 | } 129 | 130 | // Reset resets the metrics. 131 | func (a *ServerMetrics) Reset() { 132 | a.latencies.Reset() 133 | a.frontendLatencies.Reset() 134 | } 135 | 136 | // ObserveDialLatency records the latency of dial to the remote endpoint. 137 | func (a *ServerMetrics) ObserveDialLatency(elapsed time.Duration) { 138 | a.latencies.WithLabelValues().Observe(elapsed.Seconds()) 139 | } 140 | 141 | // ObserveFrontendWriteLatency records the latency of dial to the remote endpoint. 142 | func (a *ServerMetrics) ObserveFrontendWriteLatency(elapsed time.Duration) { 143 | a.frontendLatencies.WithLabelValues().Observe(elapsed.Seconds()) 144 | } 145 | 146 | // ConnectionInc increments a new grpc client connection. 147 | func (a *ServerMetrics) ConnectionInc(serviceMethod string) { 148 | a.connections.With(prometheus.Labels{"service_method": serviceMethod}).Inc() 149 | } 150 | 151 | // ConnectionDec decrements a finished grpc client connection. 152 | func (a *ServerMetrics) ConnectionDec(serviceMethod string) { 153 | a.connections.With(prometheus.Labels{"service_method": serviceMethod}).Dec() 154 | } 155 | 156 | // HTTPConnectionDec increments a new HTTP CONNECTION connection. 157 | func (a *ServerMetrics) HTTPConnectionInc() { a.httpConnections.Inc() } 158 | 159 | // HTTPConnectionDec decrements a finished HTTP CONNECTION connection. 160 | func (a *ServerMetrics) HTTPConnectionDec() { a.httpConnections.Dec() } 161 | 162 | // SetBackendCount sets the number of backend connection. 163 | func (a *ServerMetrics) SetBackendCount(count int) { 164 | a.backend.WithLabelValues().Set(float64(count)) 165 | } 166 | 167 | // SetPendingDialCount sets the number of pending dials. 168 | func (a *ServerMetrics) SetPendingDialCount(count int) { 169 | a.pendingDials.WithLabelValues().Set(float64(count)) 170 | } 171 | -------------------------------------------------------------------------------- /proto/agent/agent.pb.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | // Code generated by protoc-gen-go. DO NOT EDIT. 17 | // source: proto/agent/agent.proto 18 | 19 | package agent 20 | 21 | import ( 22 | context "context" 23 | fmt "fmt" 24 | proto "github.com/golang/protobuf/proto" 25 | grpc "google.golang.org/grpc" 26 | codes "google.golang.org/grpc/codes" 27 | status "google.golang.org/grpc/status" 28 | math "math" 29 | client "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client" 30 | ) 31 | 32 | // Reference imports to suppress errors if they are not otherwise used. 33 | var _ = proto.Marshal 34 | var _ = fmt.Errorf 35 | var _ = math.Inf 36 | 37 | // This is a compile-time assertion to ensure that this generated file 38 | // is compatible with the proto package it is being compiled against. 39 | // A compilation error at this line likely means your copy of the 40 | // proto package needs to be updated. 41 | const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package 42 | 43 | func init() { proto.RegisterFile("proto/agent/agent.proto", fileDescriptor_656b6c96a18ce683) } 44 | 45 | var fileDescriptor_656b6c96a18ce683 = []byte{ 46 | // 155 bytes of a gzipped FileDescriptorProto 47 | 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x12, 0x2f, 0x28, 0xca, 0x2f, 48 | 0xc9, 0xd7, 0x4f, 0x4c, 0x4f, 0xcd, 0x2b, 0x81, 0x90, 0x7a, 0x60, 0x11, 0x29, 0xdd, 0xec, 0xfc, 49 | 0xbc, 0xbc, 0xd4, 0xe4, 0x92, 0xcc, 0xb2, 0xcc, 0x92, 0x4a, 0xdd, 0xe4, 0x9c, 0x4c, 0x90, 0x02, 50 | 0x88, 0x62, 0x28, 0x07, 0x42, 0x41, 0x94, 0x1b, 0x19, 0x72, 0xf1, 0x38, 0x82, 0x74, 0x07, 0xa7, 51 | 0x16, 0x95, 0x65, 0x26, 0xa7, 0x0a, 0x29, 0x72, 0xb1, 0x3b, 0x43, 0x0c, 0x10, 0x62, 0xd7, 0x0b, 52 | 0x48, 0x4c, 0xce, 0x4e, 0x2d, 0x91, 0x82, 0x31, 0x94, 0x18, 0x34, 0x18, 0x0d, 0x18, 0x9d, 0x0c, 53 | 0xa3, 0xf4, 0x8b, 0x33, 0xd3, 0x8b, 0xf5, 0xb2, 0x2d, 0x8a, 0xf5, 0x32, 0xf3, 0xf5, 0x13, 0x0b, 54 | 0x32, 0x8b, 0x53, 0x8b, 0xca, 0x52, 0x8b, 0x74, 0xf3, 0x52, 0x4b, 0xca, 0xf3, 0x8b, 0xb2, 0x75, 55 | 0x0b, 0x8a, 0xf2, 0x2b, 0x2a, 0xf5, 0x91, 0x1c, 0x98, 0xc4, 0x06, 0xe6, 0x18, 0x03, 0x02, 0x00, 56 | 0x00, 0xff, 0xff, 0x85, 0x25, 0x82, 0x73, 0xb6, 0x00, 0x00, 0x00, 57 | } 58 | 59 | // Reference imports to suppress errors if they are not otherwise used. 60 | var _ context.Context 61 | var _ grpc.ClientConn 62 | 63 | // This is a compile-time assertion to ensure that this generated file 64 | // is compatible with the grpc package it is being compiled against. 65 | const _ = grpc.SupportPackageIsVersion4 66 | 67 | // AgentServiceClient is the client API for AgentService service. 68 | // 69 | // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. 70 | type AgentServiceClient interface { 71 | // Agent Identifier? 72 | Connect(ctx context.Context, opts ...grpc.CallOption) (AgentService_ConnectClient, error) 73 | } 74 | 75 | type agentServiceClient struct { 76 | cc *grpc.ClientConn 77 | } 78 | 79 | func NewAgentServiceClient(cc *grpc.ClientConn) AgentServiceClient { 80 | return &agentServiceClient{cc} 81 | } 82 | 83 | func (c *agentServiceClient) Connect(ctx context.Context, opts ...grpc.CallOption) (AgentService_ConnectClient, error) { 84 | stream, err := c.cc.NewStream(ctx, &_AgentService_serviceDesc.Streams[0], "/AgentService/Connect", opts...) 85 | if err != nil { 86 | return nil, err 87 | } 88 | x := &agentServiceConnectClient{stream} 89 | return x, nil 90 | } 91 | 92 | type AgentService_ConnectClient interface { 93 | Send(*client.Packet) error 94 | Recv() (*client.Packet, error) 95 | grpc.ClientStream 96 | } 97 | 98 | type agentServiceConnectClient struct { 99 | grpc.ClientStream 100 | } 101 | 102 | func (x *agentServiceConnectClient) Send(m *client.Packet) error { 103 | return x.ClientStream.SendMsg(m) 104 | } 105 | 106 | func (x *agentServiceConnectClient) Recv() (*client.Packet, error) { 107 | m := new(client.Packet) 108 | if err := x.ClientStream.RecvMsg(m); err != nil { 109 | return nil, err 110 | } 111 | return m, nil 112 | } 113 | 114 | // AgentServiceServer is the server API for AgentService service. 115 | type AgentServiceServer interface { 116 | // Agent Identifier? 117 | Connect(AgentService_ConnectServer) error 118 | } 119 | 120 | // UnimplementedAgentServiceServer can be embedded to have forward compatible implementations. 121 | type UnimplementedAgentServiceServer struct { 122 | } 123 | 124 | func (*UnimplementedAgentServiceServer) Connect(srv AgentService_ConnectServer) error { 125 | return status.Errorf(codes.Unimplemented, "method Connect not implemented") 126 | } 127 | 128 | func RegisterAgentServiceServer(s *grpc.Server, srv AgentServiceServer) { 129 | s.RegisterService(&_AgentService_serviceDesc, srv) 130 | } 131 | 132 | func _AgentService_Connect_Handler(srv interface{}, stream grpc.ServerStream) error { 133 | return srv.(AgentServiceServer).Connect(&agentServiceConnectServer{stream}) 134 | } 135 | 136 | type AgentService_ConnectServer interface { 137 | Send(*client.Packet) error 138 | Recv() (*client.Packet, error) 139 | grpc.ServerStream 140 | } 141 | 142 | type agentServiceConnectServer struct { 143 | grpc.ServerStream 144 | } 145 | 146 | func (x *agentServiceConnectServer) Send(m *client.Packet) error { 147 | return x.ServerStream.SendMsg(m) 148 | } 149 | 150 | func (x *agentServiceConnectServer) Recv() (*client.Packet, error) { 151 | m := new(client.Packet) 152 | if err := x.ServerStream.RecvMsg(m); err != nil { 153 | return nil, err 154 | } 155 | return m, nil 156 | } 157 | 158 | var _AgentService_serviceDesc = grpc.ServiceDesc{ 159 | ServiceName: "AgentService", 160 | HandlerType: (*AgentServiceServer)(nil), 161 | Methods: []grpc.MethodDesc{}, 162 | Streams: []grpc.StreamDesc{ 163 | { 164 | StreamName: "Connect", 165 | Handler: _AgentService_Connect_Handler, 166 | ServerStreams: true, 167 | ClientStreams: true, 168 | }, 169 | }, 170 | Metadata: "proto/agent/agent.proto", 171 | } 172 | -------------------------------------------------------------------------------- /cmd/test-server/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "context" 21 | "flag" 22 | "fmt" 23 | "net/http" 24 | "os" 25 | "os/signal" 26 | "strconv" 27 | "syscall" 28 | "time" 29 | 30 | "github.com/spf13/cobra" 31 | "github.com/spf13/pflag" 32 | "k8s.io/klog/v2" 33 | "sigs.k8s.io/apiserver-network-proxy/pkg/util" 34 | ) 35 | 36 | func main() { 37 | testServer := &TestServer{} 38 | o := newTestServerRunOptions() 39 | command := newTestServerCommand(testServer, o) 40 | flags := command.Flags() 41 | flags.AddFlagSet(o.Flags()) 42 | local := flag.NewFlagSet(os.Args[0], flag.ExitOnError) 43 | klog.InitFlags(local) 44 | local.VisitAll(func(fl *flag.Flag) { 45 | fl.Name = util.Normalize(fl.Name) 46 | flags.AddGoFlag(fl) 47 | }) 48 | if err := command.Execute(); err != nil { 49 | klog.Errorf("error: %v\n", err) 50 | klog.Flush() 51 | os.Exit(1) 52 | } 53 | } 54 | 55 | type TestServerRunOptions struct { 56 | // Port we listen for server connections on. 57 | serverPort uint 58 | } 59 | 60 | func (o *TestServerRunOptions) Flags() *pflag.FlagSet { 61 | flags := pflag.NewFlagSet("proxy-server", pflag.ContinueOnError) 62 | flags.UintVar(&o.serverPort, "server-port", o.serverPort, "Port we listen for server connections on. Set to 0 for UDS.") 63 | return flags 64 | } 65 | 66 | func (o *TestServerRunOptions) Print() { 67 | klog.Warningf("Server port set to %d.\n", o.serverPort) 68 | } 69 | 70 | func (o *TestServerRunOptions) Validate() error { 71 | if o.serverPort > 49151 { 72 | return fmt.Errorf("please do not try to use ephemeral port %d for the server port", o.serverPort) 73 | } 74 | if o.serverPort < 1024 { 75 | return fmt.Errorf("please do not try to use reserved port %d for the server port", o.serverPort) 76 | } 77 | 78 | return nil 79 | } 80 | 81 | func newTestServerRunOptions() *TestServerRunOptions { 82 | o := TestServerRunOptions{ 83 | serverPort: 8000, 84 | } 85 | return &o 86 | } 87 | 88 | func newTestServerCommand(p *TestServer, o *TestServerRunOptions) *cobra.Command { 89 | cmd := &cobra.Command{ 90 | Use: "test http server", 91 | Long: `A test http server, url determines behavior for certain tests.`, 92 | RunE: func(cmd *cobra.Command, args []string) error { 93 | return p.run(o) 94 | }, 95 | } 96 | 97 | return cmd 98 | } 99 | 100 | type TestServer struct { 101 | } 102 | 103 | type StopFunc func() 104 | 105 | func (p *TestServer) run(o *TestServerRunOptions) error { 106 | o.Print() 107 | if err := o.Validate(); err != nil { 108 | return fmt.Errorf("failed to validate server options with %v", err) 109 | } 110 | ctx, cancel := context.WithCancel(context.Background()) 111 | defer cancel() 112 | 113 | klog.Info("Starting test http server for client requests.") 114 | testStop, err := p.runTestServer(ctx, o) 115 | if err != nil { 116 | return fmt.Errorf("failed to run the test server: %v", err) 117 | } 118 | 119 | stopCh := SetupSignalHandler() 120 | <-stopCh 121 | klog.Info("Shutting down server.") 122 | 123 | if testStop != nil { 124 | testStop() 125 | } 126 | 127 | return nil 128 | } 129 | 130 | var shutdownSignals = []os.Signal{os.Interrupt, syscall.SIGTERM} 131 | 132 | func SetupSignalHandler() (stopCh <-chan struct{}) { 133 | stop := make(chan struct{}) 134 | c := make(chan os.Signal, 2) 135 | signal.Notify(c, shutdownSignals...) 136 | go func() { 137 | <-c 138 | close(stop) 139 | <-c 140 | os.Exit(1) // second signal. Exit directly. 141 | }() 142 | 143 | return stop 144 | } 145 | 146 | func returnSuccess(w http.ResponseWriter, req *http.Request) { 147 | fmt.Fprintf(w, "\n\n \n Success\n \n \n

The success test page!

\n \n") 148 | } 149 | 150 | func returnError(w http.ResponseWriter, req *http.Request) { 151 | http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) 152 | } 153 | 154 | func closeNoResponse(w http.ResponseWriter, req *http.Request) { 155 | hj, ok := w.(http.Hijacker) 156 | if !ok { 157 | http.Error(w, "webserver doesn't support hijacking", http.StatusInternalServerError) 158 | return 159 | } 160 | conn, _, err := hj.Hijack() 161 | if err != nil { 162 | http.Error(w, err.Error(), http.StatusInternalServerError) 163 | return 164 | } 165 | err = conn.Close() 166 | if err != nil { 167 | klog.ErrorS(err, "failed to close connection") 168 | } 169 | } 170 | 171 | func sleepReturnSuccess(w http.ResponseWriter, req *http.Request) { 172 | sleepArr := req.URL.Query()["time"] 173 | sleepInt := 10 174 | var err error 175 | if len(sleepArr) == 1 { 176 | sleepStr := sleepArr[0] 177 | sleepInt, err = strconv.Atoi(sleepStr) 178 | if err != nil { 179 | sleepInt = 10 180 | klog.Warningf("Failed to parse sleep time (%s), default to %d, got error %v.", sleepStr, sleepInt, err) 181 | } 182 | } else { 183 | klog.Warningf("Sleep time(%v) was length %d defaulting to sleep %d.", sleepArr, len(sleepArr), sleepInt) 184 | } 185 | sleep := time.Duration(sleepInt) * time.Second 186 | time.Sleep(sleep) 187 | returnSuccess(w, req) 188 | } 189 | 190 | func (p *TestServer) runTestServer(ctx context.Context, o *TestServerRunOptions) (StopFunc, error) { 191 | muxHandler := http.NewServeMux() 192 | muxHandler.HandleFunc("/success", returnSuccess) 193 | muxHandler.HandleFunc("/sleep", sleepReturnSuccess) 194 | muxHandler.HandleFunc("/error", returnError) 195 | muxHandler.HandleFunc("/close", closeNoResponse) 196 | server := &http.Server{ 197 | Addr: fmt.Sprintf("127.0.0.1:%d", o.serverPort), 198 | Handler: muxHandler, 199 | MaxHeaderBytes: 1 << 20, 200 | } 201 | 202 | go func() { 203 | err := server.ListenAndServe() 204 | if err != nil { 205 | klog.Warningf("HTTP test server received %v.\n", err) 206 | } 207 | klog.Warningf("HTTP test server stopped listening\n") 208 | }() 209 | 210 | stop := func() { 211 | err := server.Shutdown(ctx) 212 | if err != nil { 213 | klog.ErrorS(err, "failed to shutdown server") 214 | } 215 | } 216 | return stop, nil 217 | } 218 | -------------------------------------------------------------------------------- /proto/agent/mocks/agent_mock.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | // Code generated by MockGen. DO NOT EDIT. 17 | // Source: sigs.k8s.io/apiserver-network-proxy/proto/agent (interfaces: AgentService_ConnectServer) 18 | 19 | // Package mock_agent is a generated GoMock package. 20 | package mock_agent 21 | 22 | import ( 23 | context "context" 24 | reflect "reflect" 25 | 26 | gomock "github.com/golang/mock/gomock" 27 | metadata "google.golang.org/grpc/metadata" 28 | client "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client" 29 | ) 30 | 31 | // MockAgentService_ConnectServer is a mock of AgentService_ConnectServer interface. 32 | type MockAgentService_ConnectServer struct { 33 | ctrl *gomock.Controller 34 | recorder *MockAgentService_ConnectServerMockRecorder 35 | } 36 | 37 | // MockAgentService_ConnectServerMockRecorder is the mock recorder for MockAgentService_ConnectServer. 38 | type MockAgentService_ConnectServerMockRecorder struct { 39 | mock *MockAgentService_ConnectServer 40 | } 41 | 42 | // NewMockAgentService_ConnectServer creates a new mock instance. 43 | func NewMockAgentService_ConnectServer(ctrl *gomock.Controller) *MockAgentService_ConnectServer { 44 | mock := &MockAgentService_ConnectServer{ctrl: ctrl} 45 | mock.recorder = &MockAgentService_ConnectServerMockRecorder{mock} 46 | return mock 47 | } 48 | 49 | // EXPECT returns an object that allows the caller to indicate expected use. 50 | func (m *MockAgentService_ConnectServer) EXPECT() *MockAgentService_ConnectServerMockRecorder { 51 | return m.recorder 52 | } 53 | 54 | // Context mocks base method. 55 | func (m *MockAgentService_ConnectServer) Context() context.Context { 56 | m.ctrl.T.Helper() 57 | ret := m.ctrl.Call(m, "Context") 58 | ret0, _ := ret[0].(context.Context) 59 | return ret0 60 | } 61 | 62 | // Context indicates an expected call of Context. 63 | func (mr *MockAgentService_ConnectServerMockRecorder) Context() *gomock.Call { 64 | mr.mock.ctrl.T.Helper() 65 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Context", reflect.TypeOf((*MockAgentService_ConnectServer)(nil).Context)) 66 | } 67 | 68 | // Recv mocks base method. 69 | func (m *MockAgentService_ConnectServer) Recv() (*client.Packet, error) { 70 | m.ctrl.T.Helper() 71 | ret := m.ctrl.Call(m, "Recv") 72 | ret0, _ := ret[0].(*client.Packet) 73 | ret1, _ := ret[1].(error) 74 | return ret0, ret1 75 | } 76 | 77 | // Recv indicates an expected call of Recv. 78 | func (mr *MockAgentService_ConnectServerMockRecorder) Recv() *gomock.Call { 79 | mr.mock.ctrl.T.Helper() 80 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Recv", reflect.TypeOf((*MockAgentService_ConnectServer)(nil).Recv)) 81 | } 82 | 83 | // RecvMsg mocks base method. 84 | func (m *MockAgentService_ConnectServer) RecvMsg(arg0 interface{}) error { 85 | m.ctrl.T.Helper() 86 | ret := m.ctrl.Call(m, "RecvMsg", arg0) 87 | ret0, _ := ret[0].(error) 88 | return ret0 89 | } 90 | 91 | // RecvMsg indicates an expected call of RecvMsg. 92 | func (mr *MockAgentService_ConnectServerMockRecorder) RecvMsg(arg0 interface{}) *gomock.Call { 93 | mr.mock.ctrl.T.Helper() 94 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RecvMsg", reflect.TypeOf((*MockAgentService_ConnectServer)(nil).RecvMsg), arg0) 95 | } 96 | 97 | // Send mocks base method. 98 | func (m *MockAgentService_ConnectServer) Send(arg0 *client.Packet) error { 99 | m.ctrl.T.Helper() 100 | ret := m.ctrl.Call(m, "Send", arg0) 101 | ret0, _ := ret[0].(error) 102 | return ret0 103 | } 104 | 105 | // Send indicates an expected call of Send. 106 | func (mr *MockAgentService_ConnectServerMockRecorder) Send(arg0 interface{}) *gomock.Call { 107 | mr.mock.ctrl.T.Helper() 108 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Send", reflect.TypeOf((*MockAgentService_ConnectServer)(nil).Send), arg0) 109 | } 110 | 111 | // SendHeader mocks base method. 112 | func (m *MockAgentService_ConnectServer) SendHeader(arg0 metadata.MD) error { 113 | m.ctrl.T.Helper() 114 | ret := m.ctrl.Call(m, "SendHeader", arg0) 115 | ret0, _ := ret[0].(error) 116 | return ret0 117 | } 118 | 119 | // SendHeader indicates an expected call of SendHeader. 120 | func (mr *MockAgentService_ConnectServerMockRecorder) SendHeader(arg0 interface{}) *gomock.Call { 121 | mr.mock.ctrl.T.Helper() 122 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendHeader", reflect.TypeOf((*MockAgentService_ConnectServer)(nil).SendHeader), arg0) 123 | } 124 | 125 | // SendMsg mocks base method. 126 | func (m *MockAgentService_ConnectServer) SendMsg(arg0 interface{}) error { 127 | m.ctrl.T.Helper() 128 | ret := m.ctrl.Call(m, "SendMsg", arg0) 129 | ret0, _ := ret[0].(error) 130 | return ret0 131 | } 132 | 133 | // SendMsg indicates an expected call of SendMsg. 134 | func (mr *MockAgentService_ConnectServerMockRecorder) SendMsg(arg0 interface{}) *gomock.Call { 135 | mr.mock.ctrl.T.Helper() 136 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMsg", reflect.TypeOf((*MockAgentService_ConnectServer)(nil).SendMsg), arg0) 137 | } 138 | 139 | // SetHeader mocks base method. 140 | func (m *MockAgentService_ConnectServer) SetHeader(arg0 metadata.MD) error { 141 | m.ctrl.T.Helper() 142 | ret := m.ctrl.Call(m, "SetHeader", arg0) 143 | ret0, _ := ret[0].(error) 144 | return ret0 145 | } 146 | 147 | // SetHeader indicates an expected call of SetHeader. 148 | func (mr *MockAgentService_ConnectServerMockRecorder) SetHeader(arg0 interface{}) *gomock.Call { 149 | mr.mock.ctrl.T.Helper() 150 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetHeader", reflect.TypeOf((*MockAgentService_ConnectServer)(nil).SetHeader), arg0) 151 | } 152 | 153 | // SetTrailer mocks base method. 154 | func (m *MockAgentService_ConnectServer) SetTrailer(arg0 metadata.MD) { 155 | m.ctrl.T.Helper() 156 | m.ctrl.Call(m, "SetTrailer", arg0) 157 | } 158 | 159 | // SetTrailer indicates an expected call of SetTrailer. 160 | func (mr *MockAgentService_ConnectServerMockRecorder) SetTrailer(arg0 interface{}) *gomock.Call { 161 | mr.mock.ctrl.T.Helper() 162 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetTrailer", reflect.TypeOf((*MockAgentService_ConnectServer)(nil).SetTrailer), arg0) 163 | } 164 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # apiserver-network-proxy 2 | 3 | Created due to https://github.com/kubernetes/org/issues/715. 4 | 5 | See [the KEP proposal for architecture and details](https://github.com/kubernetes/enhancements/tree/master/keps/sig-api-machinery/1281-network-proxy#proposal). 6 | 7 | ## Community, discussion, contribution, and support 8 | 9 | Learn how to engage with the Kubernetes community on the [community page](http://kubernetes.io/community/). 10 | 11 | You can reach the maintainers of this project at: 12 | 13 | - [Slack channel](https://kubernetes.slack.com/messages/apiserver-network-proxy) 14 | - [Mailing list](https://groups.google.com/forum/#!forum/kubernetes-sig-cloud-provider) 15 | 16 | ### Code of conduct 17 | 18 | Participation in the Kubernetes community is governed by the [Kubernetes Code of Conduct](code-of-conduct.md). 19 | 20 | ## Build 21 | 22 | Please make sure you have the REGISTRY and PROJECT_ID environment variables set. 23 | For local builds these can be set to anything. 24 | For image builds these determine the location of your image. 25 | For GCE the registry should be gcr.io and PROJECT_ID should be the project you 26 | want to use the images in. 27 | 28 | ### Mockgen 29 | 30 | The [```mockgen```](https://github.com/golang/mock) tool must be installed on your system. 31 | 32 | ### Protoc 33 | 34 | Proto definitions are compiled with `protoc`. Please ensure you have protoc installed ([Instructions](https://grpc.io/docs/protoc-installation/)) and the `proto-gen-go` library at the appropriate version. 35 | 36 | Currently we are using proto-gen-go@v1.3.2 37 | 38 | `go get github.com/golang/protobuf/protoc-gen-go@v1.3.2` 39 | 40 | ### Local builds 41 | 42 | ```console 43 | make clean 44 | make certs 45 | make gen 46 | make build 47 | ``` 48 | 49 | ### Build images 50 | 51 | ```console 52 | make docker-build 53 | ``` 54 | 55 | ## Examples 56 | 57 | The current examples run two actual services as well as a sample client on one end and a sample destination for 58 | requests on the other. 59 | - *Proxy service:* The proxy service takes the API server requests and forwards them appropriately. 60 | - *Agent service:* The agent service connects to the proxy and then allows traffic to be forwarded to it. 61 | 62 | ### GRPC Client using mTLS Proxy with dial back Agent 63 | 64 | ``` 65 | Frontend client =HTTP over GRPC=> (:8090) proxy (:8091) <=GRPC= agent =HTTP=> http-test-server(:8000) 66 | | ^ 67 | | Tunnel | 68 | +---------------------------------------------------------------+ 69 | ``` 70 | 71 | - Start Simple test HTTP Server (Sample destination) 72 | ```console 73 | ./bin/http-test-server 74 | ``` 75 | 76 | - Start proxy service 77 | ```console 78 | ./bin/proxy-server --server-ca-cert=certs/frontend/issued/ca.crt --server-cert=certs/frontend/issued/proxy-frontend.crt --server-key=certs/frontend/private/proxy-frontend.key --cluster-ca-cert=certs/agent/issued/ca.crt --cluster-cert=certs/agent/issued/proxy-frontend.crt --cluster-key=certs/agent/private/proxy-frontend.key 79 | ``` 80 | 81 | - Start agent service 82 | ```console 83 | ./bin/proxy-agent --ca-cert=certs/agent/issued/ca.crt --agent-cert=certs/agent/issued/proxy-agent.crt --agent-key=certs/agent/private/proxy-agent.key 84 | ``` 85 | 86 | - Run client (mTLS enabled sample client) 87 | ```console 88 | ./bin/proxy-test-client --ca-cert=certs/frontend/issued/ca.crt --client-cert=certs/frontend/issued/proxy-client.crt --client-key=certs/frontend/private/proxy-client.key 89 | ``` 90 | 91 | ### GRPC+UDS Client using Proxy with dial back Agent 92 | 93 | ``` 94 | Frontend client =HTTP over GRPC+UDS=> (/tmp/uds-proxy) proxy (:8091) <=GRPC= agent =HTTP=> SimpleHTTPServer(:8000) 95 | | ^ 96 | | Tunnel | 97 | +----------------------------------------------------------------------------+ 98 | ``` 99 | 100 | - Start Simple test HTTP Server (Sample destination) 101 | ```console 102 | ./bin/http-test-server 103 | ``` 104 | 105 | - Start proxy service 106 | ```console 107 | ./bin/proxy-server --server-port=0 --uds-name=/tmp/uds-proxy --cluster-ca-cert=certs/agent/issued/ca.crt --cluster-cert=certs/agent/issued/proxy-frontend.crt --cluster-key=certs/agent/private/proxy-frontend.key 108 | ``` 109 | 110 | - Start agent service 111 | ```console 112 | ./bin/proxy-agent --ca-cert=certs/agent/issued/ca.crt --agent-cert=certs/agent/issued/proxy-agent.crt --agent-key=certs/agent/private/proxy-agent.key 113 | ``` 114 | 115 | - Run client (mTLS enabled sample client) 116 | ```console 117 | ./bin/proxy-test-client --proxy-port=0 --proxy-uds=/tmp/uds-proxy --proxy-host="" 118 | ``` 119 | 120 | 121 | ### HTTP-Connect Client using mTLS Proxy with dial back Agent (Either curl OR test client) 122 | 123 | ``` 124 | Frontend client =HTTP-CONNECT=> (:8090) proxy (:8091) <=GRPC= agent =HTTP=> SimpleHTTPServer(:8000) 125 | | ^ 126 | | Tunnel | 127 | +-------------------------------------------------------------+ 128 | ``` 129 | 130 | - Start SimpleHTTPServer (Sample destination) 131 | ```console 132 | ./bin/http-test-server 133 | ``` 134 | 135 | - Start proxy service 136 | ```console 137 | ./bin/proxy-server --mode=http-connect --server-ca-cert=certs/frontend/issued/ca.crt --server-cert=certs/frontend/issued/proxy-frontend.crt --server-key=certs/frontend/private/proxy-frontend.key --cluster-ca-cert=certs/agent/issued/ca.crt --cluster-cert=certs/agent/issued/proxy-frontend.crt --cluster-key=certs/agent/private/proxy-frontend.key 138 | ``` 139 | 140 | - Start agent service 141 | ```console 142 | ./bin/proxy-agent --ca-cert=certs/agent/issued/ca.crt --agent-cert=certs/agent/issued/proxy-agent.crt --agent-key=certs/agent/private/proxy-agent.key 143 | ``` 144 | 145 | - Run client (mTLS & http-connect enabled sample client) 146 | ```console 147 | ./bin/proxy-test-client --mode=http-connect --proxy-host=127.0.0.1 --ca-cert=certs/frontend/issued/ca.crt --client-cert=certs/frontend/issued/proxy-client.crt --client-key=certs/frontend/private/proxy-client.key 148 | ``` 149 | 150 | - Run curl client (curl using a mTLS http-connect proxy) 151 | ```console 152 | curl -v -p --proxy-key certs/frontend/private/proxy-client.key --proxy-cert certs/frontend/issued/proxy-client.crt --proxy-cacert certs/frontend/issued/ca.crt --proxy-cert-type PEM -x https://127.0.0.1:8090 http://localhost:8000/success 153 | ``` 154 | 155 | ### Running on kubernetes 156 | See following [README.md](examples/kubernetes/README.md) 157 | 158 | ### Clients 159 | 160 | `apiserver-network-proxy` components are intended to run as standalone binaries and should not be imported as a library. Clients communicating with the network proxy can import the `konnectivity-client` module. 161 | -------------------------------------------------------------------------------- /konnectivity-client/pkg/client/client.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package client 18 | 19 | import ( 20 | "context" 21 | "errors" 22 | "fmt" 23 | "io" 24 | "math/rand" 25 | "net" 26 | "sync" 27 | "time" 28 | 29 | "google.golang.org/grpc" 30 | "k8s.io/klog/v2" 31 | "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client" 32 | ) 33 | 34 | // Tunnel provides ability to dial a connection through a tunnel. 35 | type Tunnel interface { 36 | // Dial connects to the address on the named network, similar to 37 | // what net.Dial does. The only supported protocol is tcp. 38 | DialContext(ctx context.Context, protocol, address string) (net.Conn, error) 39 | } 40 | 41 | type dialResult struct { 42 | err string 43 | connid int64 44 | } 45 | 46 | // grpcTunnel implements Tunnel 47 | type grpcTunnel struct { 48 | stream client.ProxyService_ProxyClient 49 | pendingDial map[int64]chan<- dialResult 50 | conns map[int64]*conn 51 | pendingDialLock sync.RWMutex 52 | connsLock sync.RWMutex 53 | 54 | // The tunnel will be closed if the caller fails to read via conn.Read() 55 | // more than readTimeoutSeconds after a packet has been received. 56 | readTimeoutSeconds int 57 | } 58 | 59 | type clientConn interface { 60 | Close() error 61 | } 62 | 63 | var _ clientConn = &grpc.ClientConn{} 64 | 65 | // CreateSingleUseGrpcTunnel creates a Tunnel to dial to a remote server through a 66 | // gRPC based proxy service. 67 | // Currently, a single tunnel supports a single connection, and the tunnel is closed when the connection is terminated 68 | // The Dial() method of the returned tunnel should only be called once 69 | func CreateSingleUseGrpcTunnel(ctx context.Context, address string, opts ...grpc.DialOption) (Tunnel, error) { 70 | c, err := grpc.DialContext(ctx, address, opts...) 71 | if err != nil { 72 | return nil, err 73 | } 74 | 75 | grpcClient := client.NewProxyServiceClient(c) 76 | 77 | stream, err := grpcClient.Proxy(ctx) 78 | if err != nil { 79 | return nil, err 80 | } 81 | 82 | tunnel := &grpcTunnel{ 83 | stream: stream, 84 | pendingDial: make(map[int64]chan<- dialResult), 85 | conns: make(map[int64]*conn), 86 | readTimeoutSeconds: 10, 87 | } 88 | 89 | go tunnel.serve(c) 90 | 91 | return tunnel, nil 92 | } 93 | 94 | func (t *grpcTunnel) serve(c clientConn) { 95 | defer c.Close() 96 | 97 | for { 98 | pkt, err := t.stream.Recv() 99 | if err == io.EOF { 100 | return 101 | } 102 | if err != nil || pkt == nil { 103 | klog.ErrorS(err, "stream read failure") 104 | return 105 | } 106 | 107 | klog.V(5).InfoS("[tracing] recv packet", "type", pkt.Type) 108 | 109 | switch pkt.Type { 110 | case client.PacketType_DIAL_RSP: 111 | resp := pkt.GetDialResponse() 112 | t.pendingDialLock.RLock() 113 | ch, ok := t.pendingDial[resp.Random] 114 | t.pendingDialLock.RUnlock() 115 | 116 | if !ok { 117 | klog.V(1).InfoS("DialResp not recognized; dropped", "connectionID", resp.ConnectID, "dialID", resp.Random) 118 | return 119 | } else { 120 | result := dialResult{ 121 | err: resp.Error, 122 | connid: resp.ConnectID, 123 | } 124 | select { 125 | case ch <- result: 126 | default: 127 | klog.ErrorS(fmt.Errorf("blocked pending channel"), "Received second dial response for connection request", "connectionID", resp.ConnectID, "dialID", resp.Random) 128 | // On multiple dial responses, avoid leaking serve goroutine. 129 | return 130 | } 131 | } 132 | 133 | if resp.Error != "" { 134 | // On dial error, avoid leaking serve goroutine. 135 | return 136 | } 137 | 138 | case client.PacketType_DATA: 139 | resp := pkt.GetData() 140 | // TODO: flow control 141 | t.connsLock.RLock() 142 | conn, ok := t.conns[resp.ConnectID] 143 | t.connsLock.RUnlock() 144 | 145 | if ok { 146 | timer := time.NewTimer((time.Duration)(t.readTimeoutSeconds) * time.Second) 147 | select { 148 | case conn.readCh <- resp.Data: 149 | timer.Stop() 150 | case <-timer.C: 151 | klog.ErrorS(fmt.Errorf("timeout"), "readTimeout has been reached, the grpc connection to the proxy server will be closed", "connectionID", conn.connID, "readTimeoutSeconds", t.readTimeoutSeconds) 152 | return 153 | } 154 | } else { 155 | klog.V(1).InfoS("connection not recognized", "connectionID", resp.ConnectID) 156 | } 157 | case client.PacketType_CLOSE_RSP: 158 | resp := pkt.GetCloseResponse() 159 | t.connsLock.RLock() 160 | conn, ok := t.conns[resp.ConnectID] 161 | t.connsLock.RUnlock() 162 | 163 | if ok { 164 | close(conn.readCh) 165 | conn.closeCh <- resp.Error 166 | close(conn.closeCh) 167 | t.connsLock.Lock() 168 | delete(t.conns, resp.ConnectID) 169 | t.connsLock.Unlock() 170 | return 171 | } 172 | klog.V(1).InfoS("connection not recognized", "connectionID", resp.ConnectID) 173 | } 174 | } 175 | } 176 | 177 | // Dial connects to the address on the named network, similar to 178 | // what net.Dial does. The only supported protocol is tcp. 179 | func (t *grpcTunnel) DialContext(ctx context.Context, protocol, address string) (net.Conn, error) { 180 | if protocol != "tcp" { 181 | return nil, errors.New("protocol not supported") 182 | } 183 | 184 | random := rand.Int63() /* #nosec G404 */ 185 | resCh := make(chan dialResult, 1) 186 | t.pendingDialLock.Lock() 187 | t.pendingDial[random] = resCh 188 | t.pendingDialLock.Unlock() 189 | defer func() { 190 | t.pendingDialLock.Lock() 191 | delete(t.pendingDial, random) 192 | t.pendingDialLock.Unlock() 193 | }() 194 | 195 | req := &client.Packet{ 196 | Type: client.PacketType_DIAL_REQ, 197 | Payload: &client.Packet_DialRequest{ 198 | DialRequest: &client.DialRequest{ 199 | Protocol: protocol, 200 | Address: address, 201 | Random: random, 202 | }, 203 | }, 204 | } 205 | klog.V(5).InfoS("[tracing] send packet", "type", req.Type) 206 | 207 | err := t.stream.Send(req) 208 | if err != nil { 209 | return nil, err 210 | } 211 | 212 | klog.V(5).Infoln("DIAL_REQ sent to proxy server") 213 | 214 | c := &conn{stream: t.stream, random: random} 215 | 216 | select { 217 | case res := <-resCh: 218 | if res.err != "" { 219 | return nil, errors.New(res.err) 220 | } 221 | c.connID = res.connid 222 | c.readCh = make(chan []byte, 10) 223 | c.closeCh = make(chan string, 1) 224 | t.connsLock.Lock() 225 | t.conns[res.connid] = c 226 | t.connsLock.Unlock() 227 | case <-time.After(30 * time.Second): 228 | return nil, errors.New("dial timeout, backstop") 229 | case <-ctx.Done(): 230 | return nil, errors.New("dial timeout, context") 231 | } 232 | 233 | return c, nil 234 | } 235 | -------------------------------------------------------------------------------- /tests/agent_disconnect_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "fmt" 7 | "io/ioutil" 8 | "net" 9 | "net/http" 10 | "net/http/httptest" 11 | "net/url" 12 | "os" 13 | "testing" 14 | "time" 15 | 16 | "google.golang.org/grpc" 17 | "k8s.io/apimachinery/pkg/util/wait" 18 | "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client" 19 | ) 20 | 21 | func TestProxy_Agent_Disconnect_HTTP_Persistent_Connection(t *testing.T) { 22 | testcases := []struct { 23 | name string 24 | proxyServerFunction func() (proxy, func(), error) 25 | clientFunction func(context.Context, string, string) (*http.Client, error) 26 | }{ 27 | { 28 | name: "grpc", 29 | proxyServerFunction: runGRPCProxyServer, 30 | clientFunction: createGrpcTunnelClient, 31 | }, 32 | { 33 | name: "http-connect", 34 | proxyServerFunction: runHTTPConnProxyServer, 35 | clientFunction: createHTTPConnectClient, 36 | }, 37 | } 38 | 39 | for _, tc := range testcases { 40 | t.Run(tc.name, func(t *testing.T) { 41 | ctx := context.Background() 42 | server := httptest.NewServer(newEchoServer("hello")) 43 | defer server.Close() 44 | 45 | stopCh := make(chan struct{}) 46 | 47 | proxy, cleanup, err := tc.proxyServerFunction() 48 | if err != nil { 49 | t.Fatal(err) 50 | } 51 | defer cleanup() 52 | 53 | runAgent(proxy.agent, stopCh) 54 | 55 | // Wait for agent to register on proxy server 56 | wait.Poll(100*time.Millisecond, 5*time.Second, func() (bool, error) { 57 | 58 | ready, _ := proxy.server.Readiness.Ready() 59 | return ready, nil 60 | }) 61 | 62 | // run test client 63 | 64 | c, err := tc.clientFunction(ctx, proxy.front, server.URL) 65 | if err != nil { 66 | t.Errorf("error obtaining client: %v", err) 67 | } 68 | 69 | _, err = clientRequest(c, server.URL) 70 | 71 | if err != nil { 72 | t.Errorf("expected no error on proxy request, got %v", err) 73 | } 74 | close(stopCh) 75 | 76 | // Wait for the agent to disconnect 77 | wait.Poll(100*time.Millisecond, 5*time.Second, func() (bool, error) { 78 | ready, _ := proxy.server.Readiness.Ready() 79 | return !ready, nil 80 | }) 81 | 82 | // Reuse same client to make the request 83 | _, err = clientRequest(c, server.URL) 84 | if err == nil { 85 | t.Errorf("expect request using http persistent connections to fail after dialing on a broken connection") 86 | } else if os.IsTimeout(err) { 87 | t.Errorf("expect request using http persistent connections to fail with error use of closed network connection. Got timeout") 88 | } 89 | }) 90 | } 91 | } 92 | 93 | func TestProxy_Agent_Reconnect(t *testing.T) { 94 | testcases := []struct { 95 | name string 96 | proxyServerFunction func() (proxy, func(), error) 97 | clientFunction func(context.Context, string, string) (*http.Client, error) 98 | }{ 99 | { 100 | name: "grpc", 101 | proxyServerFunction: runGRPCProxyServer, 102 | clientFunction: createGrpcTunnelClient, 103 | }, 104 | { 105 | name: "http-connect", 106 | proxyServerFunction: runHTTPConnProxyServer, 107 | clientFunction: createHTTPConnectClient, 108 | }, 109 | } 110 | 111 | for _, tc := range testcases { 112 | t.Run(tc.name, func(t *testing.T) { 113 | ctx := context.Background() 114 | 115 | server := httptest.NewServer(newEchoServer("hello")) 116 | defer server.Close() 117 | 118 | stopCh := make(chan struct{}) 119 | 120 | proxy, cleanup, err := tc.proxyServerFunction() 121 | if err != nil { 122 | t.Fatal(err) 123 | } 124 | defer cleanup() 125 | 126 | runAgent(proxy.agent, stopCh) 127 | 128 | // Wait for agent to register on proxy server 129 | wait.Poll(100*time.Millisecond, 5*time.Second, func() (bool, error) { 130 | ready, _ := proxy.server.Readiness.Ready() 131 | return ready, nil 132 | }) 133 | 134 | // run test client 135 | 136 | c, err := tc.clientFunction(ctx, proxy.front, server.URL) 137 | if err != nil { 138 | t.Errorf("error obtaining client: %v", err) 139 | } 140 | 141 | _, err = clientRequest(c, server.URL) 142 | if err != nil { 143 | t.Errorf("expected no error on proxy request, got %v", err) 144 | } 145 | close(stopCh) 146 | 147 | // Wait for the agent to disconnect 148 | wait.Poll(100*time.Millisecond, 5*time.Second, func() (bool, error) { 149 | ready, _ := proxy.server.Readiness.Ready() 150 | return !ready, nil 151 | }) 152 | 153 | // Reconnect agent 154 | stopCh2 := make(chan struct{}) 155 | runAgent(proxy.agent, stopCh2) 156 | defer close(stopCh2) 157 | 158 | // Wait for agent to register on proxy server 159 | wait.Poll(100*time.Millisecond, 5*time.Second, func() (bool, error) { 160 | ready, _ := proxy.server.Readiness.Ready() 161 | return ready, nil 162 | }) 163 | 164 | // Proxy requests should work again after agent reconnects 165 | c2, err := tc.clientFunction(ctx, proxy.front, server.URL) 166 | if err != nil { 167 | t.Errorf("error obtaining client: %v", err) 168 | } 169 | 170 | _, err = clientRequest(c2, server.URL) 171 | 172 | if err != nil { 173 | t.Errorf("expected no error on proxy request, got %v", err) 174 | } 175 | }) 176 | } 177 | } 178 | 179 | func clientRequest(c *http.Client, addr string) ([]byte, error) { 180 | r, err := c.Get(addr) 181 | 182 | if err != nil { 183 | return nil, err 184 | } 185 | 186 | data, err := ioutil.ReadAll(r.Body) 187 | if err != nil { 188 | return nil, err 189 | } 190 | 191 | defer r.Body.Close() 192 | 193 | return data, nil 194 | } 195 | 196 | func createGrpcTunnelClient(ctx context.Context, proxyAddr, addr string) (*http.Client, error) { 197 | tunnel, err := client.CreateSingleUseGrpcTunnel(ctx, proxyAddr, grpc.WithInsecure()) 198 | if err != nil { 199 | return nil, err 200 | } 201 | 202 | c := &http.Client{ 203 | Timeout: 10 * time.Second, 204 | Transport: &http.Transport{ 205 | DialContext: tunnel.DialContext, 206 | }, 207 | } 208 | 209 | return c, nil 210 | } 211 | 212 | func createHTTPConnectClient(ctx context.Context, proxyAddr, addr string) (*http.Client, error) { 213 | conn, err := net.Dial("tcp", proxyAddr) 214 | if err != nil { 215 | return nil, err 216 | } 217 | 218 | serverURL, _ := url.Parse(addr) 219 | 220 | // Send HTTP-Connect request 221 | _, err = fmt.Fprintf(conn, "CONNECT %s HTTP/1.1\r\nHost: %s\r\n\r\n", serverURL.Host, "127.0.0.1") 222 | if err != nil { 223 | return nil, err 224 | } 225 | 226 | // Parse the HTTP response for Connect 227 | br := bufio.NewReader(conn) 228 | res, err := http.ReadResponse(br, nil) 229 | if err != nil { 230 | return nil, fmt.Errorf("reading HTTP response from CONNECT: %v", err) 231 | } 232 | if res.StatusCode != 200 { 233 | return nil, fmt.Errorf("expect 200; got %d", res.StatusCode) 234 | } 235 | if br.Buffered() > 0 { 236 | return nil, fmt.Errorf("unexpected extra buffer") 237 | } 238 | 239 | dialer := func(network, addr string) (net.Conn, error) { 240 | return conn, nil 241 | } 242 | 243 | c := &http.Client{ 244 | Timeout: 10 * time.Second, 245 | Transport: &http.Transport{ 246 | Dial: dialer, 247 | }, 248 | } 249 | 250 | return c, nil 251 | } 252 | -------------------------------------------------------------------------------- /pkg/agent/clientset.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package agent 18 | 19 | import ( 20 | "math" 21 | "sync" 22 | "time" 23 | 24 | "google.golang.org/grpc" 25 | "google.golang.org/grpc/connectivity" 26 | "k8s.io/apimachinery/pkg/util/wait" 27 | "k8s.io/klog/v2" 28 | ) 29 | 30 | // ClientSet consists of clients connected to each instance of an HA proxy server. 31 | type ClientSet struct { 32 | mu sync.Mutex //protects the clients. 33 | clients map[string]*Client // map between serverID and the client 34 | // connects to this server. 35 | 36 | agentID string // ID of this agent 37 | address string // proxy server address. Assuming HA proxy server 38 | serverCount int // number of proxy server instances, should be 1 39 | // unless it is an HA server. Initialized when the ClientSet creates 40 | // the first client. When syncForever is set, it will be the most recently seen. 41 | syncInterval time.Duration // The interval by which the agent 42 | // periodically checks that it has connections to all instances of the 43 | // proxy server. 44 | probeInterval time.Duration // The interval by which the agent 45 | // periodically checks if its connections to the proxy server is ready. 46 | syncIntervalCap time.Duration // The maximum interval 47 | // for the syncInterval to back off to when unable to connect to the proxy server 48 | 49 | dialOptions []grpc.DialOption 50 | // file path contains service account token 51 | serviceAccountTokenPath string 52 | // channel to signal shutting down the client set. Primarily for test. 53 | stopCh <-chan struct{} 54 | 55 | agentIdentifiers string // The identifiers of the agent, which will be used 56 | // by the server when choosing agent 57 | 58 | warnOnChannelLimit bool 59 | 60 | syncForever bool // Continue syncing (support dynamic server count). 61 | } 62 | 63 | func (cs *ClientSet) ClientsCount() int { 64 | cs.mu.Lock() 65 | defer cs.mu.Unlock() 66 | return len(cs.clients) 67 | } 68 | 69 | func (cs *ClientSet) HealthyClientsCount() int { 70 | cs.mu.Lock() 71 | defer cs.mu.Unlock() 72 | var count int 73 | for _, c := range cs.clients { 74 | if c.conn.GetState() == connectivity.Ready { 75 | count++ 76 | } 77 | } 78 | return count 79 | 80 | } 81 | 82 | func (cs *ClientSet) hasIDLocked(serverID string) bool { 83 | _, ok := cs.clients[serverID] 84 | return ok 85 | } 86 | 87 | func (cs *ClientSet) HasID(serverID string) bool { 88 | cs.mu.Lock() 89 | defer cs.mu.Unlock() 90 | return cs.hasIDLocked(serverID) 91 | } 92 | 93 | type DuplicateServerError struct { 94 | ServerID string 95 | } 96 | 97 | func (dse *DuplicateServerError) Error() string { 98 | return "duplicate server: " + dse.ServerID 99 | } 100 | 101 | func (cs *ClientSet) addClientLocked(serverID string, c *Client) error { 102 | if cs.hasIDLocked(serverID) { 103 | return &DuplicateServerError{ServerID: serverID} 104 | } 105 | cs.clients[serverID] = c 106 | return nil 107 | 108 | } 109 | 110 | func (cs *ClientSet) AddClient(serverID string, c *Client) error { 111 | cs.mu.Lock() 112 | defer cs.mu.Unlock() 113 | return cs.addClientLocked(serverID, c) 114 | } 115 | 116 | func (cs *ClientSet) RemoveClient(serverID string) { 117 | cs.mu.Lock() 118 | defer cs.mu.Unlock() 119 | if cs.clients[serverID] == nil { 120 | return 121 | } 122 | cs.clients[serverID].Close() 123 | delete(cs.clients, serverID) 124 | } 125 | 126 | type ClientSetConfig struct { 127 | Address string 128 | AgentID string 129 | AgentIdentifiers string 130 | SyncInterval time.Duration 131 | ProbeInterval time.Duration 132 | SyncIntervalCap time.Duration 133 | DialOptions []grpc.DialOption 134 | ServiceAccountTokenPath string 135 | WarnOnChannelLimit bool 136 | SyncForever bool 137 | } 138 | 139 | func (cc *ClientSetConfig) NewAgentClientSet(stopCh <-chan struct{}) *ClientSet { 140 | return &ClientSet{ 141 | clients: make(map[string]*Client), 142 | agentID: cc.AgentID, 143 | agentIdentifiers: cc.AgentIdentifiers, 144 | address: cc.Address, 145 | syncInterval: cc.SyncInterval, 146 | probeInterval: cc.ProbeInterval, 147 | syncIntervalCap: cc.SyncIntervalCap, 148 | dialOptions: cc.DialOptions, 149 | serviceAccountTokenPath: cc.ServiceAccountTokenPath, 150 | warnOnChannelLimit: cc.WarnOnChannelLimit, 151 | syncForever: cc.SyncForever, 152 | stopCh: stopCh, 153 | } 154 | } 155 | 156 | func (cs *ClientSet) newAgentClient() (*Client, int, error) { 157 | return newAgentClient(cs.address, cs.agentID, cs.agentIdentifiers, cs, cs.dialOptions...) 158 | } 159 | 160 | func (cs *ClientSet) resetBackoff() *wait.Backoff { 161 | return &wait.Backoff{ 162 | Steps: math.MaxInt32, 163 | Jitter: 0.1, 164 | Factor: 1.5, 165 | Duration: cs.syncInterval, 166 | Cap: cs.syncIntervalCap, 167 | } 168 | } 169 | 170 | // sync makes sure that #clients >= #proxy servers 171 | func (cs *ClientSet) sync() { 172 | defer cs.shutdown() 173 | backoff := cs.resetBackoff() 174 | var duration time.Duration 175 | for { 176 | if err := cs.connectOnce(); err != nil { 177 | if dse, ok := err.(*DuplicateServerError); ok { 178 | klog.V(4).InfoS("duplicate server", "serverID", dse.ServerID, "serverCount", cs.serverCount, "clientsCount", cs.ClientsCount()) 179 | if cs.serverCount != 0 && cs.ClientsCount() >= cs.serverCount { 180 | duration = backoff.Step() 181 | } 182 | } else { 183 | klog.ErrorS(err, "cannot connect once") 184 | duration = backoff.Step() 185 | } 186 | } else { 187 | backoff = cs.resetBackoff() 188 | duration = wait.Jitter(backoff.Duration, backoff.Jitter) 189 | } 190 | time.Sleep(duration) 191 | select { 192 | case <-cs.stopCh: 193 | return 194 | default: 195 | } 196 | } 197 | } 198 | 199 | func (cs *ClientSet) connectOnce() error { 200 | if !cs.syncForever && cs.serverCount != 0 && cs.ClientsCount() >= cs.serverCount { 201 | return nil 202 | } 203 | c, serverCount, err := cs.newAgentClient() 204 | if err != nil { 205 | return err 206 | } 207 | if cs.serverCount != 0 && cs.serverCount != serverCount { 208 | klog.V(2).InfoS("Server count change suggestion by server", 209 | "current", cs.serverCount, "serverID", c.serverID, "actual", serverCount) 210 | 211 | } 212 | cs.serverCount = serverCount 213 | if err := cs.AddClient(c.serverID, c); err != nil { 214 | if dse, ok := err.(*DuplicateServerError); ok { 215 | klog.V(4).InfoS("closing connection to duplicate server", "serverID", dse.ServerID) 216 | } else { 217 | klog.ErrorS(err, "closing connection failure when adding a client") 218 | } 219 | c.Close() 220 | return err 221 | } 222 | klog.V(2).InfoS("sync added client connecting to proxy server", "serverID", c.serverID) 223 | go c.Serve() 224 | return nil 225 | } 226 | 227 | func (cs *ClientSet) Serve() { 228 | go cs.sync() 229 | } 230 | 231 | func (cs *ClientSet) shutdown() { 232 | cs.mu.Lock() 233 | defer cs.mu.Unlock() 234 | for serverID, client := range cs.clients { 235 | client.Close() 236 | delete(cs.clients, serverID) 237 | } 238 | } 239 | -------------------------------------------------------------------------------- /pkg/server/server_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package server 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | "io" 23 | "reflect" 24 | "testing" 25 | 26 | "github.com/golang/mock/gomock" 27 | "google.golang.org/grpc/metadata" 28 | 29 | authv1 "k8s.io/api/authentication/v1" 30 | "k8s.io/apimachinery/pkg/runtime" 31 | k8sfake "k8s.io/client-go/kubernetes/fake" 32 | fakeauthenticationv1 "k8s.io/client-go/kubernetes/typed/authentication/v1/fake" 33 | k8stesting "k8s.io/client-go/testing" 34 | 35 | agentmock "sigs.k8s.io/apiserver-network-proxy/proto/agent/mocks" 36 | "sigs.k8s.io/apiserver-network-proxy/proto/header" 37 | ) 38 | 39 | func TestAgentTokenAuthenticationErrorsToken(t *testing.T) { 40 | stub := gomock.NewController(t) 41 | defer stub.Finish() 42 | 43 | ns := "test_ns" 44 | sa := "test_sa" 45 | 46 | testCases := []struct { 47 | desc string 48 | mdKey string 49 | tokens []string 50 | wantNamespace string 51 | wantServiceAccount string 52 | authenticated bool 53 | authError string 54 | tokenReviewError error 55 | wantError bool 56 | }{ 57 | { 58 | desc: "no context", 59 | wantError: true, 60 | }, 61 | { 62 | desc: "non valid metadata key", 63 | mdKey: "someKey", 64 | tokens: []string{"token1"}, 65 | wantError: true, 66 | }, 67 | { 68 | desc: "non valid token prefix", 69 | mdKey: header.AuthenticationTokenContextKey, 70 | tokens: []string{"token1"}, 71 | wantError: true, 72 | }, 73 | { 74 | desc: "multiple valid tokens", 75 | mdKey: header.AuthenticationTokenContextKey, 76 | tokens: []string{header.AuthenticationTokenContextSchemePrefix + "token1", header.AuthenticationTokenContextSchemePrefix + "token2"}, 77 | wantError: true, 78 | }, 79 | { 80 | desc: "not authenticated", 81 | authenticated: false, 82 | mdKey: header.AuthenticationTokenContextKey, 83 | tokens: []string{header.AuthenticationTokenContextSchemePrefix + "token1"}, 84 | wantNamespace: ns, 85 | wantServiceAccount: sa, 86 | wantError: true, 87 | }, 88 | { 89 | desc: "tokenReview error", 90 | authenticated: false, 91 | mdKey: header.AuthenticationTokenContextKey, 92 | tokens: []string{header.AuthenticationTokenContextSchemePrefix + "token1"}, 93 | tokenReviewError: fmt.Errorf("some error"), 94 | wantNamespace: ns, 95 | wantServiceAccount: sa, 96 | wantError: true, 97 | }, 98 | { 99 | desc: "non valid namespace", 100 | authenticated: true, 101 | mdKey: header.AuthenticationTokenContextKey, 102 | tokens: []string{header.AuthenticationTokenContextSchemePrefix + "token1"}, 103 | wantNamespace: "_" + ns, 104 | wantServiceAccount: sa, 105 | wantError: true, 106 | }, 107 | { 108 | desc: "non valid service account", 109 | authenticated: true, 110 | mdKey: header.AuthenticationTokenContextKey, 111 | tokens: []string{header.AuthenticationTokenContextSchemePrefix + "token1"}, 112 | wantNamespace: ns, 113 | wantServiceAccount: "_" + sa, 114 | wantError: true, 115 | }, 116 | { 117 | desc: "authorization succeed", 118 | authenticated: true, 119 | mdKey: header.AuthenticationTokenContextKey, 120 | tokens: []string{header.AuthenticationTokenContextSchemePrefix + "token1"}, 121 | wantNamespace: ns, 122 | wantServiceAccount: sa, 123 | }, 124 | } 125 | 126 | for _, tc := range testCases { 127 | t.Run(tc.desc, func(t *testing.T) { 128 | kcs := k8sfake.NewSimpleClientset() 129 | 130 | kcs.AuthenticationV1().(*fakeauthenticationv1.FakeAuthenticationV1).Fake.PrependReactor("create", "tokenreviews", func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) { 131 | tr := &authv1.TokenReview{ 132 | Status: authv1.TokenReviewStatus{ 133 | Authenticated: tc.authenticated, 134 | Error: tc.authError, 135 | User: authv1.UserInfo{ 136 | Username: fmt.Sprintf("system:serviceaccount:%v:%v", ns, sa), 137 | }, 138 | }, 139 | } 140 | return true, tr, tc.tokenReviewError 141 | }) 142 | 143 | var md metadata.MD 144 | for _, token := range tc.tokens { 145 | md = metadata.Join(md, metadata.Pairs(tc.mdKey, token)) 146 | } 147 | 148 | md = metadata.Join(md, metadata.Pairs(header.AgentID, "")) 149 | 150 | ctx := context.Background() 151 | defer ctx.Done() 152 | ctx = metadata.NewIncomingContext(ctx, md) 153 | conn := agentmock.NewMockAgentService_ConnectServer(stub) 154 | conn.EXPECT().Context().AnyTimes().Return(ctx) 155 | 156 | // close agent's connection if no error is expected 157 | if !tc.wantError { 158 | conn.EXPECT().SendHeader(gomock.Any()).Return(nil) 159 | conn.EXPECT().Recv().Return(nil, io.EOF) 160 | } 161 | 162 | p := NewProxyServer("", []ProxyStrategy{ProxyStrategyDefault}, 1, &AgentTokenAuthenticationOptions{ 163 | Enabled: true, 164 | KubernetesClient: kcs, 165 | AgentNamespace: tc.wantNamespace, 166 | AgentServiceAccount: tc.wantServiceAccount, 167 | }, 168 | false, 169 | ) 170 | 171 | err := p.Connect(conn) 172 | if tc.wantError { 173 | if err == nil { 174 | t.Errorf("test case expected for error") 175 | } 176 | } else { 177 | if err != nil { 178 | t.Errorf("did not expected for error but got :%v", err) 179 | } 180 | } 181 | }) 182 | } 183 | } 184 | 185 | func TestAddRemoveFrontends(t *testing.T) { 186 | agent1ConnID1 := new(ProxyClientConnection) 187 | agent1ConnID2 := new(ProxyClientConnection) 188 | agent2ConnID1 := new(ProxyClientConnection) 189 | agent2ConnID2 := new(ProxyClientConnection) 190 | agent3ConnID1 := new(ProxyClientConnection) 191 | 192 | p := NewProxyServer("", []ProxyStrategy{ProxyStrategyDefault}, 1, nil, false) 193 | p.addFrontend("agent1", int64(1), agent1ConnID1) 194 | p.removeFrontend("agent1", int64(1)) 195 | expectedFrontends := make(map[string]map[int64]*ProxyClientConnection) 196 | if e, a := expectedFrontends, p.frontends; !reflect.DeepEqual(e, a) { 197 | t.Errorf("expected %v, got %v", e, a) 198 | } 199 | 200 | p = NewProxyServer("", []ProxyStrategy{ProxyStrategyDefault}, 1, nil, false) 201 | p.addFrontend("agent1", int64(1), agent1ConnID1) 202 | p.addFrontend("agent1", int64(2), agent1ConnID2) 203 | p.addFrontend("agent2", int64(1), agent2ConnID1) 204 | p.addFrontend("agent2", int64(2), agent2ConnID2) 205 | p.addFrontend("agent3", int64(1), agent3ConnID1) 206 | p.removeFrontend("agent2", int64(1)) 207 | p.removeFrontend("agent2", int64(2)) 208 | p.removeFrontend("agent1", int64(1)) 209 | expectedFrontends = map[string]map[int64]*ProxyClientConnection{ 210 | "agent1": { 211 | int64(2): agent1ConnID2, 212 | }, 213 | "agent3": { 214 | int64(1): agent3ConnID1, 215 | }, 216 | } 217 | if e, a := expectedFrontends, p.frontends; !reflect.DeepEqual(e, a) { 218 | t.Errorf("expected %v, got %v", e, a) 219 | } 220 | } 221 | -------------------------------------------------------------------------------- /konnectivity-client/pkg/client/client_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package client 18 | 19 | import ( 20 | "bytes" 21 | "context" 22 | "errors" 23 | "testing" 24 | "time" 25 | 26 | "google.golang.org/grpc" 27 | "k8s.io/klog/v2" 28 | "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client" 29 | ) 30 | 31 | func TestDial(t *testing.T) { 32 | ctx := context.Background() 33 | s, ps := pipe() 34 | ts := testServer(ps, 100) 35 | 36 | defer ps.Close() 37 | defer s.Close() 38 | 39 | tunnel := &grpcTunnel{ 40 | stream: s, 41 | pendingDial: make(map[int64]chan<- dialResult), 42 | conns: make(map[int64]*conn), 43 | } 44 | 45 | go tunnel.serve(&fakeConn{}) 46 | go ts.serve() 47 | 48 | _, err := tunnel.DialContext(ctx, "tcp", "127.0.0.1:80") 49 | if err != nil { 50 | t.Fatalf("expect nil; got %v", err) 51 | } 52 | 53 | if ts.packets[0].Type != client.PacketType_DIAL_REQ { 54 | t.Fatalf("expect packet.type %v; got %v", client.PacketType_CLOSE_REQ, ts.packets[0].Type) 55 | } 56 | 57 | if ts.packets[0].GetDialRequest().Address != "127.0.0.1:80" { 58 | t.Errorf("expect packet.address %v; got %v", "127.0.0.1:80", ts.packets[0].GetDialRequest().Address) 59 | } 60 | } 61 | 62 | func TestData(t *testing.T) { 63 | ctx := context.Background() 64 | s, ps := pipe() 65 | ts := testServer(ps, 100) 66 | 67 | defer ps.Close() 68 | defer s.Close() 69 | 70 | tunnel := &grpcTunnel{ 71 | stream: s, 72 | pendingDial: make(map[int64]chan<- dialResult), 73 | conns: make(map[int64]*conn), 74 | } 75 | 76 | go tunnel.serve(&fakeConn{}) 77 | go ts.serve() 78 | 79 | conn, err := tunnel.DialContext(ctx, "tcp", "127.0.0.1:80") 80 | if err != nil { 81 | t.Fatalf("expect nil; got %v", err) 82 | } 83 | 84 | datas := [][]byte{ 85 | []byte("hello"), 86 | []byte(", "), 87 | []byte("world."), 88 | } 89 | 90 | // send data using conn.Write 91 | for _, data := range datas { 92 | n, err := conn.Write(data) 93 | if err != nil { 94 | t.Error(err) 95 | } 96 | if n != len(data) { 97 | t.Errorf("expect n=%d len(%q); got %d", len(data), string(data), n) 98 | } 99 | } 100 | 101 | // test server should echo data back 102 | var buf [64]byte 103 | for _, data := range datas { 104 | n, err := conn.Read(buf[:]) 105 | if err != nil { 106 | t.Error(err) 107 | } 108 | 109 | if string(buf[:n]) != "echo: "+string(data) { 110 | t.Errorf("expect 'echo: %s'; got %s", string(data), string(buf[:n])) 111 | } 112 | } 113 | 114 | // verify test server received data 115 | if ts.data.String() != "hello, world." { 116 | t.Errorf("expect server received %v; got %v", "hello, world.", ts.data.String()) 117 | } 118 | } 119 | 120 | func TestClose(t *testing.T) { 121 | ctx := context.Background() 122 | s, ps := pipe() 123 | ts := testServer(ps, 100) 124 | 125 | defer ps.Close() 126 | defer s.Close() 127 | 128 | tunnel := &grpcTunnel{ 129 | stream: s, 130 | pendingDial: make(map[int64]chan<- dialResult), 131 | conns: make(map[int64]*conn), 132 | } 133 | 134 | go tunnel.serve(&fakeConn{}) 135 | go ts.serve() 136 | 137 | conn, err := tunnel.DialContext(ctx, "tcp", "127.0.0.1:80") 138 | if err != nil { 139 | t.Fatalf("expect nil; got %v", err) 140 | } 141 | 142 | if err := conn.Close(); err != nil { 143 | t.Error(err) 144 | } 145 | 146 | if ts.packets[1].Type != client.PacketType_CLOSE_REQ { 147 | t.Fatalf("expect packet.type %v; got %v", client.PacketType_CLOSE_REQ, ts.packets[1].Type) 148 | } 149 | if ts.packets[1].GetCloseRequest().ConnectID != 100 { 150 | t.Errorf("expect connectID=100; got %d", ts.packets[1].GetCloseRequest().ConnectID) 151 | } 152 | } 153 | 154 | // TODO: Move to common testing library 155 | 156 | // fakeStream implements ProxyService_ProxyClient 157 | type fakeStream struct { 158 | grpc.ClientStream 159 | r <-chan *client.Packet 160 | w chan<- *client.Packet 161 | } 162 | 163 | type fakeConn struct { 164 | } 165 | 166 | func (f *fakeConn) Close() error { 167 | return nil 168 | } 169 | 170 | var _ clientConn = &fakeConn{} 171 | 172 | var _ client.ProxyService_ProxyClient = &fakeStream{} 173 | 174 | func pipe() (*fakeStream, *fakeStream) { 175 | r, w := make(chan *client.Packet, 2), make(chan *client.Packet, 2) 176 | s1, s2 := &fakeStream{}, &fakeStream{} 177 | s1.r, s1.w = r, w 178 | s2.r, s2.w = w, r 179 | return s1, s2 180 | } 181 | 182 | func (s *fakeStream) Send(packet *client.Packet) error { 183 | klog.V(4).InfoS("[DEBUG] send", "packet", packet) 184 | if packet == nil { 185 | return nil 186 | } 187 | s.w <- packet 188 | return nil 189 | } 190 | 191 | func (s *fakeStream) Recv() (*client.Packet, error) { 192 | select { 193 | case pkt := <-s.r: 194 | klog.V(4).InfoS("[DEBUG] recv", "packet", pkt) 195 | return pkt, nil 196 | case <-time.After(5 * time.Second): 197 | return nil, errors.New("timeout recv") 198 | } 199 | } 200 | 201 | func (s *fakeStream) Close() { 202 | close(s.w) 203 | } 204 | 205 | type proxyServer struct { 206 | t testing.T 207 | s client.ProxyService_ProxyClient 208 | handlers map[client.PacketType]handler 209 | connid int64 210 | data bytes.Buffer 211 | packets []*client.Packet 212 | } 213 | 214 | func testServer(s client.ProxyService_ProxyClient, connid int64) *proxyServer { 215 | server := &proxyServer{ 216 | s: s, 217 | connid: connid, 218 | handlers: make(map[client.PacketType]handler), 219 | packets: []*client.Packet{}, 220 | } 221 | 222 | server.handlers[client.PacketType_CLOSE_REQ] = server.handleClose 223 | server.handlers[client.PacketType_DIAL_REQ] = server.handleDial 224 | server.handlers[client.PacketType_DATA] = server.handleData 225 | 226 | return server 227 | } 228 | 229 | func (s *proxyServer) serve() { 230 | for { 231 | pkt, err := s.s.Recv() 232 | if err != nil { 233 | s.t.Error(err) 234 | return 235 | } 236 | 237 | if pkt == nil { 238 | return 239 | } 240 | 241 | if handler, ok := s.handlers[pkt.Type]; ok { 242 | if err := s.s.Send(handler(pkt)); err != nil { 243 | s.t.Error(err) 244 | } 245 | } 246 | } 247 | 248 | } 249 | 250 | func (s *proxyServer) handle(t client.PacketType, h handler) *proxyServer { 251 | s.handlers[t] = h 252 | return s 253 | } 254 | 255 | type handler func(pkt *client.Packet) *client.Packet 256 | 257 | func (s *proxyServer) handleDial(pkt *client.Packet) *client.Packet { 258 | s.packets = append(s.packets, pkt) 259 | return &client.Packet{ 260 | Type: client.PacketType_DIAL_RSP, 261 | Payload: &client.Packet_DialResponse{ 262 | DialResponse: &client.DialResponse{ 263 | Random: pkt.GetDialRequest().Random, 264 | ConnectID: s.connid, 265 | }, 266 | }, 267 | } 268 | } 269 | 270 | func (s *proxyServer) handleClose(pkt *client.Packet) *client.Packet { 271 | s.packets = append(s.packets, pkt) 272 | return &client.Packet{ 273 | Type: client.PacketType_CLOSE_RSP, 274 | Payload: &client.Packet_CloseResponse{ 275 | CloseResponse: &client.CloseResponse{ 276 | ConnectID: pkt.GetCloseRequest().ConnectID, 277 | }, 278 | }, 279 | } 280 | } 281 | 282 | func (s *proxyServer) handleData(pkt *client.Packet) *client.Packet { 283 | s.packets = append(s.packets, pkt) 284 | s.data.Write(pkt.GetData().Data) 285 | 286 | return &client.Packet{ 287 | Type: client.PacketType_DATA, 288 | Payload: &client.Packet_Data{ 289 | Data: &client.Data{ 290 | ConnectID: pkt.GetData().ConnectID, 291 | Data: append([]byte("echo: "), pkt.GetData().Data...), 292 | }, 293 | }, 294 | } 295 | } 296 | -------------------------------------------------------------------------------- /pkg/agent/client_test.go: -------------------------------------------------------------------------------- 1 | package agent 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "net/http" 7 | "net/http/httptest" 8 | "strings" 9 | "testing" 10 | "time" 11 | 12 | "google.golang.org/grpc" 13 | "k8s.io/klog/v2" 14 | "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client" 15 | "sigs.k8s.io/apiserver-network-proxy/proto/agent" 16 | ) 17 | 18 | func TestServeData_HTTP(t *testing.T) { 19 | var err error 20 | var stream agent.AgentService_ConnectClient 21 | stopCh := make(chan struct{}) 22 | testClient := &Client{ 23 | connManager: newConnectionManager(), 24 | stopCh: stopCh, 25 | } 26 | testClient.stream, stream = pipe() 27 | 28 | // Start agent 29 | go testClient.Serve() 30 | defer close(stopCh) 31 | 32 | // Start test http server as remote service 33 | expectedBody := "Hello, client" 34 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 35 | fmt.Fprint(w, expectedBody) 36 | })) 37 | defer ts.Close() 38 | 39 | // Stimulate sending KAS DIAL_REQ to (Agent) Client 40 | dialPacket := newDialPacket("tcp", ts.URL[len("http://"):], 111) 41 | err = stream.Send(dialPacket) 42 | if err != nil { 43 | t.Fatal(err.Error()) 44 | } 45 | 46 | // Expect receiving DIAL_RSP packet from (Agent) Client 47 | pkg, err := stream.Recv() 48 | if err != nil { 49 | t.Fatal(err.Error()) 50 | } 51 | if pkg == nil { 52 | t.Fatal("unexpected nil packet") 53 | } 54 | if pkg.Type != client.PacketType_DIAL_RSP { 55 | t.Errorf("expect PacketType_DIAL_RSP; got %v", pkg.Type) 56 | } 57 | dialRsp := pkg.Payload.(*client.Packet_DialResponse) 58 | connID := dialRsp.DialResponse.ConnectID 59 | if dialRsp.DialResponse.Random != 111 { 60 | t.Errorf("expect random=111; got %v", dialRsp.DialResponse.Random) 61 | } 62 | 63 | // Send Data (HTTP Request) via (Agent) Client to the test http server 64 | dataPacket := newDataPacket(connID, []byte("GET / HTTP/1.1\r\nHost: localhost\r\n\r\n")) 65 | err = stream.Send(dataPacket) 66 | if err != nil { 67 | t.Error(err.Error()) 68 | } 69 | 70 | // Expect receiving http response via (Agent) Client 71 | pkg, _ = stream.Recv() 72 | if pkg == nil { 73 | t.Fatal("unexpected nil packet") 74 | } 75 | if pkg.Type != client.PacketType_DATA { 76 | t.Errorf("expect PacketType_DATA; got %v", pkg.Type) 77 | } 78 | data := pkg.Payload.(*client.Packet_Data).Data.Data 79 | 80 | // Verify response data 81 | // 82 | // HTTP/1.1 200 OK\r\n 83 | // Date: Tue, 07 May 2019 06:44:57 GMT\r\n 84 | // Content-Length: 14\r\n 85 | // Content-Type: text/plain; charset=utf-8\r\n 86 | // \r\n 87 | // Hello, client 88 | headAndBody := strings.Split(string(data), "\r\n") 89 | if body := headAndBody[len(headAndBody)-1]; body != expectedBody { 90 | t.Errorf("expect body %v; got %v", expectedBody, body) 91 | } 92 | 93 | // Force close the test server which will cause remote connection gets droped 94 | ts.Close() 95 | 96 | // Verify receiving CLOSE_RSP 97 | pkg, _ = stream.Recv() 98 | if pkg == nil { 99 | t.Fatal("unexpected nil packet") 100 | } 101 | if pkg.Type != client.PacketType_CLOSE_RSP { 102 | t.Errorf("expect PacketType_CLOSE_RSP; got %v", pkg.Type) 103 | } 104 | closeErr := pkg.Payload.(*client.Packet_CloseResponse).CloseResponse.Error 105 | if closeErr != "" { 106 | t.Errorf("expect nil closeErr; got %v", closeErr) 107 | } 108 | 109 | // Verify internal state is consistent 110 | if _, ok := testClient.connManager.Get(connID); ok { 111 | t.Error("client.connContext not released") 112 | } 113 | } 114 | 115 | func TestClose_Client(t *testing.T) { 116 | var stream agent.AgentService_ConnectClient 117 | stopCh := make(chan struct{}) 118 | testClient := &Client{ 119 | connManager: newConnectionManager(), 120 | stopCh: stopCh, 121 | } 122 | testClient.stream, stream = pipe() 123 | 124 | // Start agent 125 | go testClient.Serve() 126 | defer close(stopCh) 127 | 128 | // Start test http server as remote service 129 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 130 | fmt.Fprint(w, "hello, world") 131 | })) 132 | defer ts.Close() 133 | 134 | // Stimulate sending KAS DIAL_REQ to (Agent) Client 135 | dialPacket := newDialPacket("tcp", ts.URL[len("http://"):], 111) 136 | err := stream.Send(dialPacket) 137 | if err != nil { 138 | t.Fatal(err) 139 | } 140 | 141 | // Expect receiving DIAL_RSP packet from (Agent) Client 142 | pkg, _ := stream.Recv() 143 | if pkg == nil { 144 | t.Fatal("unexpected nil packet") 145 | } 146 | if pkg.Type != client.PacketType_DIAL_RSP { 147 | t.Errorf("expect PacketType_DIAL_RSP; got %v", pkg.Type) 148 | } 149 | dialRsp := pkg.Payload.(*client.Packet_DialResponse) 150 | connID := dialRsp.DialResponse.ConnectID 151 | if dialRsp.DialResponse.Random != 111 { 152 | t.Errorf("expect random=111; got %v", dialRsp.DialResponse.Random) 153 | } 154 | 155 | closePacket := newClosePacket(connID) 156 | if err := stream.Send(closePacket); err != nil { 157 | t.Fatal(err) 158 | } 159 | 160 | // Expect receiving close response via (Agent) Client 161 | pkg, _ = stream.Recv() 162 | if pkg == nil { 163 | t.Error("unexpected nil packet") 164 | } 165 | if pkg.Type != client.PacketType_CLOSE_RSP { 166 | t.Errorf("expect PacketType_CLOSE_RSP; got %v", pkg.Type) 167 | } 168 | closeErr := pkg.Payload.(*client.Packet_CloseResponse).CloseResponse.Error 169 | if closeErr != "" { 170 | t.Errorf("expect nil closeErr; got %v", closeErr) 171 | } 172 | 173 | // Verify internal state is consistent 174 | if _, ok := testClient.connManager.Get(connID); ok { 175 | t.Error("client.connContext not released") 176 | } 177 | 178 | // Verify remote conn is closed 179 | if err := stream.Send(closePacket); err != nil { 180 | t.Fatal(err) 181 | } 182 | 183 | // Expect receiving close response via (Agent) Client 184 | pkg, _ = stream.Recv() 185 | if pkg == nil { 186 | t.Error("unexpected nil packet") 187 | } 188 | if pkg.Type != client.PacketType_CLOSE_RSP { 189 | t.Errorf("expect PacketType_CLOSE_RSP; got %+v", pkg) 190 | } 191 | closeErr = pkg.Payload.(*client.Packet_CloseResponse).CloseResponse.Error 192 | if closeErr != "Unknown connectID" { 193 | t.Errorf("expect Unknown connectID; got %v", closeErr) 194 | } 195 | 196 | } 197 | 198 | // fakeStream implements AgentService_ConnectClient 199 | type fakeStream struct { 200 | grpc.ClientStream 201 | r <-chan *client.Packet 202 | w chan<- *client.Packet 203 | } 204 | 205 | func pipe() (agent.AgentService_ConnectClient, agent.AgentService_ConnectClient) { 206 | r, w := make(chan *client.Packet, 2), make(chan *client.Packet, 2) 207 | s1, s2 := &fakeStream{}, &fakeStream{} 208 | s1.r, s1.w = r, w 209 | s2.r, s2.w = w, r 210 | return s1, s2 211 | } 212 | 213 | func (s *fakeStream) Send(packet *client.Packet) error { 214 | klog.V(4).InfoS("[DEBUG] send", "packet", packet) 215 | s.w <- packet 216 | return nil 217 | } 218 | 219 | func (s *fakeStream) Recv() (*client.Packet, error) { 220 | select { 221 | case pkg := <-s.r: 222 | klog.V(4).InfoS("[DEBUG] recv", "packet", pkg) 223 | return pkg, nil 224 | case <-time.After(5 * time.Second): 225 | return nil, errors.New("timeout recv") 226 | } 227 | } 228 | 229 | func newDialPacket(protocol, address string, random int64) *client.Packet { 230 | return &client.Packet{ 231 | Type: client.PacketType_DIAL_REQ, 232 | Payload: &client.Packet_DialRequest{ 233 | DialRequest: &client.DialRequest{ 234 | Protocol: protocol, 235 | Address: address, 236 | Random: random, 237 | }, 238 | }, 239 | } 240 | } 241 | 242 | func newDataPacket(connID int64, data []byte) *client.Packet { 243 | return &client.Packet{ 244 | Type: client.PacketType_DATA, 245 | Payload: &client.Packet_Data{ 246 | Data: &client.Data{ 247 | ConnectID: connID, 248 | Data: data, 249 | }, 250 | }, 251 | } 252 | } 253 | 254 | func newClosePacket(connID int64) *client.Packet { 255 | return &client.Packet{ 256 | Type: client.PacketType_CLOSE_REQ, 257 | Payload: &client.Packet_CloseRequest{ 258 | CloseRequest: &client.CloseRequest{ 259 | ConnectID: connID, 260 | }, 261 | }, 262 | } 263 | } 264 | -------------------------------------------------------------------------------- /konnectivity-client/go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 3 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= 4 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 5 | github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 6 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= 7 | github.com/go-logr/logr v0.1.0 h1:M1Tv3VzNlEHg6uyACnRdtrploV2P7wZqH8BoQMtz0cg= 8 | github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= 9 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= 10 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 11 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 12 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 13 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 14 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 15 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 16 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 17 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 18 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 19 | github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= 20 | github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= 21 | github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 22 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 23 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 24 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 25 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 26 | github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= 27 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 28 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 29 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 30 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 31 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 32 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 33 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 34 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 35 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 36 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 37 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 38 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 39 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 40 | golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA= 41 | golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= 42 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 43 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 44 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 45 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 46 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 47 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 48 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 49 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= 50 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 51 | golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= 52 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 53 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 54 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 55 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 56 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 57 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= 58 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 59 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 60 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 61 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 62 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 63 | google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= 64 | google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= 65 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 66 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 67 | google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 68 | google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk= 69 | google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 70 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 71 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 72 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 73 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 74 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 75 | google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 76 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 77 | google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 78 | google.golang.org/protobuf v1.26.0-rc.1 h1:7QnIQpGRHE5RnLKnESfDoxm2dTapTZua5a0kS0A+VXQ= 79 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 80 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 81 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 82 | k8s.io/klog/v2 v2.0.0 h1:Foj74zO6RbjjP4hBEKjnYtjjAhGg4jNynUdYF6fJrok= 83 | k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= 84 | -------------------------------------------------------------------------------- /cmd/agent/app/options/options.go: -------------------------------------------------------------------------------- 1 | package options 2 | 3 | import ( 4 | "fmt" 5 | "net/url" 6 | "os" 7 | "time" 8 | 9 | "github.com/google/uuid" 10 | "github.com/spf13/pflag" 11 | "google.golang.org/grpc" 12 | "k8s.io/klog/v2" 13 | 14 | "sigs.k8s.io/apiserver-network-proxy/pkg/agent" 15 | "sigs.k8s.io/apiserver-network-proxy/pkg/util" 16 | ) 17 | 18 | type GrpcProxyAgentOptions struct { 19 | // Configuration for authenticating with the proxy-server 20 | AgentCert string 21 | AgentKey string 22 | CaCert string 23 | 24 | // Configuration for connecting to the proxy-server 25 | ProxyServerHost string 26 | ProxyServerPort int 27 | AlpnProtos []string 28 | 29 | // Ports for the health and admin server 30 | HealthServerPort int 31 | AdminServerPort int 32 | // Enables pprof at host:adminPort/debug/pprof. 33 | EnableProfiling bool 34 | // If EnableProfiling is true, this enables the lock contention 35 | // profiling at host:adminPort/debug/pprof/block. 36 | EnableContentionProfiling bool 37 | 38 | AgentID string 39 | AgentIdentifiers string 40 | SyncInterval time.Duration 41 | ProbeInterval time.Duration 42 | SyncIntervalCap time.Duration 43 | // After a duration of this time if the agent doesn't see any activity it 44 | // pings the server to see if the transport is still alive. 45 | KeepaliveTime time.Duration 46 | 47 | // file contains service account authorization token for enabling proxy-server token based authorization 48 | ServiceAccountTokenPath string 49 | 50 | // This warns if we attempt to push onto a "full" transfer channel. 51 | // However checking that the transfer channel is full is not safe. 52 | // It violates our race condition checking. Adding locks around a potentially 53 | // blocking call has its own problems, so it cannot easily be made race condition safe. 54 | // The check is an "unlocked" read but is still use at your own peril. 55 | WarnOnChannelLimit bool 56 | 57 | SyncForever bool 58 | } 59 | 60 | func (o *GrpcProxyAgentOptions) ClientSetConfig(dialOptions ...grpc.DialOption) *agent.ClientSetConfig { 61 | return &agent.ClientSetConfig{ 62 | Address: fmt.Sprintf("%s:%d", o.ProxyServerHost, o.ProxyServerPort), 63 | AgentID: o.AgentID, 64 | AgentIdentifiers: o.AgentIdentifiers, 65 | SyncInterval: o.SyncInterval, 66 | ProbeInterval: o.ProbeInterval, 67 | SyncIntervalCap: o.SyncIntervalCap, 68 | DialOptions: dialOptions, 69 | ServiceAccountTokenPath: o.ServiceAccountTokenPath, 70 | WarnOnChannelLimit: o.WarnOnChannelLimit, 71 | SyncForever: o.SyncForever, 72 | } 73 | } 74 | 75 | func (o *GrpcProxyAgentOptions) Flags() *pflag.FlagSet { 76 | flags := pflag.NewFlagSet("proxy-agent", pflag.ContinueOnError) 77 | flags.StringVar(&o.AgentCert, "agent-cert", o.AgentCert, "If non-empty secure communication with this cert.") 78 | flags.StringVar(&o.AgentKey, "agent-key", o.AgentKey, "If non-empty secure communication with this key.") 79 | flags.StringVar(&o.CaCert, "ca-cert", o.CaCert, "If non-empty the CAs we use to validate clients.") 80 | flags.StringVar(&o.ProxyServerHost, "proxy-server-host", o.ProxyServerHost, "The hostname to use to connect to the proxy-server.") 81 | flags.IntVar(&o.ProxyServerPort, "proxy-server-port", o.ProxyServerPort, "The port the proxy server is listening on.") 82 | flags.StringSliceVar(&o.AlpnProtos, "alpn-proto", o.AlpnProtos, "Additional ALPN protocols to be presented when connecting to the server. Useful to distinguish between network proxy and apiserver connections that share the same destination address.") 83 | flags.IntVar(&o.HealthServerPort, "health-server-port", o.HealthServerPort, "The port the health server is listening on.") 84 | flags.IntVar(&o.AdminServerPort, "admin-server-port", o.AdminServerPort, "The port the admin server is listening on.") 85 | flags.BoolVar(&o.EnableProfiling, "enable-profiling", o.EnableProfiling, "enable pprof at host:admin-port/debug/pprof") 86 | flags.BoolVar(&o.EnableContentionProfiling, "enable-contention-profiling", o.EnableContentionProfiling, "enable contention profiling at host:admin-port/debug/pprof/block. \"--enable-profiling\" must also be set.") 87 | flags.StringVar(&o.AgentID, "agent-id", o.AgentID, "The unique ID of this agent. Default to a generated uuid if not set.") 88 | flags.DurationVar(&o.SyncInterval, "sync-interval", o.SyncInterval, "The initial interval by which the agent periodically checks if it has connections to all instances of the proxy server.") 89 | flags.DurationVar(&o.ProbeInterval, "probe-interval", o.ProbeInterval, "The interval by which the agent periodically checks if its connections to the proxy server are ready.") 90 | flags.DurationVar(&o.SyncIntervalCap, "sync-interval-cap", o.SyncIntervalCap, "The maximum interval for the SyncInterval to back off to when unable to connect to the proxy server") 91 | flags.DurationVar(&o.KeepaliveTime, "keepalive-time", o.KeepaliveTime, "Time for gRPC agent server keepalive.") 92 | flags.StringVar(&o.ServiceAccountTokenPath, "service-account-token-path", o.ServiceAccountTokenPath, "If non-empty proxy agent uses this token to prove its identity to the proxy server.") 93 | flags.StringVar(&o.AgentIdentifiers, "agent-identifiers", o.AgentIdentifiers, "Identifiers of the agent that will be used by the server when choosing agent. N.B. the list of identifiers must be in URL encoded format. e.g.,host=localhost&host=node1.mydomain.com&cidr=127.0.0.1/16&ipv4=1.2.3.4&ipv4=5.6.7.8&ipv6=:::::&default-route=true") 94 | flags.BoolVar(&o.WarnOnChannelLimit, "warn-on-channel-limit", o.WarnOnChannelLimit, "Turns on a warning if the system is going to push to a full channel. The check involves an unsafe read.") 95 | flags.BoolVar(&o.SyncForever, "sync-forever", o.SyncForever, "If true, the agent continues syncing, in order to support server count changes.") 96 | return flags 97 | } 98 | 99 | func (o *GrpcProxyAgentOptions) Print() { 100 | klog.V(1).Infof("AgentCert set to %q.\n", o.AgentCert) 101 | klog.V(1).Infof("AgentKey set to %q.\n", o.AgentKey) 102 | klog.V(1).Infof("CACert set to %q.\n", o.CaCert) 103 | klog.V(1).Infof("ProxyServerHost set to %q.\n", o.ProxyServerHost) 104 | klog.V(1).Infof("ProxyServerPort set to %d.\n", o.ProxyServerPort) 105 | klog.V(1).Infof("ALPNProtos set to %+s.\n", o.AlpnProtos) 106 | klog.V(1).Infof("HealthServerPort set to %d.\n", o.HealthServerPort) 107 | klog.V(1).Infof("AdminServerPort set to %d.\n", o.AdminServerPort) 108 | klog.V(1).Infof("EnableProfiling set to %v.\n", o.EnableProfiling) 109 | klog.V(1).Infof("EnableContentionProfiling set to %v.\n", o.EnableContentionProfiling) 110 | klog.V(1).Infof("AgentID set to %s.\n", o.AgentID) 111 | klog.V(1).Infof("SyncInterval set to %v.\n", o.SyncInterval) 112 | klog.V(1).Infof("ProbeInterval set to %v.\n", o.ProbeInterval) 113 | klog.V(1).Infof("SyncIntervalCap set to %v.\n", o.SyncIntervalCap) 114 | klog.V(1).Infof("Keepalive time set to %v.\n", o.KeepaliveTime) 115 | klog.V(1).Infof("ServiceAccountTokenPath set to %q.\n", o.ServiceAccountTokenPath) 116 | klog.V(1).Infof("AgentIdentifiers set to %s.\n", util.PrettyPrintURL(o.AgentIdentifiers)) 117 | klog.V(1).Infof("WarnOnChannelLimit set to %t.\n", o.WarnOnChannelLimit) 118 | klog.V(1).Infof("SyncForever set to %v.\n", o.SyncForever) 119 | } 120 | 121 | func (o *GrpcProxyAgentOptions) Validate() error { 122 | if o.AgentKey != "" { 123 | if _, err := os.Stat(o.AgentKey); os.IsNotExist(err) { 124 | return fmt.Errorf("error checking agent key %s, got %v", o.AgentKey, err) 125 | } 126 | if o.AgentCert == "" { 127 | return fmt.Errorf("cannot have agent cert empty when agent key is set to \"%s\"", o.AgentKey) 128 | } 129 | } 130 | if o.AgentCert != "" { 131 | if _, err := os.Stat(o.AgentCert); os.IsNotExist(err) { 132 | return fmt.Errorf("error checking agent cert %s, got %v", o.AgentCert, err) 133 | } 134 | if o.AgentKey == "" { 135 | return fmt.Errorf("cannot have agent key empty when agent cert is set to \"%s\"", o.AgentCert) 136 | } 137 | } 138 | if o.CaCert != "" { 139 | if _, err := os.Stat(o.CaCert); os.IsNotExist(err) { 140 | return fmt.Errorf("error checking agent CA cert %s, got %v", o.CaCert, err) 141 | } 142 | } 143 | if o.ProxyServerPort <= 0 { 144 | return fmt.Errorf("proxy server port %d must be greater than 0", o.ProxyServerPort) 145 | } 146 | if o.HealthServerPort <= 0 { 147 | return fmt.Errorf("health server port %d must be greater than 0", o.HealthServerPort) 148 | } 149 | if o.AdminServerPort <= 0 { 150 | return fmt.Errorf("admin server port %d must be greater than 0", o.AdminServerPort) 151 | } 152 | if o.EnableContentionProfiling && !o.EnableProfiling { 153 | return fmt.Errorf("if --enable-contention-profiling is set, --enable-profiling must also be set") 154 | } 155 | if o.SyncInterval > o.SyncIntervalCap { 156 | return fmt.Errorf("sync interval %v must be less than sync interval cap %v", o.SyncInterval, o.SyncIntervalCap) 157 | } 158 | if o.ServiceAccountTokenPath != "" { 159 | if _, err := os.Stat(o.ServiceAccountTokenPath); os.IsNotExist(err) { 160 | return fmt.Errorf("error checking service account token path %s, got %v", o.ServiceAccountTokenPath, err) 161 | } 162 | } 163 | if err := validateAgentIdentifiers(o.AgentIdentifiers); err != nil { 164 | return fmt.Errorf("agent address is invalid: %v", err) 165 | } 166 | return nil 167 | } 168 | 169 | func validateAgentIdentifiers(agentIdentifiers string) error { 170 | decoded, err := url.ParseQuery(agentIdentifiers) 171 | if err != nil { 172 | return err 173 | } 174 | for idType := range decoded { 175 | switch agent.IdentifierType(idType) { 176 | case agent.IPv4: 177 | case agent.IPv6: 178 | case agent.CIDR: 179 | case agent.Host: 180 | case agent.DefaultRoute: 181 | default: 182 | return fmt.Errorf("unknown address type: %s", idType) 183 | } 184 | } 185 | return nil 186 | } 187 | 188 | func NewGrpcProxyAgentOptions() *GrpcProxyAgentOptions { 189 | o := GrpcProxyAgentOptions{ 190 | AgentCert: "", 191 | AgentKey: "", 192 | CaCert: "", 193 | ProxyServerHost: "127.0.0.1", 194 | ProxyServerPort: 8091, 195 | HealthServerPort: 8093, 196 | AdminServerPort: 8094, 197 | EnableProfiling: false, 198 | EnableContentionProfiling: false, 199 | AgentID: uuid.New().String(), 200 | AgentIdentifiers: "", 201 | SyncInterval: 1 * time.Second, 202 | ProbeInterval: 1 * time.Second, 203 | SyncIntervalCap: 10 * time.Second, 204 | KeepaliveTime: 1 * time.Hour, 205 | ServiceAccountTokenPath: "", 206 | WarnOnChannelLimit: false, 207 | SyncForever: false, 208 | } 209 | return &o 210 | } 211 | -------------------------------------------------------------------------------- /pkg/server/backend_manager.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package server 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | "math/rand" 23 | "strings" 24 | "sync" 25 | "time" 26 | 27 | "k8s.io/klog/v2" 28 | client "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client" 29 | pkgagent "sigs.k8s.io/apiserver-network-proxy/pkg/agent" 30 | "sigs.k8s.io/apiserver-network-proxy/pkg/server/metrics" 31 | "sigs.k8s.io/apiserver-network-proxy/proto/agent" 32 | ) 33 | 34 | type ProxyStrategy string 35 | 36 | const ( 37 | // With this strategy the Proxy Server will randomly pick a backend from 38 | // the current healthy backends to establish the tunnel over which to 39 | // forward requests. 40 | ProxyStrategyDefault ProxyStrategy = "default" 41 | // With this strategy the Proxy Server will pick a backend that has the same 42 | // associated host as the request.Host to establish the tunnel. 43 | ProxyStrategyDestHost ProxyStrategy = "destHost" 44 | 45 | // ProxyStrategyDefaultRoute will only forward traffic to agents that have explicity advertised 46 | // they serve the default route through an agent identifier. Typically used in combination with destHost 47 | ProxyStrategyDefaultRoute ProxyStrategy = "defaultRoute" 48 | ) 49 | 50 | // GenProxyStrategiesFromStr generates the list of proxy strategies from the 51 | // comma-seperated string, i.e., destHost. 52 | func GenProxyStrategiesFromStr(proxyStrategies string) ([]ProxyStrategy, error) { 53 | var ps []ProxyStrategy 54 | strs := strings.Split(proxyStrategies, ",") 55 | for _, s := range strs { 56 | switch s { 57 | case string(ProxyStrategyDestHost): 58 | ps = append(ps, ProxyStrategyDestHost) 59 | case string(ProxyStrategyDefault): 60 | ps = append(ps, ProxyStrategyDefault) 61 | case string(ProxyStrategyDefaultRoute): 62 | ps = append(ps, ProxyStrategyDefaultRoute) 63 | default: 64 | return nil, fmt.Errorf("Unknown proxy strategy %s", s) 65 | } 66 | } 67 | return ps, nil 68 | } 69 | 70 | type Backend interface { 71 | Send(p *client.Packet) error 72 | Context() context.Context 73 | } 74 | 75 | var _ Backend = &backend{} 76 | var _ Backend = agent.AgentService_ConnectServer(nil) 77 | 78 | type backend struct { 79 | // TODO: this is a multi-writer single-reader pattern, it's tricky to 80 | // write it using channel. Let's worry about performance later. 81 | mu sync.Mutex // mu protects conn 82 | conn agent.AgentService_ConnectServer 83 | } 84 | 85 | func (b *backend) Send(p *client.Packet) error { 86 | b.mu.Lock() 87 | defer b.mu.Unlock() 88 | return b.conn.Send(p) 89 | } 90 | 91 | func (b *backend) Context() context.Context { 92 | // TODO: does Context require lock protection? 93 | return b.conn.Context() 94 | } 95 | 96 | func newBackend(conn agent.AgentService_ConnectServer) *backend { 97 | return &backend{conn: conn} 98 | } 99 | 100 | // BackendStorage is an interface to manage the storage of the backend 101 | // connections, i.e., get, add and remove 102 | type BackendStorage interface { 103 | // AddBackend adds a backend. 104 | AddBackend(identifier string, idType pkgagent.IdentifierType, conn agent.AgentService_ConnectServer) Backend 105 | // RemoveBackend removes a backend. 106 | RemoveBackend(identifier string, idType pkgagent.IdentifierType, conn agent.AgentService_ConnectServer) 107 | // NumBackends returns the number of backends. 108 | NumBackends() int 109 | } 110 | 111 | // BackendManager is an interface to manage backend connections, i.e., 112 | // connection to the proxy agents. 113 | type BackendManager interface { 114 | // Backend returns a single backend. 115 | // WARNING: the context passed to the function should be a session-scoped 116 | // context instead of a request-scoped context, as the backend manager will 117 | // pick a backend for every tunnel session and each tunnel session may 118 | // contains multiple requests. 119 | Backend(ctx context.Context) (Backend, error) 120 | BackendStorage 121 | ReadinessManager 122 | } 123 | 124 | var _ BackendManager = &DefaultBackendManager{} 125 | 126 | // DefaultBackendManager is the default backend manager. 127 | type DefaultBackendManager struct { 128 | *DefaultBackendStorage 129 | } 130 | 131 | func (dbm *DefaultBackendManager) Backend(_ context.Context) (Backend, error) { 132 | klog.V(5).InfoS("Get a random backend through the DefaultBackendManager") 133 | return dbm.DefaultBackendStorage.GetRandomBackend() 134 | } 135 | 136 | // DefaultBackendStorage is the default backend storage. 137 | type DefaultBackendStorage struct { 138 | mu sync.RWMutex //protects the following 139 | // A map between agentID and its grpc connections. 140 | // For a given agent, ProxyServer prefers backends[agentID][0] to send 141 | // traffic, because backends[agentID][1:] are more likely to be closed 142 | // by the agent to deduplicate connections to the same server. 143 | backends map[string][]*backend 144 | // agentID is tracked in this slice to enable randomly picking an 145 | // agentID in the Backend() method. There is no reliable way to 146 | // randomly pick a key from a map (in this case, the backends) in 147 | // Golang. 148 | agentIDs []string 149 | // defaultRouteAgentIDs tracks the agents that have claimed the default route. 150 | defaultRouteAgentIDs []string 151 | random *rand.Rand 152 | // idTypes contains the valid identifier types for this 153 | // DefaultBackendStorage. The DefaultBackendStorage may only tolerate certain 154 | // types of identifiers when associating to a specific BackendManager, 155 | // e.g., when associating to the DestHostBackendManager, it can only use the 156 | // identifiers of types, IPv4, IPv6 and Host. 157 | idTypes []pkgagent.IdentifierType 158 | } 159 | 160 | // NewDefaultBackendManager returns a DefaultBackendManager. 161 | func NewDefaultBackendManager() *DefaultBackendManager { 162 | return &DefaultBackendManager{ 163 | DefaultBackendStorage: NewDefaultBackendStorage( 164 | []pkgagent.IdentifierType{pkgagent.UID})} 165 | } 166 | 167 | // NewDefaultBackendStorage returns a DefaultBackendStorage 168 | func NewDefaultBackendStorage(idTypes []pkgagent.IdentifierType) *DefaultBackendStorage { 169 | return &DefaultBackendStorage{ 170 | backends: make(map[string][]*backend), 171 | random: rand.New(rand.NewSource(time.Now().UnixNano())), 172 | idTypes: idTypes, 173 | } /* #nosec G404 */ 174 | } 175 | 176 | func containIDType(idTypes []pkgagent.IdentifierType, idType pkgagent.IdentifierType) bool { 177 | for _, it := range idTypes { 178 | if it == idType { 179 | return true 180 | } 181 | } 182 | return false 183 | } 184 | 185 | // AddBackend adds a backend. 186 | func (s *DefaultBackendStorage) AddBackend(identifier string, idType pkgagent.IdentifierType, conn agent.AgentService_ConnectServer) Backend { 187 | if !containIDType(s.idTypes, idType) { 188 | klog.V(4).InfoS("fail to add backend", "backend", identifier, "error", &ErrWrongIDType{idType, s.idTypes}) 189 | return nil 190 | } 191 | klog.V(2).InfoS("Register backend for agent", "connection", conn, "agentID", identifier) 192 | s.mu.Lock() 193 | defer s.mu.Unlock() 194 | _, ok := s.backends[identifier] 195 | addedBackend := newBackend(conn) 196 | if ok { 197 | for _, v := range s.backends[identifier] { 198 | if v.conn == conn { 199 | klog.V(1).InfoS("This should not happen. Adding existing backend for agent", "connection", conn, "agentID", identifier) 200 | return v 201 | } 202 | } 203 | s.backends[identifier] = append(s.backends[identifier], addedBackend) 204 | return addedBackend 205 | } 206 | s.backends[identifier] = []*backend{addedBackend} 207 | metrics.Metrics.SetBackendCount(len(s.backends)) 208 | s.agentIDs = append(s.agentIDs, identifier) 209 | if idType == pkgagent.DefaultRoute { 210 | s.defaultRouteAgentIDs = append(s.defaultRouteAgentIDs, identifier) 211 | } 212 | return addedBackend 213 | } 214 | 215 | // RemoveBackend removes a backend. 216 | func (s *DefaultBackendStorage) RemoveBackend(identifier string, idType pkgagent.IdentifierType, conn agent.AgentService_ConnectServer) { 217 | if !containIDType(s.idTypes, idType) { 218 | klog.ErrorS(&ErrWrongIDType{idType, s.idTypes}, "fail to remove backend") 219 | return 220 | } 221 | klog.V(2).InfoS("Remove connection for agent", "connection", conn, "identifier", identifier) 222 | s.mu.Lock() 223 | defer s.mu.Unlock() 224 | backends, ok := s.backends[identifier] 225 | if !ok { 226 | klog.V(1).InfoS("Cannot find agent in backends", "identifier", identifier) 227 | return 228 | } 229 | var found bool 230 | for i, c := range backends { 231 | if c.conn == conn { 232 | s.backends[identifier] = append(s.backends[identifier][:i], s.backends[identifier][i+1:]...) 233 | if i == 0 && len(s.backends[identifier]) != 0 { 234 | klog.V(1).InfoS("This should not happen. Removed connection that is not the first connection", "connection", conn, "remainingConnections", s.backends[identifier]) 235 | } 236 | found = true 237 | } 238 | } 239 | if len(s.backends[identifier]) == 0 { 240 | delete(s.backends, identifier) 241 | for i := range s.agentIDs { 242 | if s.agentIDs[i] == identifier { 243 | s.agentIDs[i] = s.agentIDs[len(s.agentIDs)-1] 244 | s.agentIDs = s.agentIDs[:len(s.agentIDs)-1] 245 | break 246 | } 247 | } 248 | if idType == pkgagent.DefaultRoute { 249 | for i := range s.defaultRouteAgentIDs { 250 | if s.defaultRouteAgentIDs[i] == identifier { 251 | s.defaultRouteAgentIDs = append(s.defaultRouteAgentIDs[:i], s.defaultRouteAgentIDs[i+1:]...) 252 | break 253 | } 254 | } 255 | } 256 | } 257 | if !found { 258 | klog.V(1).InfoS("Could not find connection matching identifier to remove", "connection", conn, "identifier", identifier) 259 | } 260 | metrics.Metrics.SetBackendCount(len(s.backends)) 261 | } 262 | 263 | // NumBackends resturns the number of available backends 264 | func (s *DefaultBackendStorage) NumBackends() int { 265 | s.mu.RLock() 266 | defer s.mu.RUnlock() 267 | return len(s.backends) 268 | } 269 | 270 | // ErrNotFound indicates that no backend can be found. 271 | type ErrNotFound struct{} 272 | 273 | // Error returns the error message. 274 | func (e *ErrNotFound) Error() string { 275 | return "No backend available" 276 | } 277 | 278 | type ErrWrongIDType struct { 279 | got pkgagent.IdentifierType 280 | expect []pkgagent.IdentifierType 281 | } 282 | 283 | func (e *ErrWrongIDType) Error() string { 284 | return fmt.Sprintf("incorrect id type: got %s, expect %s", e.got, e.expect) 285 | } 286 | 287 | func ignoreNotFound(err error) error { 288 | if _, ok := err.(*ErrNotFound); ok { 289 | return nil 290 | } 291 | return err 292 | } 293 | 294 | // GetRandomBackend returns a random backend connection from all connected agents. 295 | func (s *DefaultBackendStorage) GetRandomBackend() (Backend, error) { 296 | s.mu.RLock() 297 | defer s.mu.RUnlock() 298 | if len(s.backends) == 0 { 299 | return nil, &ErrNotFound{} 300 | } 301 | agentID := s.agentIDs[s.random.Intn(len(s.agentIDs))] 302 | klog.V(4).InfoS("Pick agent as backend", "agentID", agentID) 303 | // always return the first connection to an agent, because the agent 304 | // will close later connections if there are multiple. 305 | return s.backends[agentID][0], nil 306 | } 307 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | --------------------------------------------------------------------------------